mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-03-11 09:49:41 +01:00
Code Issues Packages Projects Releases Wiki Activity

sstate: Parallelise checkstatus calls for sstate mirror

Browse Source 
Currently the urls are checked serially which is a performance bottleneck when
looking at http:// urls in particular. This adds code to check the url status in
parallel, mirroring the way we do this elsewhere.

We need the datastore for the fetcher so we use threads, not multiprocess.

(From OE-Core rev: 77c4865bbde4cd2a061cf333f9ad798afc6de0ef)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2015-06-01 22:16:17 +01:00
parent 5331bbc1ac
commit 1f839f233e
1 changed files with 20 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
meta/classes/sstate.bbclass
View File

@@ -715,20 +715,16 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False):
if localdata.getVar('BB_NO_NETWORK', True) == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK', True) == "1":
localdata.delVar('BB_NO_NETWORK')
for task in range(len(sq_fn)):
if task in ret:
continue
spec, extrapath, tname = getpathcomponents(task, d)
sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + extension)
def checkstatus(arg):
(task, sstatefile) = arg
localdata2 = bb.data.createCopy(localdata)
srcuri = "file://" + sstatefile
localdata.setVar('SRC_URI', srcuri)
bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
try:
fetcher = bb.fetch2.Fetch(srcuri.split(), localdata)
fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2)
fetcher.checkstatus()
bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
ret.append(task)
@@ -739,6 +735,22 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False):
bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
pass
tasklist = []
for task in range(len(sq_fn)):
if task in ret:
continue
spec, extrapath, tname = getpathcomponents(task, d)
sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + extension)
tasklist.append((task, sstatefile))
if tasklist:
import multiprocessing
nproc = min(multiprocessing.cpu_count(), len(tasklist))
pool = oe.utils.ThreadedPool(nproc)
for t in tasklist:
pool.add_task(checkstatus, t)
pool.wait_completion()
inheritlist = d.getVar("INHERIT", True)
if "toaster" in inheritlist:
evdata = {'missed': [], 'found': []};