mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-09-14 12:49:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

bitbake: cooker: Split recipes to parse amongst threads ahead of time

Browse Source 
We have two choices, split the recipes amongst the parsing threads in
blocks ahead of time, or have a queue which parsers pull from when idle.

The optimum approach depends on how similar the pieces are. For the single
recipe reparse case, there is currently a significant wait for the feeder
thread to start (around 0.25s in a 2s command).

Its possible splitting into blocks in advance may be unluckly for some other
workloads but experimentally it seems to work better overall for me at least.

(Bitbake rev: ae79868861568d673a70472e85a4bde9e2d84a8f)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2018-12-28 16:40:33 +00:00
parent 77c3b26533
commit d4036c6fcf
1 changed files with 6 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
bitbake/lib/bb/cooker.py
View File

@@ -1941,11 +1941,8 @@ class Parser(multiprocessing.Process):
result = pending.pop() result = pending.pop()
else: else:
try: try:
job = self.jobs.get(timeout=0.25) job = self.jobs.pop()
except queue.Empty: except IndexError:
continue
if job is None:
break break
result = self.parse(*job) result = self.parse(*job)
@@ -2032,12 +2029,12 @@ class CookerParser(object):
self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
self.result_queue = multiprocessing.Queue() self.result_queue = multiprocessing.Queue()
self.jobs = multiprocessing.Queue() def chunkify(lst,n):
for j in self.willparse: return [lst[i::n] for i in range(n)]
self.jobs.put(j) self.jobs = chunkify(self.willparse, self.num_processes)
for i in range(0, self.num_processes): for i in range(0, self.num_processes):
parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
parser.start() parser.start()
self.process_names.append(parser.name) self.process_names.append(parser.name)
self.processes.append(parser) self.processes.append(parser)
@@ -2065,8 +2062,6 @@ class CookerParser(object):
for process in self.processes: for process in self.processes:
self.parser_quit.put(None) self.parser_quit.put(None)
self.jobs.cancel_join_thread()
for process in self.processes: for process in self.processes:
if force: if force:
process.join(.1) process.join(.1)