diff --git a/bitbake/lib/bb/main.py b/bitbake/lib/bb/main.py index f1ea7859da..ed3e1ede61 100755 --- a/bitbake/lib/bb/main.py +++ b/bitbake/lib/bb/main.py @@ -395,6 +395,11 @@ def setup_bitbake(configParams, extrafeatures=None): # In status only mode there are no logs and no UI logger.addHandler(handler) + if configParams.dump_signatures: + if extrafeatures is None: + extrafeatures = [] + extrafeatures.append(bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO) + if configParams.server_only: featureset = [] ui_module = None diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 991aa94f67..61cb9f4c95 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py @@ -1608,28 +1608,28 @@ class RunQueue: else: self.rqexe.finish() - def rq_dump_sigfn(self, fn, options): - mc = bb.runqueue.mc_from_tid(fn) - the_data = self.cooker.databuilder.parseRecipe(fn, self.cooker.collections[mc].get_file_appends(fn)) - siggen = bb.parse.siggen - dataCaches = self.rqdata.dataCaches - siggen.dump_sigfn(fn, dataCaches, options) + def _rq_dump_sigtid(self, tids): + for tid in tids: + (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) + dataCaches = self.rqdata.dataCaches + bb.parse.siggen.dump_sigtask(taskfn, taskname, dataCaches[mc].stamp[taskfn], True) def dump_signatures(self, options): - fns = set() - bb.note("Reparsing files to collect dependency data") + if bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO not in self.cooker.featureset: + bb.fatal("The dump signatures functionality needs the RECIPE_SIGGEN_INFO feature enabled") - for tid in self.rqdata.runtaskentries: - fn = fn_from_tid(tid) - fns.add(fn) + bb.note("Writing task signature files") max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1) + def chunkify(l, n): + return [l[i::n] for i in range(n)] + tids = chunkify(list(self.rqdata.runtaskentries), max_process) # We cannot use the real multiprocessing.Pool easily due to some local data # that can't be pickled. This is a cheap multi-process solution. launched = [] - while fns: + while tids: if len(launched) < max_process: - p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options)) + p = Process(target=self._rq_dump_sigtid, args=(tids.pop(), )) p.start() launched.append(p) for q in launched: diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index e57f1ffe69..6b73843c67 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -335,8 +335,8 @@ class SignatureGeneratorBasic(SignatureGenerator): self.unihash_cache.copyfile(targetdir) def dump_sigtask(self, fn, task, stampbase, runtime): - tid = fn + ":" + task + mc = bb.runqueue.mc_from_tid(fn) referencestamp = stampbase if isinstance(runtime, str) and runtime.startswith("customfile"): sigfile = stampbase @@ -353,16 +353,27 @@ class SignatureGeneratorBasic(SignatureGenerator): data['task'] = task data['basehash_ignore_vars'] = self.basehash_ignore_vars data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks - data['taskdeps'] = self.taskdeps[fn][task] + if hasattr(self, "datacaches"): + data['taskdeps'] = self.datacaches[mc].siggen_taskdeps[fn][task] + else: + data['taskdeps'] = self.taskdeps[fn][task] data['basehash'] = self.basehash[tid] data['gendeps'] = {} data['varvals'] = {} - data['varvals'][task] = self.lookupcache[fn][task] - for dep in self.taskdeps[fn][task]: - if dep in self.basehash_ignore_vars: - continue - data['gendeps'][dep] = self.gendeps[fn][dep] - data['varvals'][dep] = self.lookupcache[fn][dep] + if hasattr(self, "datacaches"): + data['varvals'][task] = self.datacaches[mc].siggen_varvals[fn][task] + for dep in self.datacaches[mc].siggen_taskdeps[fn][task]: + if dep in self.basehash_ignore_vars: + continue + data['gendeps'][dep] = self.datacaches[mc].siggen_gendeps[fn][dep] + data['varvals'][dep] = self.datacaches[mc].siggen_varvals[fn][dep] + else: + data['varvals'][task] = self.lookupcache[fn][task] + for dep in self.taskdeps[fn][task]: + if dep in self.basehash_ignore_vars: + continue + data['gendeps'][dep] = self.gendeps[fn][dep] + data['varvals'][dep] = self.lookupcache[fn][dep] if runtime and tid in self.taskhash: data['runtaskdeps'] = self.runtaskdeps[tid] @@ -409,18 +420,6 @@ class SignatureGeneratorBasic(SignatureGenerator): pass raise err - def dump_sigfn(self, fn, dataCaches, options): - if fn in self.taskdeps: - for task in self.taskdeps[fn]: - tid = fn + ":" + task - mc = bb.runqueue.mc_from_tid(tid) - if tid not in self.taskhash: - continue - if dataCaches[mc].basetaskhash[tid] != self.basehash[tid]: - bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % tid) - bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[tid], self.basehash[tid])) - self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True) - class SignatureGeneratorBasicHash(SignatureGeneratorBasic): name = "basichash"