mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-04-21 21:32:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

bitbake: bitbake: siggen: Pass all data caches to hash functions

Browse Source 
Passing all the data caches to the task hashing functions allows them to
correctly account for mcdepends in task signatures. This allows tasks to
be correctly re-run when a mcdepends changes.

By default, the legacy behavior is maintained for derived signature
generators by passing a special proxy object that can either be used to
access all multiconfigs or the legacy behavior. If a derived signature
generator is updated, it can set the supports_multiconfig_datacaces
property to instruct bitbake it deals with multiconfigs properly.

[YOCTO #13724]

(Bitbake rev: 8ff9203de4fce9c104c2987d86980c9f34036b97)

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Joshua Watt
2020-06-09 13:34:14 -05:00
committed by Richard Purdie
parent d84136cb82
commit bf69f30b4b
2 changed files with 67 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bitbake/lib/bb/runqueue.py
View File

@@ -1190,8 +1190,9 @@ class RunQueueData:
return len(self.runtaskentries)
def prepare_task_hash(self, tid):
bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches[mc_from_tid(tid)])
self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches[mc_from_tid(tid)])
dc = bb.parse.siggen.get_data_caches(self.dataCaches, mc_from_tid(tid))
bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, dc)
self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, dc)
self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
def dump_data(self):
@@ -2305,7 +2306,8 @@ class RunQueueExecute:
if len(self.rqdata.runtaskentries[p].depends) and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
continue
orighash = self.rqdata.runtaskentries[tid].hash
newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches[mc_from_tid(tid)])
dc = bb.parse.siggen.get_data_caches(self.rqdata.dataCaches, mc_from_tid(tid))
newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, dc)
origuni = self.rqdata.runtaskentries[tid].unihash
newuni = bb.parse.siggen.get_unihash(tid)
# FIXME, need to check it can come from sstate at all for determinism?

78
bitbake/lib/bb/siggen.py
View File

@@ -38,6 +38,11 @@ class SignatureGenerator(object):
"""
name = "noop"
# If the derived class supports multiconfig datacaches, set this to True
# The default is False for backward compatibility with derived signature
# generators that do not understand multiconfig caches
supports_multiconfig_datacaches = False
def __init__(self, data):
self.basehash = {}
self.taskhash = {}
@@ -58,10 +63,10 @@ class SignatureGenerator(object):
def get_unihash(self, tid):
return self.taskhash[tid]
def prep_taskhash(self, tid, deps, dataCache):
def prep_taskhash(self, tid, deps, dataCaches):
return
def get_taskhash(self, tid, deps, dataCache):
def get_taskhash(self, tid, deps, dataCaches):
self.taskhash[tid] = hashlib.sha256(tid.encode("utf-8")).hexdigest()
return self.taskhash[tid]
@@ -105,6 +110,38 @@ class SignatureGenerator(object):
def set_setscene_tasks(self, setscene_tasks):
return
@classmethod
def get_data_caches(cls, dataCaches, mc):
"""
This function returns the datacaches that should be passed to signature
generator functions. If the signature generator supports multiconfig
caches, the entire dictionary of data caches is sent, otherwise a
special proxy is sent that support both index access to all
multiconfigs, and also direct access for the default multiconfig.
The proxy class allows code in this class itself to always use
multiconfig aware code (to ease maintenance), but derived classes that
are unaware of multiconfig data caches can still access the default
multiconfig as expected.
Do not override this function in derived classes; it will be removed in
the future when support for multiconfig data caches is mandatory
"""
class DataCacheProxy(object):
def __init__(self):
pass
def __getitem__(self, key):
return dataCaches[key]
def __getattr__(self, name):
return getattr(dataCaches[mc], name)
if cls.supports_multiconfig_datacaches:
return dataCaches
return DataCacheProxy()
class SignatureGeneratorBasic(SignatureGenerator):
"""
"""
@@ -200,7 +237,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
self.lookupcache = {}
self.taskdeps = {}
def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
def rundep_check(self, fn, recipename, task, dep, depname, dataCaches):
# Return True if we should keep the dependency, False to drop it
# We only manipulate the dependencies for packages not in the whitelist
if self.twl and not self.twl.search(recipename):
@@ -218,37 +255,40 @@ class SignatureGeneratorBasic(SignatureGenerator):
pass
return taint
def prep_taskhash(self, tid, deps, dataCache):
def prep_taskhash(self, tid, deps, dataCaches):
(mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
self.basehash[tid] = dataCache.basetaskhash[tid]
self.basehash[tid] = dataCaches[mc].basetaskhash[tid]
self.runtaskdeps[tid] = []
self.file_checksum_values[tid] = []
recipename = dataCache.pkg_fn[fn]
recipename = dataCaches[mc].pkg_fn[fn]
self.tidtopn[tid] = recipename
for dep in sorted(deps, key=clean_basepath):
(depmc, _, deptaskname, depfn) = bb.runqueue.split_tid_mcfn(dep)
if mc != depmc:
(depmc, _, _, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
depname = dataCaches[depmc].pkg_fn[depmcfn]
if not self.supports_multiconfig_datacaches and mc != depmc:
# If the signature generator doesn't understand multiconfig
# data caches, any dependency not in the same multiconfig must
# be skipped for backward compatibility
continue
depname = dataCache.pkg_fn[depfn]
if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
if not self.rundep_check(fn, recipename, task, dep, depname, dataCaches):
continue
if dep not in self.taskhash:
bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
self.runtaskdeps[tid].append(dep)
if task in dataCache.file_checksums[fn]:
if task in dataCaches[mc].file_checksums[fn]:
if self.checksum_cache:
checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude)
checksums = self.checksum_cache.get_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
else:
checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude)
checksums = bb.fetch2.get_file_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
for (f,cs) in checksums:
self.file_checksum_values[tid].append((f,cs))
taskdep = dataCache.task_deps[fn]
taskdep = dataCaches[mc].task_deps[fn]
if 'nostamp' in taskdep and task in taskdep['nostamp']:
# Nostamp tasks need an implicit taint so that they force any dependent tasks to run
if tid in self.taints and self.taints[tid].startswith("nostamp:"):
@@ -259,14 +299,14 @@ class SignatureGeneratorBasic(SignatureGenerator):
taint = str(uuid.uuid4())
self.taints[tid] = "nostamp:" + taint
taint = self.read_taint(fn, task, dataCache.stamp[fn])
taint = self.read_taint(fn, task, dataCaches[mc].stamp[fn])
if taint:
self.taints[tid] = taint
logger.warning("%s is tainted from a forced run" % tid)
return
def get_taskhash(self, tid, deps, dataCache):
def get_taskhash(self, tid, deps, dataCaches):
data = self.basehash[tid]
for dep in self.runtaskdeps[tid]:
@@ -640,6 +680,12 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG
self.server = data.getVar('BB_HASHSERVE')
self.method = "sstate_output_hash"
#
# Dummy class used for bitbake-selftest
#
class SignatureGeneratorTestMulticonfigDepends(SignatureGeneratorBasicHash):
name = "TestMulticonfigDepends"
supports_multiconfig_datacaches = True
def dump_this_task(outfile, d):
import bb.parse