diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index d083c515ed..09691d98cc 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py @@ -30,7 +30,7 @@ import os import logging -from collections import defaultdict, namedtuple +from collections import defaultdict import bb.data import bb.utils @@ -45,46 +45,11 @@ except ImportError: __cache_version__ = "138" -recipe_fields = ( - 'pn', - 'pv', - 'pr', - 'pe', - 'defaultpref', - 'depends', - 'provides', - 'task_deps', - 'stamp', - 'stamp_extrainfo', - 'broken', - 'not_world', - 'skipped', - 'timestamp', - 'packages', - 'packages_dynamic', - 'rdepends', - 'rdepends_pkg', - 'rprovides', - 'rprovides_pkg', - 'rrecommends', - 'rrecommends_pkg', - 'nocache', - 'variants', - 'file_depends', - 'tasks', - 'basetaskhashes', - 'hashfilename', - 'inherits', - 'summary', - 'license', - 'section', - 'fakerootenv', - 'fakerootdirs' -) - -class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)): - __slots__ = () +# RecipeInfoCommon defines common data retrieving methods +# from meta data for caches. CoreRecipeInfo as well as other +# Extra RecipeInfo needs to inherit this class +class RecipeInfoCommon(object): @classmethod def listvar(cls, var, metadata): @@ -117,69 +82,162 @@ class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)): def getvar(cls, var, metadata): return metadata.getVar(var, True) or '' - @classmethod - def make_optional(cls, default=None, **kwargs): - """Construct the namedtuple from the specified keyword arguments, - with every value considered optional, using the default value if - it was not specified.""" - for field in cls._fields: - kwargs[field] = kwargs.get(field, default) - return cls(**kwargs) + +class CoreRecipeInfo(RecipeInfoCommon): + __slots__ = () + + def __init__(self, filename, metadata): + self.name = "base" + # please override this member with the correct data cache file + # such as (bb_cache.dat, bb_extracache_hob.dat) + self.cachefile = "bb_cache.dat" + + self.file_depends = metadata.getVar('__depends', False) + self.timestamp = bb.parse.cached_mtime(filename) + self.variants = self.listvar('__VARIANTS', metadata) + [''] + self.nocache = self.getvar('__BB_DONT_CACHE', metadata) + + if self.getvar('__SKIPPED', metadata): + self.skipped = True + return + + self.tasks = metadata.getVar('__BBTASKS', False) + + self.pn = self.getvar('PN', metadata) + self.packages = self.listvar('PACKAGES', metadata) + if not self.pn in self.packages: + self.packages.append(self.pn) + + self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata) + self.hashfilename = self.getvar('BB_HASHFILENAME', metadata) + + self.file_depends = metadata.getVar('__depends', False) + self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}} + + self.skipped = False + self.pe = self.getvar('PE', metadata) + self.pv = self.getvar('PV', metadata) + self.pr = self.getvar('PR', metadata) + self.defaultpref = self.intvar('DEFAULT_PREFERENCE', metadata) + self.broken = self.getvar('BROKEN', metadata) + self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata) + self.stamp = self.getvar('STAMP', metadata) + self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata) + self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata) + self.depends = self.depvar('DEPENDS', metadata) + self.provides = self.depvar('PROVIDES', metadata) + self.rdepends = self.depvar('RDEPENDS', metadata) + self.rprovides = self.depvar('RPROVIDES', metadata) + self.rrecommends = self.depvar('RRECOMMENDS', metadata) + self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata) + self.rdepends_pkg = self.pkgvar('RDEPENDS', self.packages, metadata) + self.rrecommends_pkg = self.pkgvar('RRECOMMENDS', self.packages, metadata) + self.inherits = self.getvar('__inherit_cache', metadata) + self.summary = self.getvar('SUMMARY', metadata) + self.license = self.getvar('LICENSE', metadata) + self.section = self.getvar('SECTION', metadata) + self.fakerootenv = self.getvar('FAKEROOTENV', metadata) + self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata) @classmethod - def from_metadata(cls, filename, metadata): - if cls.getvar('__SKIPPED', metadata): - return cls.make_optional(skipped=True, - file_depends=metadata.getVar('__depends', False), - timestamp=bb.parse.cached_mtime(filename), - variants=cls.listvar('__VARIANTS', metadata) + ['']) + def init_cacheData(cls, cachedata): + # CacheData in Core RecipeInfo Class + cachedata.task_deps = {} + cachedata.pkg_fn = {} + cachedata.pkg_pn = defaultdict(list) + cachedata.pkg_pepvpr = {} + cachedata.pkg_dp = {} - tasks = metadata.getVar('__BBTASKS', False) + cachedata.stamp = {} + cachedata.stamp_extrainfo = {} + cachedata.fn_provides = {} + cachedata.pn_provides = defaultdict(list) + cachedata.all_depends = [] - pn = cls.getvar('PN', metadata) - packages = cls.listvar('PACKAGES', metadata) - if not pn in packages: - packages.append(pn) + cachedata.deps = defaultdict(list) + cachedata.packages = defaultdict(list) + cachedata.providers = defaultdict(list) + cachedata.rproviders = defaultdict(list) + cachedata.packages_dynamic = defaultdict(list) - return RecipeInfo( - tasks = tasks, - basetaskhashes = cls.taskvar('BB_BASEHASH', tasks, metadata), - hashfilename = cls.getvar('BB_HASHFILENAME', metadata), + cachedata.rundeps = defaultdict(lambda: defaultdict(list)) + cachedata.runrecs = defaultdict(lambda: defaultdict(list)) + cachedata.possible_world = [] + cachedata.universe_target = [] + cachedata.hashfn = {} - file_depends = metadata.getVar('__depends', False), - task_deps = metadata.getVar('_task_deps', False) or - {'tasks': [], 'parents': {}}, - variants = cls.listvar('__VARIANTS', metadata) + [''], + cachedata.basetaskhash = {} + cachedata.inherits = {} + cachedata.summary = {} + cachedata.license = {} + cachedata.section = {} + cachedata.fakerootenv = {} + cachedata.fakerootdirs = {} + + def add_cacheData(self, cachedata, fn): + cachedata.task_deps[fn] = self.task_deps + cachedata.pkg_fn[fn] = self.pn + cachedata.pkg_pn[self].append(fn) + cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr) + cachedata.pkg_dp[fn] = self.defaultpref + cachedata.stamp[fn] = self.stamp + cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo + + provides = [self.pn] + for provide in self.provides: + if provide not in provides: + provides.append(provide) + cachedata.fn_provides[fn] = provides + + for provide in provides: + cachedata.providers[provide].append(fn) + if provide not in cachedata.pn_provides[self.pn]: + cachedata.pn_provides[self.pn].append(provide) + + for dep in self.depends: + if dep not in cachedata.deps[fn]: + cachedata.deps[fn].append(dep) + if dep not in cachedata.all_depends: + cachedata.all_depends.append(dep) + + rprovides = self.rprovides + for package in self.packages: + cachedata.packages[package].append(fn) + rprovides += self.rprovides_pkg[package] + + for rprovide in rprovides: + cachedata.rproviders[rprovide].append(fn) + + for package in self.packages_dynamic: + cachedata.packages_dynamic[package].append(fn) + + # Build hash of runtime depends and rececommends + for package in self.packages + [self.pn]: + cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package] + cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package] + + # Collect files we may need for possible world-dep + # calculations + if not self.broken and not self.not_world: + cachedata.possible_world.append(fn) + + # create a collection of all targets for sanity checking + # tasks, such as upstream versions, license, and tools for + # task and image creation. + cachedata.universe_target.append(self.pn) + + cachedata.hashfn[fn] = self.hashfilename + for task, taskhash in self.basetaskhashes.iteritems(): + identifier = '%s.%s' % (fn, task) + cachedata.basetaskhash[identifier] = taskhash + + cachedata.inherits[fn] = self.inherits + cachedata.summary[fn] = self.summary + cachedata.license[fn] = self.license + cachedata.section[fn] = self.section + cachedata.fakerootenv[fn] = self.fakerootenv + cachedata.fakerootdirs[fn] = self.fakerootdirs - skipped = False, - timestamp = bb.parse.cached_mtime(filename), - packages = cls.listvar('PACKAGES', metadata), - pn = pn, - pe = cls.getvar('PE', metadata), - pv = cls.getvar('PV', metadata), - pr = cls.getvar('PR', metadata), - nocache = cls.getvar('__BB_DONT_CACHE', metadata), - defaultpref = cls.intvar('DEFAULT_PREFERENCE', metadata), - broken = cls.getvar('BROKEN', metadata), - not_world = cls.getvar('EXCLUDE_FROM_WORLD', metadata), - stamp = cls.getvar('STAMP', metadata), - stamp_extrainfo = cls.flaglist('stamp-extra-info', tasks, metadata), - packages_dynamic = cls.listvar('PACKAGES_DYNAMIC', metadata), - depends = cls.depvar('DEPENDS', metadata), - provides = cls.depvar('PROVIDES', metadata), - rdepends = cls.depvar('RDEPENDS', metadata), - rprovides = cls.depvar('RPROVIDES', metadata), - rrecommends = cls.depvar('RRECOMMENDS', metadata), - rprovides_pkg = cls.pkgvar('RPROVIDES', packages, metadata), - rdepends_pkg = cls.pkgvar('RDEPENDS', packages, metadata), - rrecommends_pkg = cls.pkgvar('RRECOMMENDS', packages, metadata), - inherits = cls.getvar('__inherit_cache', metadata), - summary = cls.getvar('SUMMARY', metadata), - license = cls.getvar('LICENSE', metadata), - section = cls.getvar('SECTION', metadata), - fakerootenv = cls.getvar('FAKEROOTENV', metadata), - fakerootdirs = cls.getvar('FAKEROOTDIRS', metadata), - ) class Cache(object): @@ -314,7 +372,7 @@ class Cache(object): depends |= (data.getVar("__depends", False) or set()) if depends and not variant: data.setVar("__depends", depends) - info = RecipeInfo.from_metadata(filename, data) + info = CoreRecipeInfo(filename, data) infos.append((virtualfn, info)) return infos @@ -500,7 +558,7 @@ class Cache(object): """ realfn = self.virtualfn2realfn(file_name)[0] - info = RecipeInfo.from_metadata(realfn, data) + info = CoreRecipeInfo(realfn, data) self.add_info(file_name, info, cacheData, parsed) @staticmethod @@ -566,38 +624,11 @@ class CacheData(object): """ def __init__(self): + CoreRecipeInfo.init_cacheData(self) # Direct cache variables - self.providers = defaultdict(list) - self.rproviders = defaultdict(list) - self.packages = defaultdict(list) - self.packages_dynamic = defaultdict(list) - self.possible_world = [] - self.universe_target = [] - self.pkg_pn = defaultdict(list) - self.pkg_fn = {} - self.pkg_pepvpr = {} - self.pkg_dp = {} - self.pn_provides = defaultdict(list) - self.fn_provides = {} - self.all_depends = [] - self.deps = defaultdict(list) - self.rundeps = defaultdict(lambda: defaultdict(list)) - self.runrecs = defaultdict(lambda: defaultdict(list)) self.task_queues = {} - self.task_deps = {} - self.stamp = {} - self.stamp_extrainfo = {} self.preferred = {} self.tasks = {} - self.basetaskhash = {} - self.hashfn = {} - self.inherits = {} - self.summary = {} - self.license = {} - self.section = {} - self.fakerootenv = {} - self.fakerootdirs = {} - # Indirect Cache variables (set elsewhere) self.ignored_dependencies = [] self.world_target = set() @@ -605,65 +636,6 @@ class CacheData(object): self.bbfile_config_priorities = [] def add_from_recipeinfo(self, fn, info): - self.task_deps[fn] = info.task_deps - self.pkg_fn[fn] = info.pn - self.pkg_pn[info.pn].append(fn) - self.pkg_pepvpr[fn] = (info.pe, info.pv, info.pr) - self.pkg_dp[fn] = info.defaultpref - self.stamp[fn] = info.stamp - self.stamp_extrainfo[fn] = info.stamp_extrainfo + info.add_cacheData(self, fn) - provides = [info.pn] - for provide in info.provides: - if provide not in provides: - provides.append(provide) - self.fn_provides[fn] = provides - - for provide in provides: - self.providers[provide].append(fn) - if provide not in self.pn_provides[info.pn]: - self.pn_provides[info.pn].append(provide) - - for dep in info.depends: - if dep not in self.deps[fn]: - self.deps[fn].append(dep) - if dep not in self.all_depends: - self.all_depends.append(dep) - - rprovides = info.rprovides - for package in info.packages: - self.packages[package].append(fn) - rprovides += info.rprovides_pkg[package] - - for rprovide in rprovides: - self.rproviders[rprovide].append(fn) - - for package in info.packages_dynamic: - self.packages_dynamic[package].append(fn) - - # Build hash of runtime depends and rececommends - for package in info.packages + [info.pn]: - self.rundeps[fn][package] = list(info.rdepends) + info.rdepends_pkg[package] - self.runrecs[fn][package] = list(info.rrecommends) + info.rrecommends_pkg[package] - - # Collect files we may need for possible world-dep - # calculations - if not info.broken and not info.not_world: - self.possible_world.append(fn) - - # create a collection of all targets for sanity checking - # tasks, such as upstream versions, license, and tools for - # task and image creation. - self.universe_target.append(info.pn) - - self.hashfn[fn] = info.hashfilename - for task, taskhash in info.basetaskhashes.iteritems(): - identifier = '%s.%s' % (fn, task) - self.basetaskhash[identifier] = taskhash - - self.inherits[fn] = info.inherits - self.summary[fn] = info.summary - self.license[fn] = info.license - self.section[fn] = info.section - self.fakerootenv[fn] = info.fakerootenv - self.fakerootdirs[fn] = info.fakerootdirs +