mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-03-17 04:39:40 +01:00
Code Issues Packages Projects Releases Wiki Activity

bitbake: bitbake: cache: Use multiconfig aware caches

Browse Source 
Splits the parsing cache to maintain one cache per multiconfig instead
of one global cache. This is necessary now that the files and appends
can vary for each multiconfig. A bb.cache.MulticonfigCache
dictionary-like proxy object is created instead of a single
bb.cache.Cache object. This object will create and properly initialize
bb.cache.Cache object for each multiconfig, and each of these caches has
a dedicated cache file with a name based on the multiconfig.

(Bitbake rev: 5272f2489586479880ae8d046dfcdbe0963ee5bb)

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Joshua Watt
2020-06-05 22:15:30 -05:00
committed by Richard Purdie
parent b9fdb6a426
commit f8163c22f4
2 changed files with 133 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
bitbake/lib/bb/cache.py
View File

@@ -19,7 +19,7 @@
import os
import logging
import pickle
from collections import defaultdict
from collections import defaultdict, Mapping
import bb.utils
import re
@@ -27,8 +27,11 @@ logger = logging.getLogger("BitBake.Cache")
__cache_version__ = "152"
def getCacheFile(path, filename, data_hash):
return os.path.join(path, filename + "." + data_hash)
def getCacheFile(path, filename, mc, data_hash):
mcspec = ''
if mc:
mcspec = ".%s" % mc
return os.path.join(path, filename + mcspec + "." + data_hash)
# RecipeInfoCommon defines common data retrieving methods
# from meta data for caches. CoreRecipeInfo as well as other
@@ -354,14 +357,14 @@ class Cache(NoCache):
"""
BitBake Cache implementation
"""
def __init__(self, databuilder, data_hash, caches_array):
def __init__(self, databuilder, mc, data_hash, caches_array):
super().__init__(databuilder)
data = databuilder.data
# Pass caches_array information into Cache Constructor
# It will be used later for deciding whether we
# need extra cache file dump/load support
self.mc = mc
self.caches_array = caches_array
self.cachedir = data.getVar("CACHE")
self.clean = set()
@@ -379,7 +382,17 @@ class Cache(NoCache):
return
self.has_cache = True
self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash)
def getCacheFile(self, cachefile):
return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash)
def prepare_cache(self, progress):
if not self.has_cache:
return 0
loaded = 0
self.cachefile = self.getCacheFile("bb_cache.dat")
logger.debug(1, "Cache dir: %s", self.cachedir)
bb.utils.mkdirhier(self.cachedir)
@@ -387,18 +400,22 @@ class Cache(NoCache):
cache_ok = True
if self.caches_array:
for cache_class in self.caches_array:
cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
cachefile = self.getCacheFile(cache_class.cachefile)
cache_ok = cache_ok and os.path.exists(cachefile)
cache_class.init_cacheData(self)
if cache_ok:
self.load_cachefile()
loaded = self.load_cachefile(progress)
elif os.path.isfile(self.cachefile):
logger.info("Out of date cache found, rebuilding...")
else:
logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
# We don't use the symlink, its just for debugging convinience
symlink = os.path.join(self.cachedir, "bb_cache.dat")
if self.mc:
symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc)
else:
symlink = os.path.join(self.cachedir, "bb_cache.dat")
if os.path.exists(symlink):
bb.utils.remove(symlink)
try:
@@ -406,21 +423,30 @@ class Cache(NoCache):
except OSError:
pass
def load_cachefile(self):
return loaded
def cachesize(self):
if not self.has_cache:
return 0
cachesize = 0
for cache_class in self.caches_array:
cachefile = self.getCacheFile(cache_class.cachefile)
try:
with open(cachefile, "rb") as cachefile:
cachesize += os.fstat(cachefile.fileno()).st_size
except FileNotFoundError:
pass
return cachesize
def load_cachefile(self, progress):
cachesize = self.cachesize()
previous_progress = 0
previous_percent = 0
# Calculate the correct cachesize of all those cache files
for cache_class in self.caches_array:
cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
with open(cachefile, "rb") as cachefile:
cachesize += os.fstat(cachefile.fileno()).st_size
bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
for cache_class in self.caches_array:
cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
cachefile = self.getCacheFile(cache_class.cachefile)
logger.debug(1, 'Loading cache file: %s' % cachefile)
with open(cachefile, "rb") as cachefile:
pickled = pickle.Unpickler(cachefile)
@@ -460,23 +486,11 @@ class Cache(NoCache):
self.depends_cache[key] = [value]
# only fire events on even percentage boundaries
current_progress = cachefile.tell() + previous_progress
if current_progress > cachesize:
# we might have calculated incorrect total size because a file
# might've been written out just after we checked its size
cachesize = current_progress
current_percent = 100 * current_progress / cachesize
if current_percent > previous_percent:
previous_percent = current_percent
bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
self.data)
progress(cachefile.tell() + previous_progress)
previous_progress += current_progress
# Note: depends cache number is corresponding to the parsing file numbers.
# The same file has several caches, still regarded as one item in the cache
bb.event.fire(bb.event.CacheLoadCompleted(cachesize,
len(self.depends_cache)),
self.data)
return len(self.depends_cache)
def parse(self, filename, appends):
"""Parse the specified filename, returning the recipe information"""
@@ -682,7 +696,7 @@ class Cache(NoCache):
for cache_class in self.caches_array:
cache_class_name = cache_class.__name__
cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
cachefile = self.getCacheFile(cache_class.cachefile)
with open(cachefile, "wb") as f:
p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL)
p.dump(__cache_version__)
@@ -701,7 +715,7 @@ class Cache(NoCache):
return bb.parse.cached_mtime_noerror(cachefile)
def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None):
if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
cacheData.add_from_recipeinfo(filename, info_array)
if watcher:
@@ -727,6 +741,65 @@ class Cache(NoCache):
info_array.append(cache_class(realfn, data))
self.add_info(file_name, info_array, cacheData, parsed)
class MulticonfigCache(Mapping):
def __init__(self, databuilder, data_hash, caches_array):
def progress(p):
nonlocal current_progress
nonlocal previous_progress
nonlocal previous_percent
nonlocal cachesize
current_progress = previous_progress + p
if current_progress > cachesize:
# we might have calculated incorrect total size because a file
# might've been written out just after we checked its size
cachesize = current_progress
current_percent = 100 * current_progress / cachesize
if current_percent > previous_percent:
previous_percent = current_percent
bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
databuilder.data)
cachesize = 0
current_progress = 0
previous_progress = 0
previous_percent = 0
self.__caches = {}
for mc, mcdata in databuilder.mcdata.items():
self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array)
cachesize += self.__caches[mc].cachesize()
bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data)
loaded = 0
for c in self.__caches.values():
loaded += c.prepare_cache(progress)
previous_progress = current_progress
# Note: depends cache number is corresponding to the parsing file numbers.
# The same file has several caches, still regarded as one item in the cache
bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data)
def __len__(self):
return len(self.__caches)
def __getitem__(self, key):
return self.__caches[key]
def __contains__(self, key):
return key in self.__caches
def __iter__(self):
for k in self.__caches:
yield k
def keys(self):
return self.__caches[key]
def init(cooker):
"""

42
bitbake/lib/bb/cooker.py
View File

@@ -541,8 +541,8 @@ class BBCooker:
if fn:
try:
bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
envdata = bb_cache.loadDataFull(fn, self.collections[mc].get_file_appends(fn))
bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn))
except Exception as e:
parselog.exception("Unable to read %s", fn)
raise
@@ -1328,9 +1328,9 @@ class BBCooker:
self.buildSetVars()
self.reset_mtime_caches()
bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
infos = bb_cache.parse(fn, self.collections[mc].get_file_appends(fn))
infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn))
infos = dict(infos)
fn = bb.cache.realfn2virtual(fn, cls, mc)
@@ -1968,7 +1968,7 @@ class Parser(multiprocessing.Process):
except queue.Full:
pending.append(result)
def parse(self, filename, appends):
def parse(self, mc, cache, filename, appends):
try:
origfilter = bb.event.LogHandler.filter
# Record the filename we're parsing into any events generated
@@ -1982,7 +1982,7 @@ class Parser(multiprocessing.Process):
bb.event.set_class_handlers(self.handlers.copy())
bb.event.LogHandler.filter = parse_filter
return True, self.bb_cache.parse(filename, appends)
return True, mc, cache.parse(filename, appends)
except Exception as exc:
tb = sys.exc_info()[2]
exc.recipe = filename
@@ -2016,16 +2016,16 @@ class CookerParser(object):
self.current = 0
self.process_names = []
self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array)
self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array)
self.fromcache = set()
self.willparse = set()
for mc in self.cooker.multiconfigs:
for filename in self.mcfilelist[mc]:
appends = self.cooker.collections[mc].get_file_appends(filename)
if not self.bb_cache.cacheValid(filename, appends):
self.willparse.add((filename, appends))
if not self.bb_caches[mc].cacheValid(filename, appends):
self.willparse.add((mc, self.bb_caches[mc], filename, appends))
else:
self.fromcache.add((filename, appends))
self.fromcache.add((mc, self.bb_caches[mc], filename, appends))
self.total = len(self.fromcache) + len(self.willparse)
self.toparse = len(self.willparse)
@@ -2043,7 +2043,6 @@ class CookerParser(object):
if self.toparse:
bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
def init():
Parser.bb_cache = self.bb_cache
bb.utils.set_process_name(multiprocessing.current_process().name)
multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
@@ -2099,7 +2098,11 @@ class CookerParser(object):
else:
process.join()
sync = threading.Thread(target=self.bb_cache.sync)
def sync_caches():
for c in self.bb_caches.values():
c.sync()
sync = threading.Thread(target=sync_caches)
sync.start()
multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
bb.codeparser.parser_cache_savemerge()
@@ -2116,8 +2119,8 @@ class CookerParser(object):
print("Processed parsing statistics saved to %s" % (pout))
def load_cached(self):
for mc, filename, appends in self.fromcache:
cached, infos = self.bb_cache.load(mc, filename, appends)
for mc, cache, filename, appends in self.fromcache:
cached, infos = cache.load(filename, appends)
yield not cached, mc, infos
def parse_generator(self):
@@ -2196,8 +2199,13 @@ class CookerParser(object):
if info_array[0].skipped:
self.skipped += 1
self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
(fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
(fn, cls, fnmc) = bb.cache.virtualfn2realfn(virtualfn)
if fnmc == mc:
cache = self.cooker.recipecaches[mc]
else:
cache = None
self.bb_caches[mc].add_info(virtualfn, info_array, cache,
parsed=parsed, watcher = self.cooker.add_filewatch)
return True
@@ -2207,6 +2215,6 @@ class CookerParser(object):
to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
for mc, filename, appends in to_reparse:
infos = self.bb_cache.parse(filename, appends)
infos = self.bb_caches[mc].parse(filename, appends)
for vfn, info_array in infos:
self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)