bitbake: cookerdata: Separate out data_hash and hook to tinfoil

Within tinfoil, the user can write to the configuration data but it won't
cause the data_hash checksum to be re-written, meaning cached parsing
data can be reused when it would now be incorrect.

Abstract out the data_hash code and add it to the invalidateCaches
command, called by tinfoil.modified_files() meaning that tinfoil can
instruct bitbake to update the caches and re-parse if necessary.

Also move the data_hash entirely into databuilder and drop the copy
in cooker as obsolete and not needed.

(Bitbake rev: d9ee77829f693ce75348fa64f406fcecfe4989aa)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/cooker.py

@@ -281,7 +281,6 @@ class BBCooker:
         self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False)
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
-        self.data_hash = self.databuilder.data_hash
         self.extraconfigdata = {}
 
         eventlog = self.data.getVar("BB_DEFAULT_EVENTLOG")
@@ -370,6 +369,11 @@ class BBCooker:
         if not clean:
             bb.parse.BBHandler.cached_statements = {}
 
+        # If writes were made to any of the data stores, we need to recalculate the data
+        # store cache
+        if hasattr(self, "databuilder"):
+            self.databuilder.calc_datastore_hashes()
+
     def parseConfiguration(self):
         self.updateCacheSync()
 
@@ -1338,7 +1342,7 @@ class BBCooker:
         self.buildSetVars()
         self.reset_mtime_caches()
 
-        bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
+        bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.databuilder.data_hash, self.caches_array)
 
         layername = self.collections[mc].calc_bbfile_priority(fn)[2]
         infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn), layername)
@@ -2112,7 +2116,7 @@ class CookerParser(object):
         self.mcfilelist = mcfilelist
         self.cooker = cooker
         self.cfgdata = cooker.data
-        self.cfghash = cooker.data_hash
+        self.cfghash = cooker.databuilder.data_hash
         self.cfgbuilder = cooker.databuilder
 
         # Accounting statistics

bitbake/lib/bb/cookerdata.py

@@ -254,9 +254,16 @@ class CookerDataBuilder(object):
         self.data = self.basedata
         self.mcdata = {}
 
+    def calc_datastore_hashes(self):
+        data_hash = hashlib.sha256()
+        data_hash.update(self.data.get_hash().encode('utf-8'))
+        multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
+        for config in multiconfig:
+            data_hash.update(self.mcdata[config].get_hash().encode('utf-8'))
+        self.data_hash = data_hash.hexdigest()
+
     def parseBaseConfiguration(self, worker=False):
         mcdata = {}
-        data_hash = hashlib.sha256()
         try:
             self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
 
@@ -279,7 +286,6 @@ class CookerDataBuilder(object):
                 bb.event.fire(bb.event.ConfigParsed(), self.data)
 
             bb.parse.init_parser(self.data)
-            data_hash.update(self.data.get_hash().encode('utf-8'))
             mcdata[''] = self.data
 
             multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
@@ -289,11 +295,9 @@ class CookerDataBuilder(object):
                 parsed_mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
                 bb.event.fire(bb.event.ConfigParsed(), parsed_mcdata)
                 mcdata[config] = parsed_mcdata
-                data_hash.update(parsed_mcdata.get_hash().encode('utf-8'))
             if multiconfig:
                 bb.event.fire(bb.event.MultiConfigParsed(mcdata), self.data)
 
-            self.data_hash = data_hash.hexdigest()
         except bb.data_smart.ExpansionError as e:
             logger.error(str(e))
             raise bb.BBHandledException()
@@ -328,6 +332,7 @@ class CookerDataBuilder(object):
         for mc in mcdata:
             self.mcdata[mc] = bb.data.createCopy(mcdata[mc])
         self.data = self.mcdata['']
+        self.calc_datastore_hashes()
 
     def reset(self):
         # We may not have run parseBaseConfiguration() yet