bitbake: data_smart/cookerdata: Add variable remapping support

This change adds support for improving the user experience when variables
are renamed. This isn't as simple as it might first appear since some
bitbake variables are used through the environment before the datastore
exists, some are bitbake variables which we know about straight away
and some are metadata defined which we don't know about until later.

This patch adds support for handling these different cases, allowing a list
of bitbake renamed variables to be defined in bitbake itself and allows this
to be extended through the metadata using BB_RENAMED_VARIABLES.

In order to give the best feedback to the user, we default to turning on
variable history tracking in the base data store from knotty, which allows
filename and line number information to be shown.

(Bitbake rev: bd50a5d5e4b4fa90844464396887ebdff0d4e5f7)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/cookerdata.py

@@ -247,6 +247,9 @@ class CookerDataBuilder(object):
         self.savedenv = bb.data.init()
         for k in cookercfg.env:
             self.savedenv.setVar(k, cookercfg.env[k])
+            if k in bb.data_smart.bitbake_renamed_vars:
+                bb.error('Variable %s from the shell environment has been renamed to %s' % (k, bb.data_smart.bitbake_renamed_vars[k]))
+                bb.fatal("Exiting to allow enviroment variables to be corrected")
 
         filtered_keys = bb.utils.approved_variables()
         bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys)
@@ -307,6 +310,24 @@ class CookerDataBuilder(object):
             logger.exception("Error parsing configuration files")
             raise bb.BBHandledException()
 
+
+        # Handle obsolete variable names
+        d = self.data
+        renamedvars = d.getVarFlags('BB_RENAMED_VARIABLES') or {}
+        renamedvars.update(bb.data_smart.bitbake_renamed_vars)
+        issues = False
+        for v in renamedvars:
+            if d.getVar(v) != None or d.hasOverrides(v):
+                issues = True
+                history = d.varhistory.get_variable_refs(v)
+                for h in history:
+                    for line in history[h]:
+                        bb.erroronce('Variable %s has been renamed to %s (file: %s line: %s)' % (v, renamedvars[v], h, line))
+                if not history:
+                    bb.erroronce('Variable %s has been renamed to %s' % (v, renamedvars[v]))
+        if issues:
+            raise bb.BBHandledException()
+
         # Create a copy so we can reset at a later date when UIs disconnect
         self.origdata = self.data
         self.data = bb.data.createCopy(self.origdata)

bitbake/lib/bb/data_smart.py

@@ -33,6 +33,9 @@ __expand_python_regexp__ = re.compile(r"\${@.+?}")
 __whitespace_split__ = re.compile(r'(\s)')
 __override_regexp__ = re.compile(r'[a-z0-9]+')
 
+bitbake_renamed_vars = {
+}
+
 def infer_caller_details(loginfo, parent = False, varval = True):
     """Save the caller the trouble of specifying everything."""
     # Save effort.
@@ -336,6 +339,16 @@ class VariableHistory(object):
                 lines.append(line)
         return lines
 
+    def get_variable_refs(self, var):
+        """Return a dict of file/line references"""
+        var_history = self.variable(var)
+        refs = {}
+        for event in var_history:
+            if event['file'] not in refs:
+                refs[event['file']] = []
+            refs[event['file']].append(event['line'])
+        return refs
+
     def get_variable_items_files(self, var):
         """
         Use variable history to map items added to a list variable and
@@ -377,6 +390,8 @@ class DataSmart(MutableMapping):
         self.inchistory = IncludeHistory()
         self.varhistory = VariableHistory(self)
         self._tracking = False
+        self._var_renames = {}
+        self._var_renames.update(bitbake_renamed_vars)
 
         self.expand_cache = {}
 
@@ -491,6 +506,16 @@ class DataSmart(MutableMapping):
     def hasOverrides(self, var):
         return var in self.overridedata
 
+    def _print_rename_error(self, var, loginfo):
+        info = ""
+        if "file" in loginfo:
+            info = " file: %s" % loginfo["file"]
+        if "line" in loginfo:
+            info += " line: %s" % loginfo["line"]
+        if info:
+            info = " (%s)" % info.strip()
+        bb.erroronce('Variable %s has been renamed to %s%s' % (var, self._var_renames[var], info))
+
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
 
@@ -502,6 +527,10 @@ class DataSmart(MutableMapping):
                 info += " line: %s" % loginfo["line"]
             bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info)
 
+        shortvar = var.split(":", 1)[0]
+        if shortvar in self._var_renames:
+            self._print_rename_error(shortvar, loginfo)
+
         self.expand_cache = {}
         parsing=False
         if 'parsing' in loginfo:
@@ -687,6 +716,12 @@ class DataSmart(MutableMapping):
     def setVarFlag(self, var, flag, value, **loginfo):
         self.expand_cache = {}
 
+        if var == "BB_RENAMED_VARIABLES":
+            self._var_renames[flag] = value
+
+        if var in self._var_renames:
+            self._print_rename_error(var, loginfo)
+
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         loginfo['flag'] = flag
@@ -926,6 +961,7 @@ class DataSmart(MutableMapping):
         data.inchistory = self.inchistory.copy()
 
         data._tracking = self._tracking
+        data._var_renames = self._var_renames
 
         data.overrides = None
         data.overridevars = copy.copy(self.overridevars)

bitbake/lib/bb/ui/knotty.py

@@ -25,7 +25,7 @@ from itertools import groupby
 
 from bb.ui import uihelper
 
-featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS]
+featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
 
 logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()