bitbake: cooker, toaster: variable definition tracking

In order to track the file where a configuration
variable was defined, this patch bring these changes:

* a new feature is defined in CookerFeatures, named
BASEDATASTORE_TRACKING. When a UI requests BASEDATASTORE_TRACKING,
the base variable definition are tracked when configuration
is parsed.

* getAllKeysWithFlags now includes variable history in the
data dump

* toaster_ui.py will record the operation, file path
and line number where the variable was changes

* toaster Simple UI will display the file path
and line number for Configuration page

There is a change in the models to accomodate the recording
of variable change history.

[YOCTO #5227]

(Bitbake rev: 78e58fed82f2a71f052485de0052d7b9cca53ffd)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/cooker.py

@@ -81,7 +81,7 @@ class SkippedPackage:
 
 
 class CookerFeatures(object):
-    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE] = range(2)
+    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE, BASEDATASTORE_TRACKING] = range(3)
 
     def __init__(self):
         self._features=set()
@@ -177,6 +177,9 @@ class BBCooker:
         self.data.disableTracking()
 
     def loadConfigurationData(self):
+        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
+            self.enableDataTracking()
+
         self.initConfigurationData()
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
@@ -189,6 +192,10 @@ class BBCooker:
         bb.data.update_data(self.event_data)
         bb.parse.init_parser(self.event_data)
 
+        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
+            self.disableDataTracking()
+
+
     def modifyConfigurationVar(self, var, val, default_file, op):
         if op == "append":
             self.appendConfigurationVar(var, val, default_file)
@@ -348,7 +355,6 @@ class BBCooker:
         open(confpath, 'w').close()
 
     def parseConfiguration(self):
-
         # Set log file verbosity
         verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", "0"))
         if verboselogs:
@@ -1203,7 +1209,10 @@ class BBCooker:
             try:
                 v = self.data.getVar(k, True)
                 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
-                    dump[k] = { 'v' : v }
+                    dump[k] = {
+    'v' : v ,
+    'history' : self.data.varhistory.variable(k),
+                    }
                     for d in flaglist:
                         dump[k][d] = self.data.getVarFlag(k, d)
             except Exception as e:

bitbake/lib/bb/ui/buildinfohelper.py

@@ -27,8 +27,10 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings")
 
 import toaster.toastermain.settings as toaster_django_settings
 from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage
-from toaster.orm.models import Target_Package, Build_Package, Variable, Build_File
+from toaster.orm.models import Variable, VariableHistory
-from toaster.orm.models import Task_Dependency, Build_Package_Dependency, Target_Package_Dependency, Recipe_Dependency
+from toaster.orm.models import Target_Package, Build_Package, Build_File
+from toaster.orm.models import Task_Dependency, Build_Package_Dependency
+from toaster.orm.models import Target_Package_Dependency, Recipe_Dependency
 from bb.msg import BBLogFormatter as format
 
 class ORMWrapper(object):
@@ -93,6 +95,12 @@ class ORMWrapper(object):
         # if we got covered by a setscene task, we're SSTATE
         if task_object.outcome == Task.OUTCOME_COVERED and 1 == Task.objects.filter(task_executed=True, build = task_object.build, recipe = task_object.recipe, task_name=task_object.task_name+"_setscene").count():
             task_object.outcome = Task.OUTCOME_SSTATE
+            outcome_task_setscene = Task.objects.get(task_executed=True, build = task_object.build,
+                                    recipe = task_object.recipe, task_name=task_object.task_name+"_setscene").outcome
+            if outcome_task_setscene == Task.OUTCOME_SUCCESS:
+                task_object.sstate_result = Task.SSTATE_RESTORED
+            elif outcome_task_setscene == Task.OUTCOME_FAILED:
+                task_object.sstate_result = Task.SSTATE_FAILED
 
         # mark down duration if we have a start time
         if 'start_time' in task_information.keys():
@@ -180,7 +188,7 @@ class ORMWrapper(object):
                                        revision = package_info['PKGR'],
                                        summary = package_info['SUMMARY'],
                                        description = package_info['DESCRIPTION'],
-                                       size = package_info['PKGSIZE'],
+                                       size = int(package_info['PKGSIZE']) * 1024,
                                        section = package_info['SECTION'],
                                        license = package_info['LICENSE'],
                                        )
@@ -232,11 +240,15 @@ class ORMWrapper(object):
                         desc = vardump[root_var]['doc']
                 if desc is None:
                     desc = ''
-                Variable.objects.create( build = build_obj,
+                variable_obj = Variable.objects.create( build = build_obj,
                     variable_name = k,
                     variable_value = value,
                     description = desc)
-
+                for vh in vardump[k]['history']:
+                    VariableHistory.objects.create( variable = variable_obj,
+                            file_name = vh['file'],
+                            line_number = vh['line'],
+                            operation = vh['op'])
 
 class BuildInfoHelper(object):
     """ This class gathers the build information from the server and sends it
@@ -386,7 +398,11 @@ class BuildInfoHelper(object):
             target = t.target
             machine = self.internal_state['build'].machine
             buildname = self.internal_state['build'].build_name
-            package = task_object.recipe.name + "-" + task_object.recipe.version.strip(":")
+            pe, pv = task_object.recipe.version.split(":",1)
+            if len(pe) > 0:
+                package = task_object.recipe.name + "-" + pe + "_" + pv
+            else:
+                package = task_object.recipe.name + "-" + pv
 
             build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, target=target,
                                                      machine=machine, buildname=buildname,
@@ -527,7 +543,7 @@ class BuildInfoHelper(object):
             task_information['disk_io'] = task_build_stats['disk_io']
             del self.internal_state[identifier]
 
-        if isinstance(event, bb.runqueue.runQueueTaskFailed):
+        if isinstance(event, (bb.runqueue.runQueueTaskFailed, bb.runCommand.sceneQueueTaskFailed)):
             task_information['outcome'] = Task.OUTCOME_FAILED
             del self.internal_state[identifier]
 
@@ -618,7 +634,6 @@ class BuildInfoHelper(object):
             recipe_info['licensing_info'] = 'Not Available'
             recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage']
             recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker']
-            recipe_info['author'] = 'Not Available'
             recipe_info['file_path'] = file_name
             recipe = self.orm_wrapper.get_update_recipe_object(recipe_info)
             if 'inherits' in event._depgraph['pn'][pn].keys():

bitbake/lib/bb/ui/toasterui.py

@@ -41,7 +41,7 @@ import sys
 import time
 import xmlrpclib
 
-featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE]
+featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
 
 logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()

bitbake/lib/toaster/bldviewer/templates/bpackage.html

@@ -13,7 +13,7 @@
             <th>Summary</th>
             <th>Section</th>
             <th>Description</th>
-            <th>Size on host disk (KBytes)</th>
+            <th>Size on host disk (Bytes)</th>
             <th>License</th>
             <th>Dependencies List (all)</th>
             </tr>

bitbake/lib/toaster/bldviewer/templates/configuration.html

@@ -5,16 +5,18 @@
 
             <tr>
             <th>Name</th>
-            <th>Value</th>
             <th>Description</th>
+            <th>Definition history</th>
+            <th>Value</th>
             </tr>
 
             {% for variable in configuration %}
 
             <tr class="data">
                 <td>{{variable.variable_name}}</td>
-                <td>{{variable.variable_value}}</td>
                 <td>{% if variable.description %}{{variable.description}}{% endif %}</td>
+                <td>{% for vh in variable.variablehistory_set.all %}{{vh.operation}} in {{vh.file_name}}:{{vh.line_number}}<br/>{%endfor%}</td>
+                <td>{{variable.variable_value}}</td>
             {% endfor %}
 
 {% endblock %}

bitbake/lib/toaster/bldviewer/templates/recipe.html

@@ -38,7 +38,6 @@
             <td>{{recipe.licensing_info}}</td>
             <td>{{recipe.homepage}}</td>
             <td>{{recipe.bugtracker}}</td>
-            <td>{{recipe.author}}</td>
             <td>{{recipe.file_path}}</td>
             <td>
         <div style="height: 5em; overflow:auto">

bitbake/lib/toaster/orm/models.py

@@ -74,11 +74,13 @@ class Task(models.Model):
         (SSTATE_RESTORED, 'Restored'), # succesfully restored
     )
 
-    CODING_NOEXEC = 0
+    CODING_NA = 0
-    CODING_PYTHON = 1
+    CODING_NOEXEC = 1
-    CODING_SHELL = 2
+    CODING_PYTHON = 2
+    CODING_SHELL = 3
 
     TASK_CODING = (
+        (CODING_NA, 'N/A'),
         (CODING_NOEXEC, 'NoExec'),
         (CODING_PYTHON, 'Python'),
         (CODING_SHELL, 'Shell'),
@@ -110,7 +112,7 @@ class Task(models.Model):
     task_name = models.CharField(max_length=100)
     source_url = models.FilePathField(max_length=255, blank=True)
     work_directory = models.FilePathField(max_length=255, blank=True)
-    script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NOEXEC)
+    script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA)
     line_number = models.IntegerField(default=0)
     disk_io = models.IntegerField(null=True)
     cpu_usage = models.DecimalField(max_digits=6, decimal_places=2, null=True)
@@ -205,7 +207,6 @@ class Recipe(models.Model):
     licensing_info = models.TextField(blank=True)
     homepage = models.URLField(blank=True)
     bugtracker = models.URLField(blank=True)
-    author = models.CharField(max_length=100, blank=True)
     file_path = models.FilePathField(max_length=255)
 
 
@@ -238,11 +239,15 @@ class Variable(models.Model):
     build = models.ForeignKey(Build, related_name='variable_build')
     variable_name = models.CharField(max_length=100)
     variable_value = models.TextField(blank=True)
-    file = models.FilePathField(max_length=255)
     changed = models.BooleanField(default=False)
     human_readable_name = models.CharField(max_length=200)
     description = models.TextField(blank=True)
 
+class VariableHistory(models.Model):
+    variable = models.ForeignKey(Variable)
+    file_name = models.FilePathField(max_length=255)
+    line_number = models.IntegerField(null=True)
+    operation = models.CharField(max_length=16)
 
 class LogMessage(models.Model):
     INFO = 0