bitbake: cooker, toaster: variable definition tracking

In order to track the file where a configuration
variable was defined, this patch bring these changes:

* a new feature is defined in CookerFeatures, named
BASEDATASTORE_TRACKING. When a UI requests BASEDATASTORE_TRACKING,
the base variable definition are tracked when configuration
is parsed.

* getAllKeysWithFlags now includes variable history in the
data dump

* toaster_ui.py will record the operation, file path
and line number where the variable was changes

* toaster Simple UI will display the file path
and line number for Configuration page

There is a change in the models to accomodate the recording
of variable change history.

[YOCTO #5227]

(Bitbake rev: 78e58fed82f2a71f052485de0052d7b9cca53ffd)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/cooker.py

@@ -81,7 +81,7 @@ class SkippedPackage:
 
 
 class CookerFeatures(object):
-    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE] = range(2)
+    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE, BASEDATASTORE_TRACKING] = range(3)
 
     def __init__(self):
         self._features=set()
@@ -177,6 +177,9 @@ class BBCooker:
         self.data.disableTracking()
 
     def loadConfigurationData(self):
+        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
+            self.enableDataTracking()
+
         self.initConfigurationData()
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
@@ -189,6 +192,10 @@ class BBCooker:
         bb.data.update_data(self.event_data)
         bb.parse.init_parser(self.event_data)
 
+        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
+            self.disableDataTracking()
+
+
     def modifyConfigurationVar(self, var, val, default_file, op):
         if op == "append":
             self.appendConfigurationVar(var, val, default_file)
@@ -320,7 +327,6 @@ class BBCooker:
         open(confpath, 'w').close()
 
     def parseConfiguration(self):
-
         # Set log file verbosity
         verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", "0"))
         if verboselogs:
@@ -1175,7 +1181,10 @@ class BBCooker:
             try:
                 v = self.data.getVar(k, True)
                 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
-                    dump[k] = { 'v' : v }
+                    dump[k] = {
+    'v' : v ,
+    'history' : self.data.varhistory.variable(k),
+                    }
                     for d in flaglist:
                         dump[k][d] = self.data.getVarFlag(k, d)
             except Exception as e:

bitbake/lib/bb/ui/buildinfohelper.py

@@ -27,8 +27,10 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings")
 
 import toaster.toastermain.settings as toaster_django_settings
 from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage
-from toaster.orm.models import Target_Package, Build_Package, Variable, Build_File
-from toaster.orm.models import Task_Dependency, Build_Package_Dependency, Target_Package_Dependency, Recipe_Dependency
+from toaster.orm.models import Variable, VariableHistory
+from toaster.orm.models import Target_Package, Build_Package, Build_File
+from toaster.orm.models import Task_Dependency, Build_Package_Dependency
+from toaster.orm.models import Target_Package_Dependency, Recipe_Dependency
 from bb.msg import BBLogFormatter as format
 
 class ORMWrapper(object):
@@ -238,11 +240,15 @@ class ORMWrapper(object):
                         desc = vardump[root_var]['doc']
                 if desc is None:
                     desc = ''
-                Variable.objects.create( build = build_obj,
+                variable_obj = Variable.objects.create( build = build_obj,
                     variable_name = k,
                     variable_value = value,
                     description = desc)
-
+                for vh in vardump[k]['history']:
+                    VariableHistory.objects.create( variable = variable_obj,
+                            file_name = vh['file'],
+                            line_number = vh['line'],
+                            operation = vh['op'])
 
 class BuildInfoHelper(object):
     """ This class gathers the build information from the server and sends it

bitbake/lib/bb/ui/toasterui.py

@@ -41,7 +41,7 @@ import sys
 import time
 import xmlrpclib
 
-featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE]
+featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
 
 logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()

bitbake/lib/toaster/bldviewer/templates/configuration.html

@@ -5,16 +5,18 @@
 
             <tr>
             <th>Name</th>
-            <th>Value</th>
             <th>Description</th>
+            <th>Definition history</th>
+            <th>Value</th>
             </tr>
 
             {% for variable in configuration %}
 
             <tr class="data">
                 <td>{{variable.variable_name}}</td>
-                <td>{{variable.variable_value}}</td>
                 <td>{% if variable.description %}{{variable.description}}{% endif %}</td>
+                <td>{% for vh in variable.variablehistory_set.all %}{{vh.operation}} in {{vh.file_name}}:{{vh.line_number}}<br/>{%endfor%}</td>
+                <td>{{variable.variable_value}}</td>
             {% endfor %}
 
 {% endblock %}

bitbake/lib/toaster/orm/models.py

@@ -239,11 +239,15 @@ class Variable(models.Model):
     build = models.ForeignKey(Build, related_name='variable_build')
     variable_name = models.CharField(max_length=100)
     variable_value = models.TextField(blank=True)
-    file = models.FilePathField(max_length=255)
     changed = models.BooleanField(default=False)
     human_readable_name = models.CharField(max_length=200)
     description = models.TextField(blank=True)
 
+class VariableHistory(models.Model):
+    variable = models.ForeignKey(Variable)
+    file_name = models.FilePathField(max_length=255)
+    line_number = models.IntegerField(null=True)
+    operation = models.CharField(max_length=16)
 
 class LogMessage(models.Model):
     INFO = 0