From 54d0e30433c249604611367cf387bc20721c4523 Mon Sep 17 00:00:00 2001 From: Alexandru DAMIAN Date: Tue, 26 Nov 2013 18:12:43 +0000 Subject: [PATCH] bitbake: toaster: change package storage model Up until this patch, package information lived in two places - one table for build packages and one table for target installed packaged. This situation leads to two problems: there is no direct link between a build package and a installed package, and a lot of data is duplicated. This change unifies all package types in a single table. The SimpleUI remains the same for continuity sake, but the REST API will be changed in a future patch. The package dependencies and package files are now kept in a single table. Since we collect target installed package information at all times, we need to expand it to supplement missing information if a package is not actually built in the current build. Small changes to the Simple UI reflect the updated database schema. [YOCTO #5565] [YOCTO #5269] (Bitbake rev: f5d655bfaeb349c8680d74530617e34aa389d1f0) Signed-off-by: Alexandru DAMIAN Signed-off-by: Richard Purdie --- bitbake/lib/bb/ui/buildinfohelper.py | 165 ++++++++---------- bitbake/lib/bb/ui/toasterui.py | 3 +- .../toaster/bldviewer/templates/bpackage.html | 6 +- .../toaster/bldviewer/templates/package.html | 2 +- bitbake/lib/toaster/bldviewer/views.py | 25 ++- bitbake/lib/toaster/orm/models.py | 50 ++---- 6 files changed, 108 insertions(+), 143 deletions(-) diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index fef849d9a0..0252efdfef 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py @@ -20,15 +20,16 @@ import datetime import sys import bb import re +import ast os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings") import toaster.toastermain.settings as toaster_django_settings from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage from toaster.orm.models import Variable, VariableHistory -from toaster.orm.models import Target_Package, Build_Package, Build_File -from toaster.orm.models import Task_Dependency, Build_Package_Dependency -from toaster.orm.models import Target_Package_Dependency, Recipe_Dependency +from toaster.orm.models import Package, Package_File, Target_Installed_Package +from toaster.orm.models import Task_Dependency, Package_Dependency +from toaster.orm.models import Recipe_Dependency from bb.msg import BBLogFormatter as format class ORMWrapper(object): @@ -148,21 +149,48 @@ class ORMWrapper(object): return layer_object[0] - def save_target_package_information(self, target_obj, packagedict, bldpkgs, recipes): + def save_target_package_information(self, build_obj, target_obj, packagedict, pkgpnmap, recipes): for p in packagedict: - packagedict[p]['object'] = Target_Package.objects.create( target = target_obj, - name = p, - size = packagedict[p]['size']) - if p in bldpkgs: - packagedict[p]['object'].version = bldpkgs[p]['version'] - packagedict[p]['object'].recipe = recipes[bldpkgs[p]['pn']] - packagedict[p]['object'].save() + packagedict[p]['object'], created = Package.objects.get_or_create( build = build_obj, name = p ) + if created: + # package was not build in the current build, but + # fill in everything we can from the runtime-reverse package data + try: + packagedict[p]['object'].recipe = recipes[pkgpnmap[p]['PN']] + packagedict[p]['object'].version = pkgpnmap[p]['PV'] + packagedict[p]['object'].revision = pkgpnmap[p]['PR'] + packagedict[p]['object'].license = pkgpnmap[p]['LICENSE'] + packagedict[p]['object'].section = pkgpnmap[p]['SECTION'] + packagedict[p]['object'].summary = pkgpnmap[p]['SUMMARY'] + packagedict[p]['object'].description = pkgpnmap[p]['DESCRIPTION'] + packagedict[p]['object'].size = int(pkgpnmap[p]['PKGSIZE']) + + # no files recorded for this package, so save files info + for targetpath in pkgpnmap[p]['FILES_INFO']: + targetfilesize = pkgpnmap[p]['FILES_INFO'][targetpath] + Package_File.objects.create( package = packagedict[p]['object'], + path = targetpath, + size = targetfilesize) + except KeyError as e: + print "Key error, package", p, "key", e + + # save disk installed size + packagedict[p]['object'].installed_size = packagedict[p]['size'] + packagedict[p]['object'].save() + + Target_Installed_Package.objects.create(target = target_obj, package = packagedict[p]['object']) for p in packagedict: for (px,deptype) in packagedict[p]['depends']: - Target_Package_Dependency.objects.create( package = packagedict[p]['object'], + if deptype == 'depends': + tdeptype = Package_Dependency.TYPE_TRDEPENDS + elif deptype == 'recommends': + tdeptype = Package_Dependency.TYPE_TRECOMMENDS + + Package_Dependency.objects.create( package = packagedict[p]['object'], depends_on = packagedict[px]['object'], - dep_type = deptype); + dep_type = tdeptype, + target = target_obj); def create_logmessage(self, log_information): @@ -180,48 +208,53 @@ class ORMWrapper(object): def save_build_package_information(self, build_obj, package_info, recipes): # create and save the object - bp_object = Build_Package.objects.create( build = build_obj, - recipe = recipes[package_info['PN']], - name = package_info['PKG'], - version = package_info['PKGV'], - revision = package_info['PKGR'], - summary = package_info['SUMMARY'], - description = package_info['DESCRIPTION'], - size = int(package_info['PKGSIZE']), - section = package_info['SECTION'], - license = package_info['LICENSE'], - ) + bp_object, created = Package.objects.get_or_create( build = build_obj, + name = package_info['PKG'] ) + + bp_object.recipe = recipes[package_info['PN']] + bp_object.version = package_info['PKGV'] + bp_object.revision = package_info['PKGR'] + bp_object.summary = package_info['SUMMARY'] + bp_object.description = package_info['DESCRIPTION'] + bp_object.size = int(package_info['PKGSIZE']) + bp_object.section = package_info['SECTION'] + bp_object.license = package_info['LICENSE'] + bp_object.save() + # save any attached file information for path in package_info['FILES_INFO']: - fo = Build_File.objects.create( bpackage = bp_object, + fo = Package_File.objects.create( package = bp_object, path = path, size = package_info['FILES_INFO'][path] ) + def _po_byname(p): + return Package.objects.get_or_create(build = build_obj, name = p)[0] + # save soft dependency information if 'RDEPENDS' in package_info and package_info['RDEPENDS']: for p in bb.utils.explode_deps(package_info['RDEPENDS']): - Build_Package_Dependency.objects.get_or_create( package = bp_object, - depends_on = p, dep_type = Build_Package_Dependency.TYPE_RDEPENDS) + Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RDEPENDS) if 'RPROVIDES' in package_info and package_info['RPROVIDES']: for p in bb.utils.explode_deps(package_info['RPROVIDES']): - Build_Package_Dependency.objects.get_or_create( package = bp_object, - depends_on = p, dep_type = Build_Package_Dependency.TYPE_RPROVIDES) + Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RPROVIDES) if 'RRECOMMENDS' in package_info and package_info['RRECOMMENDS']: for p in bb.utils.explode_deps(package_info['RRECOMMENDS']): - Build_Package_Dependency.objects.get_or_create( package = bp_object, - depends_on = p, dep_type = Build_Package_Dependency.TYPE_RRECOMMENDS) + Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RRECOMMENDS) if 'RSUGGESTS' in package_info and package_info['RSUGGESTS']: for p in bb.utils.explode_deps(package_info['RSUGGESTS']): - Build_Package_Dependency.objects.get_or_create( package = bp_object, - depends_on = p, dep_type = Build_Package_Dependency.TYPE_RSUGGESTS) + Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RSUGGESTS) if 'RREPLACES' in package_info and package_info['RREPLACES']: for p in bb.utils.explode_deps(package_info['RREPLACES']): - Build_Package_Dependency.objects.get_or_create( package = bp_object, - depends_on = p, dep_type = Build_Package_Dependency.TYPE_RREPLACES) + Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RREPLACES) if 'RCONFLICTS' in package_info and package_info['RCONFLICTS']: for p in bb.utils.explode_deps(package_info['RCONFLICTS']): - Build_Package_Dependency.objects.get_or_create( package = bp_object, - depends_on = p, dep_type = Build_Package_Dependency.TYPE_RCONFLICTS) + Package_Dependency.objects.get_or_create( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS) return bp_object @@ -469,54 +502,13 @@ class BuildInfoHelper(object): self.orm_wrapper.get_update_task_object(task_information) - def read_target_package_dep_data(self, event): - # for all targets + def store_target_package_data(self, event): + # for all image targets for target in self.internal_state['targets']: - # verify that we have something to read - if not target.is_image or not self.has_build_history: - print "not collecting package info ", target.is_image, self.has_build_history - break - - # TODO this is a temporary replication of the code in buildhistory.bbclass - # This MUST be changed to query the actual BUILD_DIR_IMAGE in the target context when - # the capability will be implemented in Bitbake - - MACHINE_ARCH, error = self.server.runCommand(['getVariable', 'MACHINE_ARCH']) - TCLIBC, error = self.server.runCommand(['getVariable', 'TCLIBC']) - BUILDHISTORY_DIR, error = self.server.runCommand(['getVariable', 'BUILDHISTORY_DIR']) - BUILDHISTORY_DIR_IMAGE = "%s/images/%s/%s/%s" % (BUILDHISTORY_DIR, MACHINE_ARCH, TCLIBC, target.target) - - self.internal_state['packages'] = {} - - with open("%s/installed-package-sizes.txt" % BUILDHISTORY_DIR_IMAGE, "r") as fin: - for line in fin: - line = line.rstrip(";") - psize, px = line.split("\t") - punit, pname = px.split(" ") - self.internal_state['packages'][pname.strip()] = {'size':int(psize)*1024, 'depends' : []} - - with open("%s/depends.dot" % BUILDHISTORY_DIR_IMAGE, "r") as fin: - p = re.compile(r' -> ') - dot = re.compile(r'.*style=dotted') - for line in fin: - line = line.rstrip(';') - linesplit = p.split(line) - if len(linesplit) == 2: - pname = linesplit[0].rstrip('"').strip('"') - dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"') - deptype = Target_Package_Dependency.TYPE_DEPENDS - if dot.match(line): - deptype = Target_Package_Dependency.TYPE_RECOMMENDS - if not pname in self.internal_state['packages']: - self.internal_state['packages'][pname] = {'size': 0, 'depends' : []} - if not dependsname in self.internal_state['packages']: - self.internal_state['packages'][dependsname] = {'size': 0, 'depends' : []} - self.internal_state['packages'][pname]['depends'].append((dependsname, deptype)) - - self.orm_wrapper.save_target_package_information(target, - self.internal_state['packages'], - self.internal_state['bldpkgs'], self.internal_state['recipes']) - + if target.is_image: + pkgdata = event.data['pkgdata'] + imgdata = event.data['imgdata'][target.target] + self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes']) def store_dependency_information(self, event): # save layer version priorities @@ -528,11 +520,6 @@ class BuildInfoHelper(object): layer_version_obj.priority = priority layer_version_obj.save() - # save build time package information - self.internal_state['bldpkgs'] = {} - for pkg in event._depgraph['packages']: - self.internal_state['bldpkgs'][pkg] = event._depgraph['packages'][pkg] - # save recipe information self.internal_state['recipes'] = {} for pn in event._depgraph['pn']: diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index 318fc28adb..e469d93e82 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py @@ -209,7 +209,6 @@ def main(server, eventHandler, params ): continue if isinstance(event, (bb.event.BuildCompleted)): - buildinfohelper.read_target_package_dep_data(event) buildinfohelper.update_build_information(event, errors, warnings, taskfailures) continue @@ -240,6 +239,8 @@ def main(server, eventHandler, params ): buildinfohelper.store_layer_info(event) if event.type == "BuildStatsList": buildinfohelper.store_tasks_stats(event) + if event.type == "ImagePkgList": + buildinfohelper.store_target_package_data(event) continue # ignore diff --git a/bitbake/lib/toaster/bldviewer/templates/bpackage.html b/bitbake/lib/toaster/bldviewer/templates/bpackage.html index 4e6d9c6778..ca092ca6a0 100644 --- a/bitbake/lib/toaster/bldviewer/templates/bpackage.html +++ b/bitbake/lib/toaster/bldviewer/templates/bpackage.html @@ -23,7 +23,7 @@ {{package.name}} ({{package.filelist_bpackage.count}} files) {{package.version}}-{{package.revision}} - {{package.recipe.name}}{{package.package_name}} + {%if package.recipe%}{{package.recipe.name}}{{package.package_name}}{%endif%} {{package.summary}} {{package.section}} @@ -32,8 +32,8 @@ {{package.license}}
- {% for bpd in package.bpackage_dependencies_package.all %} - {{bpd.dep_type}}: {{bpd.depends_on}}
+ {% for bpd in package.package_dependencies_source.all %} + {{bpd.dep_type}}: {{bpd.depends_on.name}}
{% endfor %}
diff --git a/bitbake/lib/toaster/bldviewer/templates/package.html b/bitbake/lib/toaster/bldviewer/templates/package.html index c22e988e95..b1246e788a 100644 --- a/bitbake/lib/toaster/bldviewer/templates/package.html +++ b/bitbake/lib/toaster/bldviewer/templates/package.html @@ -23,7 +23,7 @@ {{package.recipe.name}}{{package.package_name}}{%endif%}
- {% for d in package.tpackage_dependencies_package.all %} + {% for d in package.package_dependencies_source.all %} {{d.depends_on.name}}
{% endfor %}
diff --git a/bitbake/lib/toaster/bldviewer/views.py b/bitbake/lib/toaster/bldviewer/views.py index 7be4d4b899..3eb785b6c6 100644 --- a/bitbake/lib/toaster/bldviewer/views.py +++ b/bitbake/lib/toaster/bldviewer/views.py @@ -20,8 +20,9 @@ import operator from django.db.models import Q from django.shortcuts import render -from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, Target_Package, LogMessage, Variable -from orm.models import Task_Dependency, Recipe_Dependency, Build_Package, Build_File, Build_Package_Dependency +from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable +from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency +from orm.models import Target_Installed_Package from django.views.decorators.cache import cache_control @cache_control(no_store=True) @@ -78,23 +79,20 @@ def configuration(request, build_id): def bpackage(request, build_id): template = 'bpackage.html' - packages = Build_Package.objects.filter(build = build_id) + packages = Package.objects.filter(build = build_id) context = {'build': Build.objects.filter(pk=build_id)[0], 'packages' : packages} return render(request, template, context) def bfile(request, build_id, package_id): template = 'bfile.html' - files = Build_File.objects.filter(bpackage = package_id) + files = Package_File.objects.filter(package = package_id) context = {'build': Build.objects.filter(pk=build_id)[0], 'files' : files} return render(request, template, context) def tpackage(request, build_id, target_id): template = 'package.html' - - packages = Target_Package.objects.filter(target=target_id) - - context = {'build' : Build.objects.filter(pk=build_id)[0],'packages': packages} - + packages = map(lambda x: x.package, list(Target_Installed_Package.objects.filter(target=target_id))) + context = {'build': Build.objects.filter(pk=build_id)[0], 'packages' : packages} return render(request, template, context) def layer(request): @@ -135,17 +133,16 @@ def model_explorer(request, model_name): model_mapping = { 'build': Build, 'target': Target, - 'target_package': Target_Package, 'task': Task, 'task_dependency': Task_Dependency, - 'package': Build_Package, + 'package': Package, 'layer': Layer, 'layerversion': Layer_Version, 'recipe': Recipe, 'recipe_dependency': Recipe_Dependency, - 'build_package': Build_Package, - 'build_package_dependency': Build_Package_Dependency, - 'build_file': Build_File, + 'package': Package, + 'package_dependency': Package_Dependency, + 'build_file': Package_File, 'variable': Variable, 'logmessage': LogMessage, } diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py index 0bb048c756..b30e405c0e 100644 --- a/bitbake/lib/toaster/orm/models.py +++ b/bitbake/lib/toaster/orm/models.py @@ -130,7 +130,7 @@ class Task_Dependency(models.Model): depends_on = models.ForeignKey(Task, related_name='task_dependencies_depends') -class Build_Package(models.Model): +class Package(models.Model): build = models.ForeignKey('Build') recipe = models.ForeignKey('Recipe', null=True) name = models.CharField(max_length=100) @@ -139,16 +139,19 @@ class Build_Package(models.Model): summary = models.CharField(max_length=200, blank=True) description = models.CharField(max_length=200, blank=True) size = models.IntegerField(default=0) + installed_size = models.IntegerField(default=0) section = models.CharField(max_length=80, blank=True) license = models.CharField(max_length=80, blank=True) -class Build_Package_Dependency(models.Model): +class Package_Dependency(models.Model): TYPE_RDEPENDS = 0 TYPE_RPROVIDES = 1 TYPE_RRECOMMENDS = 2 TYPE_RSUGGESTS = 3 TYPE_RREPLACES = 4 TYPE_RCONFLICTS = 5 + TYPE_TRDEPENDS = 6 + TYPE_TRECOMMENDS = 7 DEPENDS_TYPE = ( (TYPE_RDEPENDS, "rdepends"), (TYPE_RPROVIDES, "rprovides"), @@ -156,46 +159,23 @@ class Build_Package_Dependency(models.Model): (TYPE_RSUGGESTS, "rsuggests"), (TYPE_RREPLACES, "rreplaces"), (TYPE_RCONFLICTS, "rconflicts"), + (TYPE_TRDEPENDS, "trdepends"), + (TYPE_TRECOMMENDS, "trecommends"), ) - package = models.ForeignKey(Build_Package, related_name='bpackage_dependencies_package') - depends_on = models.CharField(max_length=100) # soft dependency + package = models.ForeignKey(Package, related_name='package_dependencies_source') + depends_on = models.ForeignKey(Package, related_name='package_dependencies_target') # soft dependency dep_type = models.IntegerField(choices=DEPENDS_TYPE) + target = models.ForeignKey(Target, null=True) +class Target_Installed_Package(models.Model): + target = models.ForeignKey(Target) + package = models.ForeignKey(Package) -class Target_Package(models.Model): - target = models.ForeignKey('Target') - recipe = models.ForeignKey('Recipe', null=True) - name = models.CharField(max_length=100) - version = models.CharField(max_length=100, blank=True) - size = models.IntegerField() - - -class Target_Package_Dependency(models.Model): - TYPE_DEPENDS = 0 - TYPE_RDEPENDS = 1 - TYPE_RECOMMENDS = 2 - - DEPENDS_TYPE = ( - (TYPE_DEPENDS, "depends"), - (TYPE_RDEPENDS, "rdepends"), - (TYPE_RECOMMENDS, "recommends"), - ) - package = models.ForeignKey(Target_Package, related_name='tpackage_dependencies_package') - depends_on = models.ForeignKey(Target_Package, related_name='tpackage_dependencies_depends') - dep_type = models.IntegerField(choices=DEPENDS_TYPE) - - -class Build_File(models.Model): - bpackage = models.ForeignKey(Build_Package, related_name='filelist_bpackage') +class Package_File(models.Model): + package = models.ForeignKey(Package, related_name='buildfilelist_package') path = models.FilePathField(max_length=255, blank=True) size = models.IntegerField() -class Target_File(models.Model): - tpackage = models.ForeignKey(Target_Package, related_name='filelist_tpackage') - path = models.FilePathField(max_length=255, blank=True) - size = models.IntegerField() - - class Recipe(models.Model): name = models.CharField(max_length=100, blank=True) version = models.CharField(max_length=100, blank=True)