diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index 8cd9371b06..b093b37a5d 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py @@ -150,55 +150,48 @@ class ORMWrapper(object): # pylint: disable=bad-continuation # we do not follow the python conventions for continuation indentation due to long lines here - def create_build_object(self, build_info, brbe, project_id): + def create_build_object(self, build_info, brbe): assert 'machine' in build_info assert 'distro' in build_info assert 'distro_version' in build_info assert 'started_on' in build_info assert 'cooker_log_path' in build_info - assert 'build_name' in build_info assert 'bitbake_version' in build_info prj = None buildrequest = None - if brbe is not None: # this build was triggered by a request from a user + if brbe is not None: + # Toaster-triggered build logger.debug(1, "buildinfohelper: brbe is %s" % brbe) br, _ = brbe.split(":") - buildrequest = BuildRequest.objects.get(pk = br) + buildrequest = BuildRequest.objects.get(pk=br) prj = buildrequest.project - - elif project_id is not None: # this build was triggered by an external system for a specific project - logger.debug(1, "buildinfohelper: project is %s" % prj) - prj = Project.objects.get(pk = project_id) - - else: # this build was triggered by a legacy system, or command line interactive mode + else: + # CLI build prj = Project.objects.get_or_create_default_project() logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) - if buildrequest is not None: build = buildrequest.build logger.info("Updating existing build, with %s", build_info) build.project = prj - build.machine=build_info['machine'] - build.distro=build_info['distro'] - build.distro_version=build_info['distro_version'] - build.cooker_log_path=build_info['cooker_log_path'] - build.build_name=build_info['build_name'] - build.bitbake_version=build_info['bitbake_version'] + build.machine = build_info['machine'] + build.distro = build_info['distro'] + build.distro_version = build_info['distro_version'] + build.cooker_log_path = build_info['cooker_log_path'] + build.bitbake_version = build_info['bitbake_version'] build.save() else: build = Build.objects.create( - project = prj, - machine=build_info['machine'], - distro=build_info['distro'], - distro_version=build_info['distro_version'], - started_on=build_info['started_on'], - completed_on=build_info['started_on'], - cooker_log_path=build_info['cooker_log_path'], - build_name=build_info['build_name'], - bitbake_version=build_info['bitbake_version']) + project=prj, + machine=build_info['machine'], + distro=build_info['distro'], + distro_version=build_info['distro_version'], + started_on=build_info['started_on'], + completed_on=build_info['started_on'], + cooker_log_path=build_info['cooker_log_path'], + bitbake_version=build_info['bitbake_version']) logger.debug(1, "buildinfohelper: build is created %s" % build) @@ -208,6 +201,10 @@ class ORMWrapper(object): return build + def update_build_name(self, build, build_name): + build.build_name = build_name + build.save() + @staticmethod def get_or_create_targets(target_info): """ @@ -924,9 +921,7 @@ class BuildInfoHelper(object): build_info['started_on'] = timezone.now() build_info['completed_on'] = timezone.now() build_info['cooker_log_path'] = build_log_path - build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] - build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0] return build_info def _get_task_information(self, event, recipe): @@ -1032,17 +1027,28 @@ class BuildInfoHelper(object): except NotExisting as nee: logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee) - - def store_started_build(self, event, build_log_path): - assert '_pkgs' in vars(event) + def store_started_build(self, build_log_path): build_information = self._get_build_information(build_log_path) + self.internal_state['build'] = \ + self.orm_wrapper.create_build_object(build_information, self.brbe) - # Update brbe and project as they can be changed for every build - self.project = build_information['project'] + def save_build_name_and_targets(self, event): + # NB the BUILDNAME variable isn't set until BuildInit (or + # BuildStarted for older bitbakes) + build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0] + self.orm_wrapper.update_build_name(self.internal_state['build'], + build_name) - build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) + # create target information + assert '_pkgs' in vars(event) + target_information = {} + target_information['targets'] = event._pkgs + target_information['build'] = self.internal_state['build'] - self.internal_state['build'] = build_obj + self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) + + def save_build_layers_and_variables(self): + build_obj = self.internal_state['build'] # save layer version information for this build if not 'lvs' in self.internal_state: @@ -1053,13 +1059,6 @@ class BuildInfoHelper(object): del self.internal_state['lvs'] - # create target information - target_information = {} - target_information['targets'] = event._pkgs - target_information['build'] = build_obj - - self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) - # Save build configuration data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0] @@ -1090,6 +1089,41 @@ class BuildInfoHelper(object): return self.brbe + def set_recipes_to_parse(self, num_recipes): + """ + Set the number of recipes which need to be parsed for this build. + This is set the first time ParseStarted is received by toasterui. + """ + if self.internal_state['build']: + self.internal_state['build'].recipes_to_parse = num_recipes + self.internal_state['build'].save() + + def set_recipes_parsed(self, num_recipes): + """ + Set the number of recipes parsed so far for this build; this is updated + each time a ParseProgress or ParseCompleted event is received by + toasterui. + """ + if self.internal_state['build']: + if num_recipes <= self.internal_state['build'].recipes_to_parse: + self.internal_state['build'].recipes_parsed = num_recipes + self.internal_state['build'].save() + + def update_target_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + + for t in self.internal_state['targets']: + if t.is_image == True: + output_files = list(evdata.keys()) + for output in output_files: + if t.target in output and 'rootfs' in output and not output.endswith(".manifest"): + self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) + + def update_artifact_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + for artifact_path in evdata.keys(): + self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) + def update_build_information(self, event, errors, warnings, taskfailures): if 'build' in self.internal_state: self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index f399a7d316..7d670bef6f 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py @@ -102,6 +102,7 @@ _evt_list = [ "bb.command.CommandExit", "bb.command.CommandFailed", "bb.cooker.CookerExit", + "bb.event.BuildInit", "bb.event.BuildCompleted", "bb.event.BuildStarted", "bb.event.CacheLoadCompleted", @@ -115,6 +116,7 @@ _evt_list = [ "bb.event.NoProvider", "bb.event.ParseCompleted", "bb.event.ParseProgress", + "bb.event.ParseStarted", "bb.event.RecipeParsed", "bb.event.SanityCheck", "bb.event.SanityCheckPassed", @@ -231,19 +233,30 @@ def main(server, eventHandler, params): # pylint: disable=protected-access # the code will look into the protected variables of the event; no easy way around this - # we treat ParseStarted as the first event of toaster-triggered - # builds; that way we get the Build Configuration included in the log - # and any errors that occur before BuildStarted is fired if isinstance(event, bb.event.ParseStarted): if not (build_log and build_log_file_path): build_log, build_log_file_path = _open_build_log(log_dir) + + buildinfohelper.store_started_build(build_log_file_path) + buildinfohelper.set_recipes_to_parse(event.total) continue - if isinstance(event, bb.event.BuildStarted): - if not (build_log and build_log_file_path): - build_log, build_log_file_path = _open_build_log(log_dir) + # create a build object in buildinfohelper from either BuildInit + # (if available) or BuildStarted (for jethro and previous versions) + if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)): + buildinfohelper.save_build_name_and_targets(event) - buildinfohelper.store_started_build(event, build_log_file_path) + # get additional data from BuildStarted + if isinstance(event, bb.event.BuildStarted): + buildinfohelper.save_build_layers_and_variables() + continue + + if isinstance(event, bb.event.ParseProgress): + buildinfohelper.set_recipes_parsed(event.current) + continue + + if isinstance(event, bb.event.ParseCompleted): + buildinfohelper.set_recipes_parsed(event.total) continue if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): @@ -289,10 +302,6 @@ def main(server, eventHandler, params): # timing and error informations from the parsing phase in Toaster if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)): continue - if isinstance(event, bb.event.ParseProgress): - continue - if isinstance(event, bb.event.ParseCompleted): - continue if isinstance(event, bb.event.CacheLoadStarted): continue if isinstance(event, bb.event.CacheLoadProgress): diff --git a/bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py b/bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py new file mode 100644 index 0000000000..cc5c96d2dd --- /dev/null +++ b/bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0012_use_release_instead_of_up_branch'), + ] + + operations = [ + migrations.AddField( + model_name='build', + name='recipes_parsed', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='build', + name='recipes_to_parse', + field=models.IntegerField(default=1), + ), + ] diff --git a/bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py b/bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py new file mode 100644 index 0000000000..4749a14b26 --- /dev/null +++ b/bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0013_recipe_parse_progress_fields'), + ] + + operations = [ + migrations.AlterField( + model_name='build', + name='build_name', + field=models.CharField(default='', max_length=100), + ), + ] diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py index 2df6d4910a..4641736add 100644 --- a/bitbake/lib/toaster/orm/models.py +++ b/bitbake/lib/toaster/orm/models.py @@ -397,9 +397,15 @@ class Build(models.Model): completed_on = models.DateTimeField() outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS) cooker_log_path = models.CharField(max_length=500) - build_name = models.CharField(max_length=100) + build_name = models.CharField(max_length=100, default='') bitbake_version = models.CharField(max_length=50) + # number of recipes to parse for this build + recipes_to_parse = models.IntegerField(default=1) + + # number of recipes parsed so far for this build + recipes_parsed = models.IntegerField(default=0) + @staticmethod def get_recent(project=None): """ @@ -615,6 +621,13 @@ class Build(models.Model): else: return False + def is_parsing(self): + """ + True if the build is still parsing recipes + """ + return self.outcome == Build.IN_PROGRESS and \ + self.recipes_parsed < self.recipes_to_parse + def get_state(self): """ Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress', @@ -628,6 +641,8 @@ class Build(models.Model): return 'Cancelling'; elif self.is_queued(): return 'Queued' + elif self.is_parsing(): + return 'Parsing' else: return self.get_outcome_text() diff --git a/bitbake/lib/toaster/toastergui/api.py b/bitbake/lib/toaster/toastergui/api.py index aa3cbd83b2..b57f670ba3 100644 --- a/bitbake/lib/toaster/toastergui/api.py +++ b/bitbake/lib/toaster/toastergui/api.py @@ -87,7 +87,7 @@ class XhrBuildRequest(View): br.save() except BuildRequest.DoesNotExist: - return error_response('No such build id %s' % i) + return error_response('No such build request id %s' % i) return error_response('ok') @@ -256,6 +256,14 @@ class MostRecentBuildsView(View): build['id'] = build_obj.pk build['dashboard_url'] = dashboard_url + buildrequest_id = None + if hasattr(build_obj, 'buildrequest'): + buildrequest_id = build_obj.buildrequest.pk + build['buildrequest_id'] = buildrequest_id + + build['recipes_parsed_percentage'] = \ + int((build_obj.recipes_parsed / build_obj.recipes_to_parse) * 100) + tasks_complete_percentage = 0 if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED): tasks_complete_percentage = 100 diff --git a/bitbake/lib/toaster/toastergui/static/js/mrbsection.js b/bitbake/lib/toaster/toastergui/static/js/mrbsection.js index d8c3bf7750..e7fbf01731 100644 --- a/bitbake/lib/toaster/toastergui/static/js/mrbsection.js +++ b/bitbake/lib/toaster/toastergui/static/js/mrbsection.js @@ -33,8 +33,8 @@ function mrbSectionInit(ctx){ return buildData[build.id] || {}; } - // returns true if a build's state changed to "Succeeded" or "Failed" - // from some other value + // returns true if a build's state changed to "Succeeded", "Failed" + // or "Cancelled" from some other value function buildFinished(build) { var cached = getCached(build); return cached.state && @@ -49,12 +49,18 @@ function mrbSectionInit(ctx){ return (cached.state !== build.state); } - // returns true if the complete_percentage changed - function progressChanged(build) { + // returns true if the tasks_complete_percentage changed + function tasksProgressChanged(build) { var cached = getCached(build); return (cached.tasks_complete_percentage !== build.tasks_complete_percentage); } + // returns true if the number of recipes parsed/to parse changed + function recipeProgressChanged(build) { + var cached = getCached(build); + return (cached.recipes_parsed_percentage !== build.recipes_parsed_percentage); + } + function refreshMostRecentBuilds(){ libtoaster.getMostRecentBuilds( libtoaster.ctx.mostRecentBuildsUrl, @@ -68,10 +74,6 @@ function mrbSectionInit(ctx){ var colourClass; var elements; - // classes on the parent which signify the build state and affect - // the colour of the container for the build - var buildStateClasses = 'alert-info alert-success alert-danger'; - for (var i = 0; i < data.length; i++) { build = data[i]; @@ -91,31 +93,25 @@ function mrbSectionInit(ctx){ container = $(selector); container.html(html); - - // style the outermost container for this build to reflect - // the new build state (red, green, blue); - // NB class set here should be in buildStateClasses - colourClass = 'alert-info'; - if (build.state == 'Succeeded') { - colourClass = 'alert-success'; - } - else if (build.state == 'Failed') { - colourClass = 'alert-danger'; - } - - elements = $('[data-latest-build-result="' + build.id + '"]'); - elements.removeClass(buildStateClasses); - elements.addClass(colourClass); } - else if (progressChanged(build)) { - // update the progress text + else if (tasksProgressChanged(build)) { + // update the task progress text selector = '#build-pc-done-' + build.id; $(selector).html(build.tasks_complete_percentage); - // update the progress bar + // update the task progress bar selector = '#build-pc-done-bar-' + build.id; $(selector).width(build.tasks_complete_percentage + '%'); } + else if (recipeProgressChanged(build)) { + // update the recipe progress text + selector = '#recipes-parsed-percentage-' + build.id; + $(selector).html(build.recipes_parsed_percentage); + + // update the recipe progress bar + selector = '#recipes-parsed-percentage-bar-' + build.id; + $(selector).width(build.recipes_parsed_percentage + '%'); + } buildData[build.id] = build; } @@ -128,6 +124,6 @@ function mrbSectionInit(ctx){ ); } - window.setInterval(refreshMostRecentBuilds, 1000); + window.setInterval(refreshMostRecentBuilds, 1500); refreshMostRecentBuilds(); } diff --git a/bitbake/lib/toaster/toastergui/templates/mrb_section.html b/bitbake/lib/toaster/toastergui/templates/mrb_section.html index 302b4b0da4..880485d45f 100644 --- a/bitbake/lib/toaster/toastergui/templates/mrb_section.html +++ b/bitbake/lib/toaster/toastergui/templates/mrb_section.html @@ -26,7 +26,9 @@
- {{build.project.name}} + + {{build.project.name}} +
@@ -52,14 +54,18 @@ <%:targets_abbreviated%> - <%else%> + <%else targets_abbreviated !== ''%> <%:targets_abbreviated%> + <%else%> + ...targets not yet available... <%/if%> - <%if state == 'Queued'%> + <%if state == 'Parsing'%> + <%include tmpl='#parsing-recipes-build-template'/%> + <%else state == 'Queued'%> <%include tmpl='#queued-build-template'/%> <%else state == 'Succeeded' || state == 'Failed'%> <%include tmpl='#succeeded-or-failed-build-template'/%> @@ -75,21 +81,38 @@ + + + @@ -110,17 +133,9 @@ <%:tasks_complete_percentage%>% of tasks complete - <%if is_default_project_build%> - - - <%else%> - - - - Cancel - - <%/if%> + + + <%include tmpl='#cancel-template'/%> @@ -162,19 +177,8 @@
Build time: <%:buildtime%> - <%if is_default_project_build%> - - - - <%else%> - - - - Rebuild - - <%/if%> + + <%include tmpl='#rebuild-template'/%>
@@ -187,12 +191,40 @@
- +
+ + + + + + +