bitbake: toaster: show progress of recipe parsing in recent builds area

Modify buildinfohelper and toasterui so that they record the
recipe parse progress (from ParseProgress events in bitbake)
on the Build object.

Note that because the Build object is now created at the
point when ParseStarted occurs, it is necessary to set the
build name to the empty string initially (hence the migration).
The build name can be set when the build properly starts,
i.e. at the BuildStarted event.

Then use this additional data to determine whether a Build
is in a "Parsing" state, and report this in the JSON API.
This enables the most recent builds area to show the recipe
parse progress.

Add additional logic to update the progress bar if the progress
for a build object changes.

[YOCTO #9631]

(Bitbake rev: f33d51d46d70e73e04e325807c1bc4eb68462f7b)

Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Elliot Smith 2016-07-11 14:47:06 +01:00 committed by Richard Purdie
parent 952ffb3e1f
commit dd99cf957d
8 changed files with 259 additions and 122 deletions

View File

@ -150,55 +150,48 @@ class ORMWrapper(object):
# pylint: disable=bad-continuation
# we do not follow the python conventions for continuation indentation due to long lines here
def create_build_object(self, build_info, brbe, project_id):
def create_build_object(self, build_info, brbe):
assert 'machine' in build_info
assert 'distro' in build_info
assert 'distro_version' in build_info
assert 'started_on' in build_info
assert 'cooker_log_path' in build_info
assert 'build_name' in build_info
assert 'bitbake_version' in build_info
prj = None
buildrequest = None
if brbe is not None: # this build was triggered by a request from a user
if brbe is not None:
# Toaster-triggered build
logger.debug(1, "buildinfohelper: brbe is %s" % brbe)
br, _ = brbe.split(":")
buildrequest = BuildRequest.objects.get(pk = br)
buildrequest = BuildRequest.objects.get(pk=br)
prj = buildrequest.project
elif project_id is not None: # this build was triggered by an external system for a specific project
logger.debug(1, "buildinfohelper: project is %s" % prj)
prj = Project.objects.get(pk = project_id)
else: # this build was triggered by a legacy system, or command line interactive mode
else:
# CLI build
prj = Project.objects.get_or_create_default_project()
logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj)
if buildrequest is not None:
build = buildrequest.build
logger.info("Updating existing build, with %s", build_info)
build.project = prj
build.machine=build_info['machine']
build.distro=build_info['distro']
build.distro_version=build_info['distro_version']
build.cooker_log_path=build_info['cooker_log_path']
build.build_name=build_info['build_name']
build.bitbake_version=build_info['bitbake_version']
build.machine = build_info['machine']
build.distro = build_info['distro']
build.distro_version = build_info['distro_version']
build.cooker_log_path = build_info['cooker_log_path']
build.bitbake_version = build_info['bitbake_version']
build.save()
else:
build = Build.objects.create(
project = prj,
machine=build_info['machine'],
distro=build_info['distro'],
distro_version=build_info['distro_version'],
started_on=build_info['started_on'],
completed_on=build_info['started_on'],
cooker_log_path=build_info['cooker_log_path'],
build_name=build_info['build_name'],
bitbake_version=build_info['bitbake_version'])
project=prj,
machine=build_info['machine'],
distro=build_info['distro'],
distro_version=build_info['distro_version'],
started_on=build_info['started_on'],
completed_on=build_info['started_on'],
cooker_log_path=build_info['cooker_log_path'],
bitbake_version=build_info['bitbake_version'])
logger.debug(1, "buildinfohelper: build is created %s" % build)
@ -208,6 +201,10 @@ class ORMWrapper(object):
return build
def update_build_name(self, build, build_name):
build.build_name = build_name
build.save()
@staticmethod
def get_or_create_targets(target_info):
"""
@ -924,9 +921,7 @@ class BuildInfoHelper(object):
build_info['started_on'] = timezone.now()
build_info['completed_on'] = timezone.now()
build_info['cooker_log_path'] = build_log_path
build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0]
return build_info
def _get_task_information(self, event, recipe):
@ -1032,17 +1027,28 @@ class BuildInfoHelper(object):
except NotExisting as nee:
logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee)
def store_started_build(self, event, build_log_path):
assert '_pkgs' in vars(event)
def store_started_build(self, build_log_path):
build_information = self._get_build_information(build_log_path)
self.internal_state['build'] = \
self.orm_wrapper.create_build_object(build_information, self.brbe)
# Update brbe and project as they can be changed for every build
self.project = build_information['project']
def save_build_name_and_targets(self, event):
# NB the BUILDNAME variable isn't set until BuildInit (or
# BuildStarted for older bitbakes)
build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
self.orm_wrapper.update_build_name(self.internal_state['build'],
build_name)
build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
# create target information
assert '_pkgs' in vars(event)
target_information = {}
target_information['targets'] = event._pkgs
target_information['build'] = self.internal_state['build']
self.internal_state['build'] = build_obj
self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
def save_build_layers_and_variables(self):
build_obj = self.internal_state['build']
# save layer version information for this build
if not 'lvs' in self.internal_state:
@ -1053,13 +1059,6 @@ class BuildInfoHelper(object):
del self.internal_state['lvs']
# create target information
target_information = {}
target_information['targets'] = event._pkgs
target_information['build'] = build_obj
self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
# Save build configuration
data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
@ -1090,6 +1089,41 @@ class BuildInfoHelper(object):
return self.brbe
def set_recipes_to_parse(self, num_recipes):
"""
Set the number of recipes which need to be parsed for this build.
This is set the first time ParseStarted is received by toasterui.
"""
if self.internal_state['build']:
self.internal_state['build'].recipes_to_parse = num_recipes
self.internal_state['build'].save()
def set_recipes_parsed(self, num_recipes):
"""
Set the number of recipes parsed so far for this build; this is updated
each time a ParseProgress or ParseCompleted event is received by
toasterui.
"""
if self.internal_state['build']:
if num_recipes <= self.internal_state['build'].recipes_to_parse:
self.internal_state['build'].recipes_parsed = num_recipes
self.internal_state['build'].save()
def update_target_image_file(self, event):
evdata = BuildInfoHelper._get_data_from_event(event)
for t in self.internal_state['targets']:
if t.is_image == True:
output_files = list(evdata.keys())
for output in output_files:
if t.target in output and 'rootfs' in output and not output.endswith(".manifest"):
self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
def update_artifact_image_file(self, event):
evdata = BuildInfoHelper._get_data_from_event(event)
for artifact_path in evdata.keys():
self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path])
def update_build_information(self, event, errors, warnings, taskfailures):
if 'build' in self.internal_state:
self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)

View File

@ -102,6 +102,7 @@ _evt_list = [
"bb.command.CommandExit",
"bb.command.CommandFailed",
"bb.cooker.CookerExit",
"bb.event.BuildInit",
"bb.event.BuildCompleted",
"bb.event.BuildStarted",
"bb.event.CacheLoadCompleted",
@ -115,6 +116,7 @@ _evt_list = [
"bb.event.NoProvider",
"bb.event.ParseCompleted",
"bb.event.ParseProgress",
"bb.event.ParseStarted",
"bb.event.RecipeParsed",
"bb.event.SanityCheck",
"bb.event.SanityCheckPassed",
@ -231,19 +233,30 @@ def main(server, eventHandler, params):
# pylint: disable=protected-access
# the code will look into the protected variables of the event; no easy way around this
# we treat ParseStarted as the first event of toaster-triggered
# builds; that way we get the Build Configuration included in the log
# and any errors that occur before BuildStarted is fired
if isinstance(event, bb.event.ParseStarted):
if not (build_log and build_log_file_path):
build_log, build_log_file_path = _open_build_log(log_dir)
buildinfohelper.store_started_build(build_log_file_path)
buildinfohelper.set_recipes_to_parse(event.total)
continue
if isinstance(event, bb.event.BuildStarted):
if not (build_log and build_log_file_path):
build_log, build_log_file_path = _open_build_log(log_dir)
# create a build object in buildinfohelper from either BuildInit
# (if available) or BuildStarted (for jethro and previous versions)
if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)):
buildinfohelper.save_build_name_and_targets(event)
buildinfohelper.store_started_build(event, build_log_file_path)
# get additional data from BuildStarted
if isinstance(event, bb.event.BuildStarted):
buildinfohelper.save_build_layers_and_variables()
continue
if isinstance(event, bb.event.ParseProgress):
buildinfohelper.set_recipes_parsed(event.current)
continue
if isinstance(event, bb.event.ParseCompleted):
buildinfohelper.set_recipes_parsed(event.total)
continue
if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
@ -289,10 +302,6 @@ def main(server, eventHandler, params):
# timing and error informations from the parsing phase in Toaster
if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)):
continue
if isinstance(event, bb.event.ParseProgress):
continue
if isinstance(event, bb.event.ParseCompleted):
continue
if isinstance(event, bb.event.CacheLoadStarted):
continue
if isinstance(event, bb.event.CacheLoadProgress):

View File

@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('orm', '0012_use_release_instead_of_up_branch'),
]
operations = [
migrations.AddField(
model_name='build',
name='recipes_parsed',
field=models.IntegerField(default=0),
),
migrations.AddField(
model_name='build',
name='recipes_to_parse',
field=models.IntegerField(default=1),
),
]

View File

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('orm', '0013_recipe_parse_progress_fields'),
]
operations = [
migrations.AlterField(
model_name='build',
name='build_name',
field=models.CharField(default='', max_length=100),
),
]

View File

@ -397,9 +397,15 @@ class Build(models.Model):
completed_on = models.DateTimeField()
outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
cooker_log_path = models.CharField(max_length=500)
build_name = models.CharField(max_length=100)
build_name = models.CharField(max_length=100, default='')
bitbake_version = models.CharField(max_length=50)
# number of recipes to parse for this build
recipes_to_parse = models.IntegerField(default=1)
# number of recipes parsed so far for this build
recipes_parsed = models.IntegerField(default=0)
@staticmethod
def get_recent(project=None):
"""
@ -615,6 +621,13 @@ class Build(models.Model):
else:
return False
def is_parsing(self):
"""
True if the build is still parsing recipes
"""
return self.outcome == Build.IN_PROGRESS and \
self.recipes_parsed < self.recipes_to_parse
def get_state(self):
"""
Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress',
@ -628,6 +641,8 @@ class Build(models.Model):
return 'Cancelling';
elif self.is_queued():
return 'Queued'
elif self.is_parsing():
return 'Parsing'
else:
return self.get_outcome_text()

View File

@ -87,7 +87,7 @@ class XhrBuildRequest(View):
br.save()
except BuildRequest.DoesNotExist:
return error_response('No such build id %s' % i)
return error_response('No such build request id %s' % i)
return error_response('ok')
@ -256,6 +256,14 @@ class MostRecentBuildsView(View):
build['id'] = build_obj.pk
build['dashboard_url'] = dashboard_url
buildrequest_id = None
if hasattr(build_obj, 'buildrequest'):
buildrequest_id = build_obj.buildrequest.pk
build['buildrequest_id'] = buildrequest_id
build['recipes_parsed_percentage'] = \
int((build_obj.recipes_parsed / build_obj.recipes_to_parse) * 100)
tasks_complete_percentage = 0
if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
tasks_complete_percentage = 100

View File

@ -33,8 +33,8 @@ function mrbSectionInit(ctx){
return buildData[build.id] || {};
}
// returns true if a build's state changed to "Succeeded" or "Failed"
// from some other value
// returns true if a build's state changed to "Succeeded", "Failed"
// or "Cancelled" from some other value
function buildFinished(build) {
var cached = getCached(build);
return cached.state &&
@ -49,12 +49,18 @@ function mrbSectionInit(ctx){
return (cached.state !== build.state);
}
// returns true if the complete_percentage changed
function progressChanged(build) {
// returns true if the tasks_complete_percentage changed
function tasksProgressChanged(build) {
var cached = getCached(build);
return (cached.tasks_complete_percentage !== build.tasks_complete_percentage);
}
// returns true if the number of recipes parsed/to parse changed
function recipeProgressChanged(build) {
var cached = getCached(build);
return (cached.recipes_parsed_percentage !== build.recipes_parsed_percentage);
}
function refreshMostRecentBuilds(){
libtoaster.getMostRecentBuilds(
libtoaster.ctx.mostRecentBuildsUrl,
@ -68,10 +74,6 @@ function mrbSectionInit(ctx){
var colourClass;
var elements;
// classes on the parent which signify the build state and affect
// the colour of the container for the build
var buildStateClasses = 'alert-info alert-success alert-danger';
for (var i = 0; i < data.length; i++) {
build = data[i];
@ -91,31 +93,25 @@ function mrbSectionInit(ctx){
container = $(selector);
container.html(html);
// style the outermost container for this build to reflect
// the new build state (red, green, blue);
// NB class set here should be in buildStateClasses
colourClass = 'alert-info';
if (build.state == 'Succeeded') {
colourClass = 'alert-success';
}
else if (build.state == 'Failed') {
colourClass = 'alert-danger';
}
elements = $('[data-latest-build-result="' + build.id + '"]');
elements.removeClass(buildStateClasses);
elements.addClass(colourClass);
}
else if (progressChanged(build)) {
// update the progress text
else if (tasksProgressChanged(build)) {
// update the task progress text
selector = '#build-pc-done-' + build.id;
$(selector).html(build.tasks_complete_percentage);
// update the progress bar
// update the task progress bar
selector = '#build-pc-done-bar-' + build.id;
$(selector).width(build.tasks_complete_percentage + '%');
}
else if (recipeProgressChanged(build)) {
// update the recipe progress text
selector = '#recipes-parsed-percentage-' + build.id;
$(selector).html(build.recipes_parsed_percentage);
// update the recipe progress bar
selector = '#recipes-parsed-percentage-bar-' + build.id;
$(selector).width(build.recipes_parsed_percentage + '%');
}
buildData[build.id] = build;
}
@ -128,6 +124,6 @@ function mrbSectionInit(ctx){
);
}
window.setInterval(refreshMostRecentBuilds, 1000);
window.setInterval(refreshMostRecentBuilds, 1500);
refreshMostRecentBuilds();
}

View File

@ -26,7 +26,9 @@
<div class="row project-name">
<div class="col-md-12">
<small>
<a class="alert-link text-uppercase" href={% project_url build.project %}>{{build.project.name}}</a>
<a class="alert-link text-uppercase" href="{% project_url build.project %}">
{{build.project.name}}
</a>
</small>
</div>
</div>
@ -52,14 +54,18 @@
<%:targets_abbreviated%>
</span>
</a>
<%else%>
<%else targets_abbreviated !== ''%>
<span data-toggle="tooltip" data-role="targets-text" title="Recipes: <%:targets%>">
<%:targets_abbreviated%>
</span>
<%else%>
...targets not yet available...
<%/if%>
</div>
<%if state == 'Queued'%>
<%if state == 'Parsing'%>
<%include tmpl='#parsing-recipes-build-template'/%>
<%else state == 'Queued'%>
<%include tmpl='#queued-build-template'/%>
<%else state == 'Succeeded' || state == 'Failed'%>
<%include tmpl='#succeeded-or-failed-build-template'/%>
@ -75,21 +81,38 @@
<!-- queued build -->
<script id="queued-build-template" type="text/x-jsrender">
<div class="col-md-5">
<span class="glyphicon glyphicon-question-sign get-help get-help-blue" title="This build is waiting for
the build directory to become available"></span>
Build queued
</div>
<div class="col-md-4">
<%if is_default_project_build%>
<!-- no cancel icon -->
<span class="glyphicon glyphicon-question-sign get-help get-help-blue pull-right" title="Builds in this project cannot be cancelled from Toaster: they can only be cancelled from the command line"></span>
<%else%>
<!-- cancel button -->
<span class="cancel-build-btn pull-right alert-link"
data-buildrequest-id="<%:id%>" data-request-url="<%:cancel_url%>">
<span class="glyphicon glyphicon-remove-circle"></span>
Cancel
</span>
<%/if%>
<!-- cancel button -->
<%include tmpl='#cancel-template'/%>
</div>
</script>
<!-- parsing recipes build -->
<script id="parsing-recipes-build-template" type="text/x-jsrender">
<!-- progress bar and parse completion percentage -->
<div data-role="build-status" class="col-md-4 col-md-offset-1 progress-info">
<!-- progress bar -->
<div class="progress">
<div id="recipes-parsed-percentage-bar-<%:id%>"
style="width: <%:recipes_parsed_percentage%>%;"
class="progress-bar">
</div>
</div>
</div>
<div class="col-md-4 progress-info">
<!-- parse completion percentage -->
<span class="glyphicon glyphicon-question-sign get-help get-help-blue" title="BitBake is parsing the layers required for your build"></span>
Parsing <span id="recipes-parsed-percentage-<%:id%>"><%:recipes_parsed_percentage%></span>% complete
<%include tmpl='#cancel-template'/%>
</div>
</script>
@ -110,17 +133,9 @@
<!-- task completion percentage -->
<span id="build-pc-done-<%:id%>"><%:tasks_complete_percentage%></span>% of
tasks complete
<%if is_default_project_build%>
<!-- no cancel icon -->
<span class="glyphicon glyphicon-question-sign get-help get-help-blue pull-right" title="Builds in this project cannot be cancelled from Toaster: they can only be cancelled from the command line"></span>
<%else%>
<!-- cancel button -->
<span class="cancel-build-btn pull-right alert-link"
data-buildrequest-id="<%:id%>" data-request-url="<%:cancel_url%>">
<span class="glyphicon glyphicon-remove-circle"></span>
Cancel
</span>
<%/if%>
<!-- cancel button -->
<%include tmpl='#cancel-template'/%>
</div>
</script>
@ -162,19 +177,8 @@
<div class="col-md-3">
Build time: <a class="alert-link" href="<%:buildtime_url%>"><%:buildtime%></a>
<%if is_default_project_build%>
<!-- info icon -->
<span class="pull-right glyphicon glyphicon-question-sign get-help <%if state == 'Success'%>get-help-green<%else state == 'Failed'%>get-help-red<%else%>get-help-blue<%/if%>"
title="Builds in this project cannot be started from Toaster: they are started from the command line">
</span>
<%else%>
<!-- rebuild button -->
<span class="rebuild-btn alert-link <%if state == 'Success'%>success<%else state == 'Failed'%>danger<%else%>info<%/if%> pull-right"
data-request-url="<%:rebuild_url%>" data-target='<%:build_targets_json%>'>
<span class="glyphicon glyphicon-repeat"></span>
Rebuild
</span>
<%/if%>
<!-- rebuild button -->
<%include tmpl='#rebuild-template'/%>
</div>
</script>
@ -187,12 +191,40 @@
<!-- rebuild button -->
<div class="col-md-3">
<span class="info pull-right rebuild-btn alert-link"
<%include tmpl='#rebuild-template'/%>
</div>
</script>
<!-- rebuild button or no rebuild icon -->
<script id="rebuild-template" type="text/x-jsrender">
<%if is_default_project_build%>
<!-- no rebuild info icon -->
<span class="pull-right glyphicon glyphicon-question-sign get-help <%if state == 'Success'%>get-help-green<%else state == 'Failed'%>get-help-red<%else%>get-help-blue<%/if%>"
title="Builds in this project cannot be started from Toaster: they are started from the command line">
</span>
<%else%>
<!-- rebuild button -->
<span class="rebuild-btn alert-link <%if state == 'Success'%>success<%else state == 'Failed'%>danger<%else%>info<%/if%> pull-right"
data-request-url="<%:rebuild_url%>" data-target='<%:build_targets_json%>'>
<span class="glyphicon glyphicon-repeat"></span>
Rebuild
</span>
</div>
<%/if%>
</script>
<!-- cancel button or no cancel icon -->
<script id="cancel-template" type="text/x-jsrender">
<%if is_default_project_build%>
<!-- no cancel icon -->
<span class="glyphicon glyphicon-question-sign get-help get-help-blue pull-right" title="Builds in this project cannot be cancelled from Toaster: they can only be cancelled from the command line"></span>
<%else%>
<!-- cancel button -->
<span class="cancel-build-btn pull-right alert-link"
data-buildrequest-id="<%:buildrequest_id%>" data-request-url="<%:cancel_url%>">
<span class="glyphicon glyphicon-remove-circle"></span>
Cancel
</span>
<%/if%>
</script>
<script>