sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake: toasterui: ensure that the Build object is always available 

Many of the methods in toasterui and buildinfohelper rely
on the internal state of the buildinfohelper; in particular, they
need a Build object to have been created on the buildinfohelper.

If the creation of this Build object is tied to an event which
may or may not occur, there's no guarantee that it will exist.
This then causes assertion errors in those methods.

To prevent this from happening, add an _ensure_build() method
to buildinfohelper. This ensures that a minimal Build object
is always available whenever it is needed, either by retrieving
it from the BuildRequest or creating it; it also ensures that
the Build object is up to date with whatever data is available
on the bitbake server (DISTRO, MACHINE etc.).

This method is then called by any other method which relies on
a Build object being in the internal state, ensuring that the
object is either available, or creating it.

(Bitbake rev: 0990b4c73f194ec0be1762e4e48b1a525d8349fb)

Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Elliot Smith 2016-08-04 16:32:33 +01:00 committed by Richard Purdie
parent f17ab95c79
commit f6261da9c0
2 changed files with 122 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

196
bitbake/lib/bb/ui/buildinfohelper.py
View File

@ -150,14 +150,7 @@ class ORMWrapper(object):
# pylint: disable=bad-continuation # pylint: disable=bad-continuation
# we do not follow the python conventions for continuation indentation due to long lines here # we do not follow the python conventions for continuation indentation due to long lines here
def create_build_object(self, build_info, brbe): def get_or_create_build_object(self, brbe):
assert 'machine' in build_info
assert 'distro' in build_info
assert 'distro_version' in build_info
assert 'started_on' in build_info
assert 'cooker_log_path' in build_info
assert 'bitbake_version' in build_info
prj = None prj = None
buildrequest = None buildrequest = None
if brbe is not None: if brbe is not None:
@ -172,26 +165,18 @@ class ORMWrapper(object):
logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj)
if buildrequest is not None: if buildrequest is not None:
# reuse existing Build object
build = buildrequest.build build = buildrequest.build
logger.info("Updating existing build, with %s", build_info)
build.project = prj build.project = prj
build.machine = build_info['machine']
build.distro = build_info['distro']
build.distro_version = build_info['distro_version']
build.cooker_log_path = build_info['cooker_log_path']
build.bitbake_version = build_info['bitbake_version']
build.save() build.save()
else: else:
# create new Build object
now = timezone.now()
build = Build.objects.create( build = Build.objects.create(
project=prj, project=prj,
machine=build_info['machine'], started_on=now,
distro=build_info['distro'], completed_on=now,
distro_version=build_info['distro_version'], build_name='')
started_on=build_info['started_on'],
completed_on=build_info['started_on'],
cooker_log_path=build_info['cooker_log_path'],
bitbake_version=build_info['bitbake_version'])
logger.debug(1, "buildinfohelper: build is created %s" % build) logger.debug(1, "buildinfohelper: build is created %s" % build)
@ -201,8 +186,9 @@ class ORMWrapper(object):
return build return build
def update_build_name(self, build, build_name): def update_build(self, build, data_dict):
build.build_name = build_name for key in data_dict:
setattr(build, key, data_dict[key])
build.save() build.save()
@staticmethod @staticmethod
@ -227,7 +213,7 @@ class ORMWrapper(object):
result.append(obj) result.append(obj)
return result return result
def update_build_object(self, build, errors, warnings, taskfailures): def update_build_stats_and_outcome(self, build, errors, warnings, taskfailures):
assert isinstance(build,Build) assert isinstance(build,Build)
assert isinstance(errors, int) assert isinstance(errors, int)
assert isinstance(warnings, int) assert isinstance(warnings, int)
@ -912,20 +898,55 @@ class BuildInfoHelper(object):
################### ###################
## methods to convert event/external info into objects that the ORM layer uses ## methods to convert event/external info into objects that the ORM layer uses
def _ensure_build(self):
"""
Ensure the current build object exists and is up to date with
data on the bitbake server
"""
if not 'build' in self.internal_state or not self.internal_state['build']:
# create the Build object
self.internal_state['build'] = \
self.orm_wrapper.get_or_create_build_object(self.brbe)
def _get_build_information(self, build_log_path): build = self.internal_state['build']
# update missing fields on the Build object with found data
build_info = {} build_info = {}
build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0]
build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0] # set to True if at least one field is going to be set
build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] changed = False
build_info['started_on'] = timezone.now()
build_info['completed_on'] = timezone.now() if not build.build_name:
build_info['cooker_log_path'] = build_log_path build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
return build_info # only reset the build name if the one on the server is actually
# a valid value for the build_name field
if build_name != None:
build_info['build_name'] = build_name
changed = True
if not build.machine:
build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0]
changed = True
if not build.distro:
build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0]
changed = True
if not build.distro_version:
build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
changed = True
if not build.bitbake_version:
build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
changed = True
if changed:
self.orm_wrapper.update_build(self.internal_state['build'], build_info)
def _get_task_information(self, event, recipe): def _get_task_information(self, event, recipe):
assert 'taskname' in vars(event) assert 'taskname' in vars(event)
self._ensure_build()
task_information = {} task_information = {}
task_information['build'] = self.internal_state['build'] task_information['build'] = self.internal_state['build']
@ -940,8 +961,9 @@ class BuildInfoHelper(object):
return task_information return task_information
def _get_layer_version_for_path(self, path): def _get_layer_version_for_path(self, path):
self._ensure_build()
assert path.startswith("/") assert path.startswith("/")
assert 'build' in self.internal_state
def _slkey_interactive(layer_version): def _slkey_interactive(layer_version):
assert isinstance(layer_version, Layer_Version) assert isinstance(layer_version, Layer_Version)
@ -985,6 +1007,8 @@ class BuildInfoHelper(object):
return recipe_info return recipe_info
def _get_path_information(self, task_object): def _get_path_information(self, task_object):
self._ensure_build()
assert isinstance(task_object, Task) assert isinstance(task_object, Task)
build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/" build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
build_stats_path = [] build_stats_path = []
@ -1027,17 +1051,18 @@ class BuildInfoHelper(object):
except NotExisting as nee: except NotExisting as nee:
logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee) logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee)
def store_started_build(self, build_log_path): def store_started_build(self):
build_information = self._get_build_information(build_log_path) self._ensure_build()
self.internal_state['build'] = \
self.orm_wrapper.create_build_object(build_information, self.brbe)
def save_build_name_and_targets(self, event): def save_build_log_file_path(self, build_log_path):
# NB the BUILDNAME variable isn't set until BuildInit (or self._ensure_build()
# BuildStarted for older bitbakes)
build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0] if not self.internal_state['build'].cooker_log_path:
self.orm_wrapper.update_build_name(self.internal_state['build'], data_dict = {'cooker_log_path': build_log_path}
build_name) self.orm_wrapper.update_build(self.internal_state['build'], data_dict)
def save_build_targets(self, event):
self._ensure_build()
# create target information # create target information
assert '_pkgs' in vars(event) assert '_pkgs' in vars(event)
@ -1048,6 +1073,8 @@ class BuildInfoHelper(object):
self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
def save_build_layers_and_variables(self): def save_build_layers_and_variables(self):
self._ensure_build()
build_obj = self.internal_state['build'] build_obj = self.internal_state['build']
# save layer version information for this build # save layer version information for this build
@ -1094,9 +1121,9 @@ class BuildInfoHelper(object):
Set the number of recipes which need to be parsed for this build. Set the number of recipes which need to be parsed for this build.
This is set the first time ParseStarted is received by toasterui. This is set the first time ParseStarted is received by toasterui.
""" """
if self.internal_state['build']: self._ensure_build()
self.internal_state['build'].recipes_to_parse = num_recipes self.internal_state['build'].recipes_to_parse = num_recipes
self.internal_state['build'].save() self.internal_state['build'].save()
def set_recipes_parsed(self, num_recipes): def set_recipes_parsed(self, num_recipes):
""" """
@ -1104,10 +1131,10 @@ class BuildInfoHelper(object):
each time a ParseProgress or ParseCompleted event is received by each time a ParseProgress or ParseCompleted event is received by
toasterui. toasterui.
""" """
if self.internal_state['build']: self._ensure_build()
if num_recipes <= self.internal_state['build'].recipes_to_parse: if num_recipes <= self.internal_state['build'].recipes_to_parse:
self.internal_state['build'].recipes_parsed = num_recipes self.internal_state['build'].recipes_parsed = num_recipes
self.internal_state['build'].save() self.internal_state['build'].save()
def update_target_image_file(self, event): def update_target_image_file(self, event):
evdata = BuildInfoHelper._get_data_from_event(event) evdata = BuildInfoHelper._get_data_from_event(event)
@ -1120,13 +1147,17 @@ class BuildInfoHelper(object):
self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
def update_artifact_image_file(self, event): def update_artifact_image_file(self, event):
self._ensure_build()
evdata = BuildInfoHelper._get_data_from_event(event) evdata = BuildInfoHelper._get_data_from_event(event)
for artifact_path in evdata.keys(): for artifact_path in evdata.keys():
self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) self.orm_wrapper.save_artifact_information(
self.internal_state['build'], artifact_path,
evdata[artifact_path])
def update_build_information(self, event, errors, warnings, taskfailures): def update_build_information(self, event, errors, warnings, taskfailures):
if 'build' in self.internal_state: self._ensure_build()
self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) self.orm_wrapper.update_build_stats_and_outcome(
self.internal_state['build'], errors, warnings, taskfailures)
def store_started_task(self, event): def store_started_task(self, event):
assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)) assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
@ -1169,6 +1200,7 @@ class BuildInfoHelper(object):
def store_tasks_stats(self, event): def store_tasks_stats(self, event):
self._ensure_build()
task_data = BuildInfoHelper._get_data_from_event(event) task_data = BuildInfoHelper._get_data_from_event(event)
for (task_file, task_name, task_stats, recipe_name) in task_data: for (task_file, task_name, task_stats, recipe_name) in task_data:
@ -1264,6 +1296,8 @@ class BuildInfoHelper(object):
def store_target_package_data(self, event): def store_target_package_data(self, event):
self._ensure_build()
# for all image targets # for all image targets
for target in self.internal_state['targets']: for target in self.internal_state['targets']:
if target.is_image: if target.is_image:
@ -1297,10 +1331,9 @@ class BuildInfoHelper(object):
note that this only gets called for command line builds which are note that this only gets called for command line builds which are
interrupted, so it doesn't touch any BuildRequest objects interrupted, so it doesn't touch any BuildRequest objects
""" """
build = self.internal_state['build'] self._ensure_build()
if build: self.internal_state['build'].outcome = Build.CANCELLED
build.outcome = Build.CANCELLED self.internal_state['build'].save()
build.save()
def store_dependency_information(self, event): def store_dependency_information(self, event):
assert '_depgraph' in vars(event) assert '_depgraph' in vars(event)
@ -1446,6 +1479,8 @@ class BuildInfoHelper(object):
def store_build_package_information(self, event): def store_build_package_information(self, event):
self._ensure_build()
package_info = BuildInfoHelper._get_data_from_event(event) package_info = BuildInfoHelper._get_data_from_event(event)
self.orm_wrapper.save_build_package_information( self.orm_wrapper.save_build_package_information(
self.internal_state['build'], self.internal_state['build'],
@ -1461,6 +1496,10 @@ class BuildInfoHelper(object):
def _store_build_done(self, errorcode): def _store_build_done(self, errorcode):
logger.info("Build exited with errorcode %d", errorcode) logger.info("Build exited with errorcode %d", errorcode)
if not self.brbe:
return
br_id, be_id = self.brbe.split(":") br_id, be_id = self.brbe.split(":")
be = BuildEnvironment.objects.get(pk = be_id) be = BuildEnvironment.objects.get(pk = be_id)
be.lock = BuildEnvironment.LOCK_LOCK be.lock = BuildEnvironment.LOCK_LOCK
@ -1482,7 +1521,6 @@ class BuildInfoHelper(object):
br.state = BuildRequest.REQ_FAILED br.state = BuildRequest.REQ_FAILED
br.save() br.save()
def store_log_error(self, text): def store_log_error(self, text):
mockevent = MockEvent() mockevent = MockEvent()
mockevent.levelno = formatter.ERROR mockevent.levelno = formatter.ERROR
@ -1501,24 +1539,22 @@ class BuildInfoHelper(object):
def store_log_event(self, event): def store_log_event(self, event):
self._ensure_build()
if event.levelno < formatter.WARNING: if event.levelno < formatter.WARNING:
return return
if 'args' in vars(event): if 'args' in vars(event):
event.msg = event.msg % event.args event.msg = event.msg % event.args
if not 'build' in self.internal_state: # early return for CLI builds
if self.brbe is None: if self.brbe is None:
if not 'backlog' in self.internal_state: if not 'backlog' in self.internal_state:
self.internal_state['backlog'] = [] self.internal_state['backlog'] = []
self.internal_state['backlog'].append(event) self.internal_state['backlog'].append(event)
return return
else: # we're under Toaster control, the build is already created
br, _ = self.brbe.split(":")
buildrequest = BuildRequest.objects.get(pk = br)
self.internal_state['build'] = buildrequest.build
if 'build' in self.internal_state and 'backlog' in self.internal_state: if 'backlog' in self.internal_state:
# if we have a backlog of events, do our best to save them here # if we have a backlog of events, do our best to save them here
if len(self.internal_state['backlog']): if len(self.internal_state['backlog']):
tempevent = self.internal_state['backlog'].pop() tempevent = self.internal_state['backlog'].pop()
@ -1847,18 +1883,12 @@ class BuildInfoHelper(object):
sdk_target) sdk_target)
def close(self, errorcode): def close(self, errorcode):
if self.brbe is not None: self._store_build_done(errorcode)
self._store_build_done(errorcode)
if 'backlog' in self.internal_state: if 'backlog' in self.internal_state:
if 'build' in self.internal_state: # we save missed events in the database for the current build
# we save missed events in the database for the current build tempevent = self.internal_state['backlog'].pop()
tempevent = self.internal_state['backlog'].pop() self.store_log_event(tempevent)
self.store_log_event(tempevent)
else:
# we have no build, and we still have events; something amazingly wrong happend
for event in self.internal_state['backlog']:
logger.error("UNSAVED log: %s", event.msg)
if not connection.features.autocommits_when_autocommit_is_off: if not connection.features.autocommits_when_autocommit_is_off:
transaction.set_autocommit(True) transaction.set_autocommit(True)
@ -1867,3 +1897,7 @@ class BuildInfoHelper(object):
# being incorrectly attached to the previous Toaster-triggered build; # being incorrectly attached to the previous Toaster-triggered build;
# see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021 # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021
self.brbe = None self.brbe = None
# unset the internal Build object to prevent it being reused for the
# next build
self.internal_state['build'] = None

9
bitbake/lib/bb/ui/toasterui.py
View File

@ -237,14 +237,19 @@ def main(server, eventHandler, params):
if not (build_log and build_log_file_path): if not (build_log and build_log_file_path):
build_log, build_log_file_path = _open_build_log(log_dir) build_log, build_log_file_path = _open_build_log(log_dir)
buildinfohelper.store_started_build(build_log_file_path) buildinfohelper.store_started_build()
buildinfohelper.save_build_log_file_path(build_log_file_path)
buildinfohelper.set_recipes_to_parse(event.total) buildinfohelper.set_recipes_to_parse(event.total)
continue continue
# create a build object in buildinfohelper from either BuildInit # create a build object in buildinfohelper from either BuildInit
# (if available) or BuildStarted (for jethro and previous versions) # (if available) or BuildStarted (for jethro and previous versions)
if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)): if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)):
buildinfohelper.save_build_name_and_targets(event) if not (build_log and build_log_file_path):
build_log, build_log_file_path = _open_build_log(log_dir)
buildinfohelper.save_build_targets(event)
buildinfohelper.save_build_log_file_path(build_log_file_path)
# get additional data from BuildStarted # get additional data from BuildStarted
if isinstance(event, bb.event.BuildStarted): if isinstance(event, bb.event.BuildStarted):