diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index 491fd1566d..1096ccf4de 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py @@ -1114,7 +1114,8 @@ class BuildInfoHelper(object): be.save() br = BuildRequest.objects.get(pk = br_id) if errorcode == 0: - br.state = BuildRequest.REQ_COMPLETED + # request archival of the project artifacts + br.state = BuildRequest.REQ_ARCHIVE else: br.state = BuildRequest.REQ_FAILED br.save() diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index a85ad5a06a..df9f362284 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py @@ -58,7 +58,12 @@ def _log_settings_from_server(server): if error: logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error) raise BaseException(error) - return includelogs, loglines + consolelogfile, error = server.runCommand(["getVariable", "BB_CONSOLELOG"]) + if error: + logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" % error) + raise BaseException(error) + return includelogs, loglines, consolelogfile + def main(server, eventHandler, params ): @@ -71,7 +76,7 @@ def main(server, eventHandler, params ): console.setFormatter(format) logger.addHandler(console) - includelogs, loglines = _log_settings_from_server(server) + includelogs, loglines, consolelogfile = _log_settings_from_server(server) # verify and warn build_history_enabled = True @@ -96,6 +101,16 @@ def main(server, eventHandler, params ): buildinfohelper = BuildInfoHelper(server, build_history_enabled) + if buildinfohelper.brbe is not None and consolelogfile: + # if we are under managed mode we have no other UI and we need to write our own file + bb.utils.mkdirhier(os.path.dirname(consolelogfile)) + conlogformat = bb.msg.BBLogFormatter(format_str) + consolelog = logging.FileHandler(consolelogfile) + bb.msg.addDefaultlogFilter(consolelog) + consolelog.setFormatter(conlogformat) + logger.addHandler(consolelog) + + while True: try: event = eventHandler.waitEvent(0.25) @@ -115,8 +130,12 @@ def main(server, eventHandler, params ): if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): buildinfohelper.update_and_store_task(event) + logger.warn("Logfile for task %s" % event.logfile) continue + if isinstance(event, bb.build.TaskBase): + logger.info(event._message) + if isinstance(event, bb.event.LogExecTTY): logger.warn(event.msg) continue @@ -162,7 +181,12 @@ def main(server, eventHandler, params ): if isinstance(event, bb.event.CacheLoadCompleted): continue if isinstance(event, bb.event.MultipleProviders): + logger.info("multiple providers are available for %s%s (%s)", event._is_runtime and "runtime " or "", + event._item, + ", ".join(event._candidates)) + logger.info("consider defining a PREFERRED_PROVIDER entry to match %s", event._item) continue + if isinstance(event, bb.event.NoProvider): return_value = 1 errors = errors + 1 @@ -229,6 +253,7 @@ def main(server, eventHandler, params ): buildinfohelper.store_log_event(event) errors += 1 errorcode = 1 + logger.error("Command execution failed: %s", event.error) buildinfohelper.update_build_information(event, errors, warnings, taskfailures) buildinfohelper.close(errorcode) diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py index 5d80bc7155..1ff5c92833 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -62,6 +62,23 @@ class Command(NoArgsCommand): def handle(self, **options): + # verify that we have a settings for downloading artifacts + while ToasterSetting.objects.filter(name="ARTIFACTS_STORAGE_DIR").count() == 0: + guessedpath = os.getcwd() + "/toaster_build_artifacts/" + print("Toaster needs to know in which directory it can download build log files and other artifacts.\n Toaster suggests \"%s\"." % guessedpath) + artifacts_storage_dir = raw_input(" Press Enter to select \"%s\" or type the full path to a different directory: " % guessedpath) + if len(artifacts_storage_dir) == 0: + artifacts_storage_dir = guessedpath + if len(artifacts_storage_dir) > 0 and artifacts_storage_dir.startswith("/"): + try: + os.makedirs(artifacts_storage_dir) + except OSError as ose: + if "File exists" in str(ose): + pass + else: + raise ose + ToasterSetting.objects.create(name="ARTIFACTS_STORAGE_DIR", value=artifacts_storage_dir) + self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__))))))) # refuse to start if we have no build environments while BuildEnvironment.objects.count() == 0: diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index 3de582cc86..808318f14f 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -1,6 +1,6 @@ from django.core.management.base import NoArgsCommand, CommandError from django.db import transaction -from orm.models import Build +from orm.models import Build, ToasterSetting from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable import os @@ -93,7 +93,33 @@ class Command(NoArgsCommand): bec.be.lock = BuildEnvironment.LOCK_FREE bec.be.save() + def archive(self): + ''' archives data from the builds ''' + artifact_storage_dir = ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value + for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): + # save cooker log + if br.build == None: + br.state = BuildRequest.REQ_FAILED + br.save() + continue + build_artifact_storage_dir = os.path.join(artifact_storage_dir, "%d" % br.build.pk) + try: + os.makedirs(build_artifact_storage_dir) + except OSError as ose: + if "File exists" in str(ose): + pass + else: + raise ose + file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") + try: + with open(file_name, "w") as f: + f.write(br.environment.get_artifact(br.build.cooker_log_path).read()) + except IOError: + os.unlink(file_name) + + br.state = BuildRequest.REQ_COMPLETED + br.save() def cleanup(self): from django.utils import timezone @@ -104,4 +130,5 @@ class Command(NoArgsCommand): def handle_noargs(self, **options): self.cleanup() + self.archive() self.schedule() diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0008_brarchive.py b/bitbake/lib/toaster/bldcontrol/migrations/0008_brarchive.py new file mode 100644 index 0000000000..f5469607f3 --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/migrations/0008_brarchive.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models + +class Migration(DataMigration): + # ids that cannot be imported from BuildRequest + + def forwards(self, orm): + REQ_COMPLETED = 3 + REQ_ARCHIVE = 6 + "Write your forwards methods here." + # Note: Don't use "from appname.models import ModelName". + # Use orm.ModelName to refer to models in this application, + # and orm['appname.ModelName'] for models in other applications. + orm.BuildRequest.objects.filter(state=REQ_COMPLETED).update(state=REQ_ARCHIVE) + + def backwards(self, orm): + REQ_COMPLETED = 3 + REQ_ARCHIVE = 6 + "Write your backwards methods here." + orm.BuildRequest.objects.filter(state=REQ_ARCHIVE).update(state=REQ_COMPLETED) + + models = { + u'bldcontrol.brbitbake': { + 'Meta': {'object_name': 'BRBitbake'}, + 'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']", 'unique': 'True'}) + }, + u'bldcontrol.brerror': { + 'Meta': {'object_name': 'BRError'}, + 'errmsg': ('django.db.models.fields.TextField', [], {}), + 'errtype': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'traceback': ('django.db.models.fields.TextField', [], {}) + }, + u'bldcontrol.brlayer': { + 'Meta': {'object_name': 'BRLayer'}, + 'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}) + }, + u'bldcontrol.brtarget': { + 'Meta': {'object_name': 'BRTarget'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'}) + }, + u'bldcontrol.brvariable': { + 'Meta': {'object_name': 'BRVariable'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'value': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + u'bldcontrol.buildenvironment': { + 'Meta': {'object_name': 'BuildEnvironment'}, + 'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}), + 'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}), + 'betype': ('django.db.models.fields.IntegerField', [], {}), + 'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + u'bldcontrol.buildrequest': { + 'Meta': {'object_name': 'BuildRequest'}, + 'build': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['orm.Build']", 'unique': 'True', 'null': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'environment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildEnvironment']", 'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}), + 'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + u'orm.bitbakeversion': { + 'Meta': {'object_name': 'BitbakeVersion'}, + 'branch': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'giturl': ('django.db.models.fields.URLField', [], {'max_length': '200'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'}) + }, + u'orm.build': { + 'Meta': {'object_name': 'Build'}, + 'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'completed_on': ('django.db.models.fields.DateTimeField', [], {}), + 'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}), + 'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}), + 'started_on': ('django.db.models.fields.DateTimeField', [], {}), + 'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + u'orm.project': { + 'Meta': {'object_name': 'Project'}, + 'bitbake_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.BitbakeVersion']"}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'release': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Release']"}), + 'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'}) + }, + u'orm.release': { + 'Meta': {'object_name': 'Release'}, + 'bitbake_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.BitbakeVersion']"}), + 'branch_name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50'}), + 'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'helptext': ('django.db.models.fields.TextField', [], {'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'}) + } + } + + complete_apps = ['bldcontrol'] + symmetrical = True diff --git a/bitbake/lib/toaster/bldcontrol/models.py b/bitbake/lib/toaster/bldcontrol/models.py index dc4afca2f7..25d94cd3fe 100644 --- a/bitbake/lib/toaster/bldcontrol/models.py +++ b/bitbake/lib/toaster/bldcontrol/models.py @@ -94,6 +94,7 @@ class BuildRequest(models.Model): REQ_COMPLETED = 3 REQ_FAILED = 4 REQ_DELETED = 5 + REQ_ARCHIVE = 6 REQUEST_STATE = ( (REQ_CREATED, "created"), @@ -102,6 +103,7 @@ class BuildRequest(models.Model): (REQ_COMPLETED, "completed"), (REQ_FAILED, "failed"), (REQ_DELETED, "deleted"), + (REQ_ARCHIVE, "archive"), ) search_allowed_fields = ("brtarget__target",) diff --git a/bitbake/lib/toaster/toastergui/templates/build.html b/bitbake/lib/toaster/toastergui/templates/build.html index e71e38feb9..684ec65884 100644 --- a/bitbake/lib/toaster/toastergui/templates/build.html +++ b/bitbake/lib/toaster/toastergui/templates/build.html @@ -40,7 +40,9 @@ {% for build in objects %}
The artifact you are seeking to download is not available. We are sorry. You can go back +
+