2016-09-16 16:22:34 +00:00
|
|
|
from django.core.management.base import NoArgsCommand
|
2014-06-12 11:57:22 +00:00
|
|
|
from django.db import transaction
|
2016-04-06 16:46:40 +00:00
|
|
|
from django.db.models import Q
|
|
|
|
|
2016-04-06 16:46:41 +00:00
|
|
|
from bldcontrol.bbcontroller import getBuildEnvironmentController
|
|
|
|
from bldcontrol.models import BuildRequest, BuildEnvironment
|
|
|
|
from bldcontrol.models import BRError, BRVariable
|
|
|
|
|
2016-09-16 16:22:34 +00:00
|
|
|
from orm.models import Build, LogMessage, Target
|
2016-04-06 16:46:41 +00:00
|
|
|
|
2014-01-20 09:39:34 +00:00
|
|
|
import logging
|
2016-04-06 16:46:41 +00:00
|
|
|
import traceback
|
2016-09-16 16:22:31 +00:00
|
|
|
import signal
|
2016-11-24 11:19:52 +00:00
|
|
|
import os
|
2014-01-20 09:39:34 +00:00
|
|
|
|
2016-04-06 16:46:41 +00:00
|
|
|
logger = logging.getLogger("toaster")
|
2014-06-12 11:57:22 +00:00
|
|
|
|
2016-11-24 11:19:52 +00:00
|
|
|
|
2014-06-12 11:57:22 +00:00
|
|
|
class Command(NoArgsCommand):
|
2016-09-16 16:22:33 +00:00
|
|
|
args = ""
|
|
|
|
help = "Schedules and executes build requests as possible. "\
|
|
|
|
"Does not return (interrupt with Ctrl-C)"
|
2014-06-12 11:57:22 +00:00
|
|
|
|
2015-12-10 03:56:32 +00:00
|
|
|
@transaction.atomic
|
2014-06-12 11:57:22 +00:00
|
|
|
def _selectBuildEnvironment(self):
|
2016-09-16 16:22:33 +00:00
|
|
|
bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE)
|
2014-06-12 11:57:22 +00:00
|
|
|
bec.be.lock = BuildEnvironment.LOCK_LOCK
|
|
|
|
bec.be.save()
|
|
|
|
return bec
|
|
|
|
|
2015-12-10 03:56:32 +00:00
|
|
|
@transaction.atomic
|
2014-06-12 11:57:22 +00:00
|
|
|
def _selectBuildRequest(self):
|
2016-04-06 16:46:41 +00:00
|
|
|
br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first()
|
2014-06-12 11:57:22 +00:00
|
|
|
return br
|
|
|
|
|
|
|
|
def schedule(self):
|
|
|
|
try:
|
2016-04-06 16:46:41 +00:00
|
|
|
# select the build environment and the request to build
|
|
|
|
br = self._selectBuildRequest()
|
|
|
|
if br:
|
|
|
|
br.state = BuildRequest.REQ_INPROGRESS
|
|
|
|
br.save()
|
|
|
|
else:
|
2014-06-12 11:57:22 +00:00
|
|
|
return
|
2016-04-06 16:46:41 +00:00
|
|
|
|
2014-06-12 11:57:22 +00:00
|
|
|
try:
|
|
|
|
bec = self._selectBuildEnvironment()
|
|
|
|
except IndexError as e:
|
|
|
|
# we could not find a BEC; postpone the BR
|
|
|
|
br.state = BuildRequest.REQ_QUEUED
|
|
|
|
br.save()
|
2014-01-20 09:39:34 +00:00
|
|
|
logger.debug("runbuilds: No build env")
|
2014-06-12 11:57:22 +00:00
|
|
|
return
|
|
|
|
|
2016-11-24 11:19:52 +00:00
|
|
|
logger.info("runbuilds: starting build %s, environment %s" %
|
2016-05-10 08:04:49 +00:00
|
|
|
(br, bec.be))
|
2015-01-28 13:18:09 +00:00
|
|
|
|
2014-10-13 16:10:39 +00:00
|
|
|
# let the build request know where it is being executed
|
|
|
|
br.environment = bec.be
|
|
|
|
br.save()
|
2014-06-12 11:57:22 +00:00
|
|
|
|
2015-06-02 11:57:03 +00:00
|
|
|
# this triggers an async build
|
2016-04-06 16:46:41 +00:00
|
|
|
bec.triggerBuild(br.brbitbake, br.brlayer_set.all(),
|
|
|
|
br.brvariable_set.all(), br.brtarget_set.all(),
|
|
|
|
"%d:%d" % (br.pk, bec.be.pk))
|
2014-06-12 11:57:22 +00:00
|
|
|
|
|
|
|
except Exception as e:
|
2015-06-02 11:57:03 +00:00
|
|
|
logger.error("runbuilds: Error launching build %s" % e)
|
2016-06-10 09:34:12 +00:00
|
|
|
traceback.print_exc()
|
2015-02-10 16:25:17 +00:00
|
|
|
if "[Errno 111] Connection refused" in str(e):
|
|
|
|
# Connection refused, read toaster_server.out
|
|
|
|
errmsg = bec.readServerLogFile()
|
|
|
|
else:
|
|
|
|
errmsg = str(e)
|
|
|
|
|
2016-09-16 16:22:33 +00:00
|
|
|
BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg,
|
|
|
|
traceback=traceback.format_exc())
|
2014-07-15 18:31:10 +00:00
|
|
|
br.state = BuildRequest.REQ_FAILED
|
|
|
|
br.save()
|
|
|
|
bec.be.lock = BuildEnvironment.LOCK_FREE
|
|
|
|
bec.be.save()
|
|
|
|
|
2015-02-16 17:47:07 +00:00
|
|
|
def archive(self):
|
2016-09-16 16:22:33 +00:00
|
|
|
for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE):
|
2016-11-24 11:19:52 +00:00
|
|
|
if br.build is None:
|
2015-02-16 17:47:07 +00:00
|
|
|
br.state = BuildRequest.REQ_FAILED
|
bitbake: toaster: Avoid unnecessary local to local copy of cooker log
The cooker log is copied from its original (bitbake) location
to an artifact directory chosen by the user (chosen when
checksettings.py runs as part of the managed mode;
it's one of the annoying questions you're asked at startup).
The copy happens as part of the runbuilds script run, which
is started in a loop from the toaster startup script
in managed mode.
When a user requests the log for a build via toaster, they
are getting the log which has been copied to the artifact
directory, not the original bitbake log.
This works for the managed case, where the runbuilds command is
running in a loop and copying log files for completed builds to
the artifact directory. However, in analysis mode, there are two
problems:
1. checksettings isn't run, so the artifacts directory isn't
set. toaster is then unable to figure out where the log files
should have been copied to.
2. The log files aren't copied to the artifacts directory
anyway, as runbuilds isn't running in analysis mode.
To fix this, just point the user to the local bitbake log file
in its original location. This avoids the copy step, and means
we can remove a whole question from the toaster startup sequence,
as we no longer need an artifact directory.
The only downside to this is that it's not appropriate for
remote bitbake servers. We will need to revisit this and
possibly reinstate the copy step once we have to reconcile
local and remote builds and make their logs available in
the same way.
[YOCTO #8209]
(Bitbake rev: 5697bbcc88edad85891f66d28b8803a9c9d27ff2)
Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: brian avery <avery.brian@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2015-09-11 20:57:29 +00:00
|
|
|
else:
|
|
|
|
br.state = BuildRequest.REQ_COMPLETED
|
2015-02-16 17:47:07 +00:00
|
|
|
br.save()
|
2014-01-20 09:39:34 +00:00
|
|
|
|
2014-06-12 11:57:22 +00:00
|
|
|
def cleanup(self):
|
|
|
|
from django.utils import timezone
|
|
|
|
from datetime import timedelta
|
2016-04-06 16:46:40 +00:00
|
|
|
# environments locked for more than 30 seconds
|
|
|
|
# they should be unlocked
|
|
|
|
BuildEnvironment.objects.filter(
|
|
|
|
Q(buildrequest__state__in=[BuildRequest.REQ_FAILED,
|
|
|
|
BuildRequest.REQ_COMPLETED,
|
|
|
|
BuildRequest.REQ_CANCELLING]) &
|
|
|
|
Q(lock=BuildEnvironment.LOCK_LOCK) &
|
2016-09-16 16:22:33 +00:00
|
|
|
Q(updated__lt=timezone.now() - timedelta(seconds=30))
|
2016-04-06 16:46:40 +00:00
|
|
|
).update(lock=BuildEnvironment.LOCK_FREE)
|
|
|
|
|
2016-04-06 16:46:41 +00:00
|
|
|
# update all Builds that were in progress and failed to start
|
2016-11-24 11:19:52 +00:00
|
|
|
for br in BuildRequest.objects.filter(
|
|
|
|
state=BuildRequest.REQ_FAILED,
|
|
|
|
build__outcome=Build.IN_PROGRESS):
|
2015-06-11 17:27:53 +00:00
|
|
|
# transpose the launch errors in ToasterExceptions
|
2015-06-17 16:30:34 +00:00
|
|
|
br.build.outcome = Build.FAILED
|
2015-06-11 17:27:53 +00:00
|
|
|
for brerror in br.brerror_set.all():
|
2015-06-17 11:27:48 +00:00
|
|
|
logger.debug("Saving error %s" % brerror)
|
2016-04-06 16:46:41 +00:00
|
|
|
LogMessage.objects.create(build=br.build,
|
|
|
|
level=LogMessage.EXCEPTION,
|
|
|
|
message=brerror.errmsg)
|
2015-06-11 17:27:53 +00:00
|
|
|
br.build.save()
|
|
|
|
|
2016-04-06 16:46:41 +00:00
|
|
|
# we don't have a true build object here; hence, toasterui
|
|
|
|
# didn't have a change to release the BE lock
|
2015-06-17 16:30:34 +00:00
|
|
|
br.environment.lock = BuildEnvironment.LOCK_FREE
|
|
|
|
br.environment.save()
|
|
|
|
|
2015-06-11 17:27:53 +00:00
|
|
|
# update all BuildRequests without a build created
|
2016-09-16 16:22:33 +00:00
|
|
|
for br in BuildRequest.objects.filter(build=None):
|
2016-04-06 16:46:41 +00:00
|
|
|
br.build = Build.objects.create(project=br.project,
|
|
|
|
completed_on=br.updated,
|
|
|
|
started_on=br.created)
|
2015-08-18 16:28:56 +00:00
|
|
|
br.build.outcome = Build.FAILED
|
2015-06-11 17:27:53 +00:00
|
|
|
try:
|
|
|
|
br.build.machine = br.brvariable_set.get(name='MACHINE').value
|
|
|
|
except BRVariable.DoesNotExist:
|
|
|
|
pass
|
|
|
|
br.save()
|
|
|
|
# transpose target information
|
|
|
|
for brtarget in br.brtarget_set.all():
|
2016-04-06 16:46:41 +00:00
|
|
|
Target.objects.create(build=br.build,
|
|
|
|
target=brtarget.target,
|
|
|
|
task=brtarget.task)
|
2015-06-11 17:27:53 +00:00
|
|
|
# transpose the launch errors in ToasterExceptions
|
|
|
|
for brerror in br.brerror_set.all():
|
2016-04-06 16:46:41 +00:00
|
|
|
LogMessage.objects.create(build=br.build,
|
|
|
|
level=LogMessage.EXCEPTION,
|
|
|
|
message=brerror.errmsg)
|
2015-06-11 17:27:53 +00:00
|
|
|
|
|
|
|
br.build.save()
|
2014-06-12 11:57:22 +00:00
|
|
|
|
2016-04-06 16:46:40 +00:00
|
|
|
# Make sure the LOCK is removed for builds which have been fully
|
|
|
|
# cancelled
|
2016-11-24 11:19:52 +00:00
|
|
|
for br in BuildRequest.objects.filter(
|
2016-09-16 16:22:33 +00:00
|
|
|
Q(build__outcome=Build.CANCELLED) &
|
|
|
|
Q(state=BuildRequest.REQ_CANCELLING) &
|
|
|
|
~Q(environment=None)):
|
2016-04-06 16:46:40 +00:00
|
|
|
br.environment.lock = BuildEnvironment.LOCK_FREE
|
|
|
|
br.environment.save()
|
|
|
|
|
2016-05-19 12:59:29 +00:00
|
|
|
def runbuild(self):
|
|
|
|
try:
|
|
|
|
self.cleanup()
|
|
|
|
except Exception as e:
|
|
|
|
logger.warn("runbuilds: cleanup exception %s" % str(e))
|
2014-06-12 11:57:22 +00:00
|
|
|
|
2016-05-19 12:59:29 +00:00
|
|
|
try:
|
|
|
|
self.archive()
|
|
|
|
except Exception as e:
|
|
|
|
logger.warn("runbuilds: archive exception %s" % str(e))
|
2016-04-06 16:46:41 +00:00
|
|
|
|
2016-05-19 12:59:29 +00:00
|
|
|
try:
|
|
|
|
self.schedule()
|
|
|
|
except Exception as e:
|
|
|
|
logger.warn("runbuilds: schedule exception %s" % str(e))
|
2015-12-02 18:02:40 +00:00
|
|
|
|
2016-05-19 12:59:29 +00:00
|
|
|
def handle_noargs(self, **options):
|
2016-11-24 11:19:52 +00:00
|
|
|
pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
|
|
|
|
".runbuilds.pid")
|
|
|
|
|
|
|
|
with open(pidfile_path, 'w') as pidfile:
|
|
|
|
pidfile.write("%s" % os.getpid())
|
|
|
|
|
2016-09-16 16:22:32 +00:00
|
|
|
self.runbuild()
|
|
|
|
|
2016-09-16 16:22:31 +00:00
|
|
|
signal.signal(signal.SIGUSR1, lambda sig, frame: None)
|
|
|
|
|
2016-05-19 12:59:29 +00:00
|
|
|
while True:
|
2016-09-16 16:22:31 +00:00
|
|
|
signal.pause()
|
2016-05-19 12:59:29 +00:00
|
|
|
self.runbuild()
|