154 lines
6.1 KiB
Python
154 lines
6.1 KiB
Python
from django.core.management.base import NoArgsCommand, CommandError
|
|
from django.db import transaction
|
|
from orm.models import Build, ToasterSetting, LogMessage, Target
|
|
from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException
|
|
from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable
|
|
import os
|
|
import logging
|
|
|
|
logger = logging.getLogger("ToasterScheduler")
|
|
|
|
class Command(NoArgsCommand):
|
|
args = ""
|
|
help = "Schedules and executes build requests as possible. Does not return (interrupt with Ctrl-C)"
|
|
|
|
|
|
@transaction.commit_on_success
|
|
def _selectBuildEnvironment(self):
|
|
bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE)
|
|
bec.be.lock = BuildEnvironment.LOCK_LOCK
|
|
bec.be.save()
|
|
return bec
|
|
|
|
@transaction.commit_on_success
|
|
def _selectBuildRequest(self):
|
|
br = BuildRequest.objects.filter(state = BuildRequest.REQ_QUEUED).order_by('pk')[0]
|
|
br.state = BuildRequest.REQ_INPROGRESS
|
|
br.save()
|
|
return br
|
|
|
|
def schedule(self):
|
|
import traceback
|
|
try:
|
|
br = None
|
|
try:
|
|
# select the build environment and the request to build
|
|
br = self._selectBuildRequest()
|
|
except IndexError as e:
|
|
#logger.debug("runbuilds: No build request")
|
|
return
|
|
try:
|
|
bec = self._selectBuildEnvironment()
|
|
except IndexError as e:
|
|
# we could not find a BEC; postpone the BR
|
|
br.state = BuildRequest.REQ_QUEUED
|
|
br.save()
|
|
logger.debug("runbuilds: No build env")
|
|
return
|
|
|
|
logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be))
|
|
|
|
# write the build identification variable
|
|
BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk))
|
|
|
|
# let the build request know where it is being executed
|
|
br.environment = bec.be
|
|
br.save()
|
|
|
|
# this triggers an async build
|
|
bec.triggerBuild(br.brbitbake_set.all(), br.brlayer_set.all(), br.brvariable_set.all(), br.brtarget_set.all())
|
|
|
|
except Exception as e:
|
|
logger.error("runbuilds: Error launching build %s" % e)
|
|
traceback.print_exc(e)
|
|
if "[Errno 111] Connection refused" in str(e):
|
|
# Connection refused, read toaster_server.out
|
|
errmsg = bec.readServerLogFile()
|
|
else:
|
|
errmsg = str(e)
|
|
|
|
BRError.objects.create(req = br,
|
|
errtype = str(type(e)),
|
|
errmsg = errmsg,
|
|
traceback = traceback.format_exc(e))
|
|
br.state = BuildRequest.REQ_FAILED
|
|
br.save()
|
|
bec.be.lock = BuildEnvironment.LOCK_FREE
|
|
bec.be.save()
|
|
|
|
def archive(self):
|
|
''' archives data from the builds '''
|
|
artifact_storage_dir = ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value
|
|
for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE):
|
|
# save cooker log
|
|
if br.build == None:
|
|
br.state = BuildRequest.REQ_FAILED
|
|
br.save()
|
|
continue
|
|
build_artifact_storage_dir = os.path.join(artifact_storage_dir, "%d" % br.build.pk)
|
|
try:
|
|
os.makedirs(build_artifact_storage_dir)
|
|
except OSError as ose:
|
|
if "File exists" in str(ose):
|
|
pass
|
|
else:
|
|
raise ose
|
|
|
|
file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt")
|
|
try:
|
|
with open(file_name, "w") as f:
|
|
f.write(br.environment.get_artifact(br.build.cooker_log_path).read())
|
|
except IOError:
|
|
os.unlink(file_name)
|
|
|
|
br.state = BuildRequest.REQ_COMPLETED
|
|
br.save()
|
|
|
|
def cleanup(self):
|
|
from django.utils import timezone
|
|
from datetime import timedelta
|
|
# environments locked for more than 30 seconds - they should be unlocked
|
|
BuildEnvironment.objects.filter(buildrequest__state__in=[BuildRequest.REQ_FAILED, BuildRequest.REQ_COMPLETED]).filter(lock=BuildEnvironment.LOCK_LOCK).filter(updated__lt = timezone.now() - timedelta(seconds = 30)).update(lock = BuildEnvironment.LOCK_FREE)
|
|
|
|
|
|
# update all Builds that failed to start
|
|
|
|
for br in BuildRequest.objects.filter(state = BuildRequest.REQ_FAILED, build__outcome = Build.IN_PROGRESS):
|
|
# transpose the launch errors in ToasterExceptions
|
|
br.build.outcome = Build.FAILED
|
|
for brerror in br.brerror_set.all():
|
|
logger.debug("Saving error %s" % brerror)
|
|
LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg)
|
|
br.build.save()
|
|
|
|
# we don't have a true build object here; hence, toasterui didn't have a change to release the BE lock
|
|
br.environment.lock = BuildEnvironment.LOCK_FREE
|
|
br.environment.save()
|
|
|
|
|
|
|
|
# update all BuildRequests without a build created
|
|
for br in BuildRequest.objects.filter(build = None):
|
|
br.build = Build.objects.create(project = br.project, completed_on = br.updated, started_on = br.created)
|
|
br.build.outcome = Build.REQ_FAILED
|
|
try:
|
|
br.build.machine = br.brvariable_set.get(name='MACHINE').value
|
|
except BRVariable.DoesNotExist:
|
|
pass
|
|
br.save()
|
|
# transpose target information
|
|
for brtarget in br.brtarget_set.all():
|
|
Target.objects.create(build = br.build, target= brtarget.target)
|
|
# transpose the launch errors in ToasterExceptions
|
|
for brerror in br.brerror_set.all():
|
|
LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg)
|
|
|
|
br.build.save()
|
|
pass
|
|
|
|
|
|
def handle_noargs(self, **options):
|
|
self.cleanup()
|
|
self.archive()
|
|
self.schedule()
|