sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake: toaster: change startup parameter passing to avoid race 

We avoid a race between the setting the TOASTER_BRBE variable
and reading the variable in toaster ui by supplying the variable
at server startup time through the toaster.conf post-read file.

Additional small changes are included, including marking the
build request with the environment id of where the build took place.

(Bitbake rev: 7c333350418c4140e6c988c5272940f8057d327d)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Alexandru DAMIAN 2014-10-13 17:10:39 +01:00 committed by Richard Purdie
parent a0660718e6
commit 2837b110ae
10 changed files with 219 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bitbake/bin/toaster
View File

@ -83,8 +83,10 @@ function webserverStartAll()
function addtoConfiguration()
{
echo "#Created by toaster start script" > ${BUILDDIR}/conf/$2
echo $1 >> ${BUILDDIR}/conf/$2
file=$1
shift
echo "#Created by toaster start script" > ${BUILDDIR}/conf/$file
for var in "$@"; do echo $var >> ${BUILDDIR}/conf/$file; done
}
INSTOPSYSTEM=0
@ -196,6 +198,7 @@ fi
NOTOASTERUI=0
WEBSERVER=1
TOASTER_BRBE=""
for param in $*; do
case $param in
noui )
@ -204,6 +207,8 @@ for param in $*; do
noweb )
WEBSERVER=0
;;
brbe=* )
TOASTER_BRBE=$'\n'"TOASTER_BRBE=\""${param#*=}"\""
esac
done
@ -228,7 +233,7 @@ fi
case $CMD in
start )
start_success=1
addtoConfiguration "INHERIT+=\"toaster buildhistory\"" toaster.conf
addtoConfiguration toaster.conf "INHERIT+=\"toaster buildhistory\"" $TOASTER_BRBE
if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then
echo "Failed ${CMD}."
return 4
@ -248,10 +253,12 @@ case $CMD in
# set fail safe stop system on terminal exit
trap stop_system SIGHUP
echo "Successful ${CMD}."
return 0
else
# failed start, do stop
stop_system
echo "Failed ${CMD}."
return 1
fi
# stop system on terminal exit
set -o monitor

13
bitbake/lib/bb/ui/buildinfohelper.py
View File

@ -933,14 +933,17 @@ class BuildInfoHelper(object):
self.internal_state['recipes'],
)
def _store_build_done(self):
def _store_build_done(self, errorcode):
br_id, be_id = self.brbe.split(":")
from bldcontrol.models import BuildEnvironment, BuildRequest
be = BuildEnvironment.objects.get(pk = be_id)
be.lock = BuildEnvironment.LOCK_LOCK
be.save()
br = BuildRequest.objects.get(pk = br_id)
br.state = BuildRequest.REQ_COMPLETED
if errorcode == 0:
br.state = BuildRequest.REQ_COMPLETED
else:
br.state = BuildRequest.REQ_FAILED
br.save()
@ -964,7 +967,7 @@ class BuildInfoHelper(object):
self.internal_state['backlog'].append(event)
else: # we're under Toaster control, post the errors to the build request
from bldcontrol.models import BuildRequest, BRError
br, be = brbe.split(":")
br, be = self.brbe.split(":")
buildrequest = BuildRequest.objects.get(pk = br)
brerror = BRError.objects.create(req = buildrequest, errtype="build", errmsg = event.msg)
@ -992,9 +995,9 @@ class BuildInfoHelper(object):
log_information['lineno'] = event.lineno
self.orm_wrapper.create_logmessage(log_information)
def close(self):
def close(self, errorcode):
if self.brbe is not None:
buildinfohelper._store_build_done()
self._store_build_done(errorcode)
if 'backlog' in self.internal_state:
for event in self.internal_state['backlog']:

5
bitbake/lib/bb/ui/toasterui.py
View File

@ -219,6 +219,7 @@ def main(server, eventHandler, params ):
if isinstance(event, (bb.command.CommandCompleted,
bb.command.CommandFailed,
bb.command.CommandExit)):
errorcode = 0
if (isinstance(event, bb.command.CommandFailed)):
event.levelno = format.ERROR
event.msg = "Command Failed " + event.error
@ -226,10 +227,10 @@ def main(server, eventHandler, params ):
event.lineno = 0
buildinfohelper.store_log_event(event)
errors += 1
errorcode = 1
buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
buildinfohelper.close()
buildinfohelper.close(errorcode)
# we start a new build info
if buildinfohelper.brbe is not None:

6
bitbake/lib/toaster/bldcontrol/bbcontroller.py
View File

@ -118,7 +118,7 @@ class BuildEnvironmentController(object):
self.connection = None
def startBBServer(self):
def startBBServer(self, brbe):
""" Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI.
After this method executes, self.be bbaddress/bbport MUST point to a running and free server,
and the bbstate MUST be updated to "started".
@ -142,12 +142,12 @@ class BuildEnvironmentController(object):
raise Exception("Must override setLayers")
def getBBController(self):
def getBBController(self, brbe):
""" returns a BitbakeController to an already started server; this is the point where the server
starts if needed; or reconnects to the server if we can
"""
if not self.connection:
self.startBBServer()
self.startBBServer(brbe)
self.be.lock = BuildEnvironment.LOCK_RUNNING
self.be.save()

39
bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
View File

@ -74,14 +74,35 @@ class LocalhostBEController(BuildEnvironmentController):
self._createdirpath(self.be.builddir)
self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
def startBBServer(self):
def startBBServer(self, brbe):
assert self.pokydirname and os.path.exists(self.pokydirname)
assert self.islayerset
print("DEBUG: executing ", "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
# FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
# but since they start async without any return, we just wait a bit
print "Started server"
try:
os.remove(os.path.join(self.be.builddir, "toaster_ui.log"))
except OSError as e:
import errno
if e.errno != errno.ENOENT:
raise
cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)
print("DEBUG: executing ", cmd)
print self._shellcmd(cmd)
def _toaster_ui_started(filepath):
if not os.path.exists(filepath):
return False
with open(filepath, "r") as f:
for line in f:
if line.startswith("NOTE: ToasterUI waiting for events"):
return True
return False
while not _toaster_ui_started(os.path.join(self.be.builddir, "toaster_ui.log")):
import time
print "DEBUG: Waiting server to start"
time.sleep(0.5)
print("DEBUG: Started server")
assert self.be.sourcedir and os.path.exists(self.be.builddir)
self.be.bbaddress = "localhost"
self.be.bbport = "8200"
@ -172,9 +193,13 @@ class LocalhostBEController(BuildEnvironmentController):
conflines = open(bblayerconf, "r").readlines()
bblayerconffile = open(bblayerconf, "w")
skip = 0
for i in xrange(len(conflines)):
if skip > 0:
skip =- 1
continue
if conflines[i].startswith("# line added by toaster"):
i += 2
skip = 1
else:
bblayerconffile.write(conflines[i])

19
bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
View File

@ -39,24 +39,29 @@ class Command(NoArgsCommand):
# we could not find a BEC; postpone the BR
br.state = BuildRequest.REQ_QUEUED
br.save()
print "No build env"
return
# set up the buid environment with the needed layers
print "Build %s, Environment %s" % (br, bec.be)
# let the build request know where it is being executed
br.environment = bec.be
br.save()
# set up the buid environment with the needed layers
bec.setLayers(br.brbitbake_set.all(), br.brlayer_set.all())
# get the bb server running
bbctrl = bec.getBBController()
# let toasterui that this is a managed build
bbctrl.setVariable("TOASTER_BRBE", "%d:%d" % (br.pk, bec.be.pk))
# get the bb server running with the build req id and build env id
bbctrl = bec.getBBController("%d:%d" % (br.pk, bec.be.pk))
# set the build configuration
for variable in br.brvariable_set.all():
bbctrl.setVariable(variable.name, variable.value)
# trigger the build command
bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all())))
task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, br.brtarget_set.all()))
if len(task) == 0:
task = None
bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all())), task)
print "Build launched, exiting"
# disconnect from the server

145
bitbake/lib/toaster/bldcontrol/migrations/0007_auto__add_field_buildrequest_environment__chg_field_buildrequest_build.py Normal file
View File

@ -0,0 +1,145 @@
# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'BuildRequest.environment'
db.add_column(u'bldcontrol_buildrequest', 'environment',
self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bldcontrol.BuildEnvironment'], null=True),
keep_default=False)
# Changing field 'BuildRequest.build'
db.alter_column(u'bldcontrol_buildrequest', 'build_id', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['orm.Build'], unique=True, null=True))
# Adding unique constraint on 'BuildRequest', fields ['build']
db.create_unique(u'bldcontrol_buildrequest', ['build_id'])
def backwards(self, orm):
# Removing unique constraint on 'BuildRequest', fields ['build']
db.delete_unique(u'bldcontrol_buildrequest', ['build_id'])
# Deleting field 'BuildRequest.environment'
db.delete_column(u'bldcontrol_buildrequest', 'environment_id')
# Changing field 'BuildRequest.build'
db.alter_column(u'bldcontrol_buildrequest', 'build_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['orm.Build'], null=True))
models = {
u'bldcontrol.brbitbake': {
'Meta': {'object_name': 'BRBitbake'},
'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']", 'unique': 'True'})
},
u'bldcontrol.brerror': {
'Meta': {'object_name': 'BRError'},
'errmsg': ('django.db.models.fields.TextField', [], {}),
'errtype': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
'traceback': ('django.db.models.fields.TextField', [], {})
},
u'bldcontrol.brlayer': {
'Meta': {'object_name': 'BRLayer'},
'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"})
},
u'bldcontrol.brtarget': {
'Meta': {'object_name': 'BRTarget'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'})
},
u'bldcontrol.brvariable': {
'Meta': {'object_name': 'BRVariable'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
u'bldcontrol.buildenvironment': {
'Meta': {'object_name': 'BuildEnvironment'},
'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}),
'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}),
'betype': ('django.db.models.fields.IntegerField', [], {}),
'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
},
u'bldcontrol.buildrequest': {
'Meta': {'object_name': 'BuildRequest'},
'build': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['orm.Build']", 'unique': 'True', 'null': 'True'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'environment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildEnvironment']", 'null': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}),
'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
},
u'orm.bitbakeversion': {
'Meta': {'object_name': 'BitbakeVersion'},
'branch': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'giturl': ('django.db.models.fields.URLField', [], {'max_length': '200'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'})
},
u'orm.build': {
'Meta': {'object_name': 'Build'},
'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'completed_on': ('django.db.models.fields.DateTimeField', [], {}),
'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}),
'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}),
'started_on': ('django.db.models.fields.DateTimeField', [], {}),
'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
u'orm.project': {
'Meta': {'object_name': 'Project'},
'bitbake_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.BitbakeVersion']"}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'release': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Release']"}),
'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'})
},
u'orm.release': {
'Meta': {'object_name': 'Release'},
'bitbake_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.BitbakeVersion']"}),
'branch': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'})
}
}
complete_apps = ['bldcontrol']

3
bitbake/lib/toaster/bldcontrol/models.py
View File

@ -59,7 +59,8 @@ class BuildRequest(models.Model):
)
project = models.ForeignKey(Project)
build = models.ForeignKey(Build, null = True) # TODO: toasterui should set this when Build is created
build = models.OneToOneField(Build, null = True) # TODO: toasterui should set this when Build is created
environment = models.ForeignKey(BuildEnvironment, null = True)
state = models.IntegerField(choices = REQUEST_STATE, default = REQ_CREATED)
created = models.DateTimeField(auto_now_add = True)
updated = models.DateTimeField(auto_now = True)

4
bitbake/lib/toaster/bldcontrol/sshbecontroller.py
View File

@ -77,10 +77,10 @@ class SSHBEController(BuildEnvironmentController):
self._pathcreate(self.be.builddir)
self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
def startBBServer(self):
def startBBServer(self, brbe):
assert self.pokydirname and self._pathexists(self.pokydirname)
assert self.islayerset
print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe))
# FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
# but since they start async without any return, we just wait a bit
print "Started server"

10
bitbake/lib/toaster/bldcontrol/tests.py
View File

@ -44,7 +44,7 @@ class BEControllerTests(object):
# test start server and stop
self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Port already occupied")
bc.startBBServer()
bc.startBBServer("0:0")
self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not answering")
bc.stopBBServer()
@ -57,14 +57,8 @@ class BEControllerTests(object):
bc = self._getBEController(obe)
bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info
bbc = bc.getBBController()
bbc = bc.getBBController("%d:%d" % (-1, obe.pk))
self.assertTrue(isinstance(bbc, BitbakeController))
# test set variable, use no build marker -1 for BR value
try:
bbc.setVariable("TOASTER_BRBE", "%d:%d" % (-1, obe.pk))
except Exception as e :
self.fail("setVariable raised %s", e)
bc.stopBBServer()
self._serverForceStop(bc)