sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake: toaster: new layer checkout logic 

This patch implements a new layer checkout logic that brings more
flexibility in getting layers under different commits work with
Toaster.

The new hibrid logic will checkout separately each layer and commit id;
the task execution will be delegated to the checkedout bitbake, but the
data logger will be executed from the current toaster version as to
bring in enough data to sustain the updates in UI models.

[YOCTO #7171]
[YOCTO #6892]

(Bitbake rev: c6eb0f7f16c59530c2525b2e5629fe86de4e8f0f)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Alexandru DAMIAN 2015-01-28 13:18:09 +00:00 committed by Richard Purdie
parent c58c94824b
commit afe85485fe
4 changed files with 90 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bitbake/lib/toaster/bldcontrol/bbcontroller.py
View File

@ -81,12 +81,17 @@ def getBuildEnvironmentController(**kwargs):
raise Exception("FIXME: Implement BEC for type %s" % str(be.betype))
def _getgitcheckoutdirectoryname(url):
def _get_git_clonedirectory(url, branch):
""" Utility that returns the last component of a git path as directory
"""
import re
components = re.split(r'[:\.\/]', url)
return components[-2] if components[-1] == "git" else components[-1]
base = components[-2] if components[-1] == "git" else components[-1]
if branch != "HEAD":
return "_%s_%s.toaster_cloned" % (base, branch)
return base
class BuildEnvironmentController(object):
@ -166,12 +171,12 @@ class BuildEnvironmentController(object):
raise Exception("Must override setLayers")
def getBBController(self, brbe):
def getBBController(self):
""" returns a BitbakeController to an already started server; this is the point where the server
starts if needed; or reconnects to the server if we can
"""
if not self.connection:
self.startBBServer(brbe)
self.startBBServer()
self.be.lock = BuildEnvironment.LOCK_RUNNING
self.be.save()

103
bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
View File

@ -30,11 +30,12 @@ import subprocess
from toastermain import settings
from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname
from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _get_git_clonedirectory
import logging
logger = logging.getLogger("toaster")
from pprint import pprint, pformat
class LocalhostBEController(BuildEnvironmentController):
""" Implementation of the BuildEnvironmentController for the localhost;
@ -55,15 +56,16 @@ class LocalhostBEController(BuildEnvironmentController):
p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out,err) = p.communicate()
p.wait()
if p.returncode:
if len(err) == 0:
err = "command: %s \n%s" % (command, out)
else:
err = "command: %s \n%s" % (command, err)
logger.debug("localhostbecontroller: shellcmd error %s" % err)
#logger.debug("localhostbecontroller: shellcmd error %s" % err)
raise ShellCmdException(err)
else:
logger.debug("localhostbecontroller: shellcmd success")
#logger.debug("localhostbecontroller: shellcmd success")
return out
def _createdirpath(self, path):
@ -80,10 +82,19 @@ class LocalhostBEController(BuildEnvironmentController):
self._createdirpath(self.be.builddir)
self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
def startBBServer(self, brbe):
def startBBServer(self):
assert self.pokydirname and os.path.exists(self.pokydirname)
assert self.islayerset
# find our own toasterui listener/bitbake
from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path
own_bitbake = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/bitbake"))
assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake)
logger.debug("localhostbecontroller: running the listener at %s" % own_bitbake)
try:
os.remove(os.path.join(self.be.builddir, "toaster_ui.log"))
except OSError as e:
@ -91,7 +102,10 @@ class LocalhostBEController(BuildEnvironmentController):
if e.errno != errno.ENOENT:
raise
cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)
cmd = "bash -c \"source %s/oe-init-build-env %s && bitbake --read conf/toaster-pre.conf --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 && DATABASE_URL=%s BBSERVER=0.0.0.0:-1 daemon -d -i -D %s -o toaster_ui.log -- %s --observe-only -u toasterui &\"" % (self.pokydirname, self.be.builddir,
self.dburl, self.be.builddir, own_bitbake)
logger.debug("fullcommand |%s| " % cmd)
port = "-1"
for i in self._shellcmd(cmd).split("\n"):
if i.startswith("Bitbake server address"):
@ -113,6 +127,16 @@ class LocalhostBEController(BuildEnvironmentController):
time.sleep(0.5)
logger.debug("localhostbecontroller: Started bitbake server")
while port == "-1":
# the port specification is "autodetect"; read the bitbake.lock file
with open("%s/bitbake.lock" % self.be.builddir, "r") as f:
for line in f.readlines():
if ":" in line:
port = line.split(":")[1].strip()
logger.debug("localhostbecontroller: Autodetected bitbake port %s", port)
break
assert self.be.sourcedir and os.path.exists(self.be.builddir)
self.be.bbaddress = "localhost"
self.be.bbport = port
@ -135,42 +159,62 @@ class LocalhostBEController(BuildEnvironmentController):
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos, and map dirpaths for each layer
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[bitbakes[0].giturl] = []
gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = []
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) )
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not layer.giturl in gitrepos:
gitrepos[layer.giturl] = []
gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
for giturl in gitrepos.keys():
commitid = gitrepos[giturl][0][2]
for e in gitrepos[giturl]:
if commitid != e[2]:
import pprint
raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos))
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
logger.debug("localhostbecontroller, our git repos are %s" % gitrepos)
logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
# 2. find checked-out git repos in the sourcedir directory that may help faster cloning
cached_layers = {}
for ldir in os.listdir(self.be.sourcedir):
fldir = os.path.join(self.be.sourcedir, ldir)
if os.path.isdir(fldir):
try:
for line in self._shellcmd("git remote -v", fldir).split("\n"):
try:
remote = line.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = fldir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes
pass
layerlist = []
# 2. checkout the repositories
for giturl in gitrepos.keys():
localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
logger.debug("localhostbecontroller: giturl %s checking out in current directory %s" % (giturl, localdirname))
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(self.be.sourcedir, _get_git_clonedirectory(giturl, commit))
logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
if not giturl in self._shellcmd("git remote -v", localdirname):
raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
else:
self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
# checkout the needed commit
commit = gitrepos[giturl][0][2]
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin", localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
else:
logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname))
self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
@ -188,21 +232,22 @@ class LocalhostBEController(BuildEnvironmentController):
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath, commit in gitrepos[giturl]:
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " % layerlist)
logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
# 3. configure the build environment, so we have a conf/bblayers.conf
# 4. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 4. update the bblayers.conf
# 5. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)

7
bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
View File

@ -2,7 +2,7 @@ from django.core.management.base import NoArgsCommand, CommandError
from django.db import transaction
from orm.models import Build
from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException
from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable
import os
import logging
@ -47,6 +47,9 @@ class Command(NoArgsCommand):
return
logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be))
# write the build identification variable
BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk))
# let the build request know where it is being executed
br.environment = bec.be
br.save()
@ -56,7 +59,7 @@ class Command(NoArgsCommand):
bec.writePreConfFile(br.brvariable_set.all())
# get the bb server running with the build req id and build env id
bbctrl = bec.getBBController("%d:%d" % (br.pk, bec.be.pk))
bbctrl = bec.getBBController()
# trigger the build command
task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, br.brtarget_set.all()))

58
bitbake/lib/toaster/bldcontrol/sshbecontroller.py
View File

@ -29,7 +29,7 @@ import subprocess
from toastermain import settings
from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname
from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _get_git_clonedirectory
def DN(path):
return "/".join(path.split("/")[0:-1])
@ -124,62 +124,8 @@ class SSHBEController(BuildEnvironmentController):
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos, and map dirpaths for each layer
gitrepos = {}
gitrepos[bitbakes[0].giturl] = []
gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not layer.giturl in gitrepos:
gitrepos[layer.giturl] = []
gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
for giturl in gitrepos.keys():
commitid = gitrepos[giturl][0][2]
for e in gitrepos[giturl]:
if commitid != e[2]:
raise BuildSetupException("More than one commit per git url, unsupported configuration")
layerlist = []
# 2. checkout the repositories
for giturl in gitrepos.keys():
import os
localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname
# make sure our directory is a git repository
if self._pathexists(localdirname):
if not giturl in self._shellcmd("git remote -v", localdirname):
raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
else:
self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
# checkout the needed commit
commit = gitrepos[giturl][0][2]
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
print "DEBUG: checking out commit ", commit, "to", localdirname
self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and self._pathexists(os.path.join(localdirname, "oe-init-build-env")):
print "DEBUG: selected poky dir name", localdirname
self.pokydirname = localdirname
# verify our repositories
for name, dirpath, commit in gitrepos[giturl]:
localdirpath = os.path.join(localdirname, dirpath)
if not self._pathexists(localdirpath):
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath)
print "DEBUG: current layer list ", layerlist
raise Exception("Not implemented: SSH setLayers")
# 3. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()