bitbake: toaster script: webport option and other improvements
We add the "webport=" command line option as to allow starting the web server on a custom port. The bitbake server port is now auto-allocated. This is needed to be able to run multiple toaster environments on a single machine. We tackle bug 6023 (toaster refusing to start when lock file is present) by using more specific checks, and automatically recover from bitbake server down / webserver up error mode. Command line parameters are now read on both interactive and managed modes. The localhost and ssh controllers are updated to work with the modified toaster launcher script. [YOCTO #6023] (Bitbake rev: cd3eb5b051743463cfe51dba97cae4da75420048) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Alexandru DAMIAN
Richard Purdie
parent
baa1082c64
commit
0ca70ce37a
|
|
@ -37,7 +37,7 @@ function webserverKillAll()
|
|||
kill -SIGTERM -$(< ${pidfile}) 2>/dev/null
|
||||
sleep 1;
|
||||
# Kill processes if they are still running - may happen in interactive shells
|
||||
ps fux | grep "python.*manage.py" | awk '{print $2}' | xargs kill
|
||||
ps fux | grep "python.*manage.py runserver" | awk '{print $2}' | xargs kill
|
||||
done;
|
||||
rm ${pidfile}
|
||||
fi
|
||||
|
|
@ -69,13 +69,13 @@ function webserverStartAll()
|
|||
fi
|
||||
if [ $retval -eq 0 ]; then
|
||||
echo "Starting webserver"
|
||||
python $BBBASEDIR/lib/toaster/manage.py runserver 0.0.0.0:8000 </dev/null >${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid
|
||||
python $BBBASEDIR/lib/toaster/manage.py runserver "0.0.0.0:$WEB_PORT" </dev/null >${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid
|
||||
sleep 1
|
||||
if ! cat "${BUILDDIR}/.toastermain.pid" | xargs -I{} kill -0 {} ; then
|
||||
retval=1
|
||||
rm "${BUILDDIR}/.toastermain.pid"
|
||||
else
|
||||
echo "Webserver address: 0.0.0.0:8000"
|
||||
echo "Webserver address: http://0.0.0.0:$WEB_PORT/"
|
||||
fi
|
||||
fi
|
||||
return $retval
|
||||
|
|
@ -103,7 +103,7 @@ function stop_system()
|
|||
kill $(< ${BUILDDIR}/.toasterui.pid ) 2>/dev/null
|
||||
rm ${BUILDDIR}/.toasterui.pid
|
||||
fi
|
||||
BBSERVER=0.0.0.0:8200 bitbake -m
|
||||
BBSERVER=0.0.0.0:-1 bitbake -m
|
||||
unset BBSERVER
|
||||
webserverKillAll
|
||||
# force stop any misbehaving bitbake server
|
||||
|
|
@ -126,9 +126,46 @@ function notify_chldexit() {
|
|||
}
|
||||
|
||||
|
||||
|
||||
# Verify prerequisites
|
||||
|
||||
if ! echo "import django; print (1,) == django.VERSION[0:1] and django.VERSION[1:2][0] in (5,6)" | python 2>/dev/null | grep True >/dev/null; then
|
||||
echo -e "This program needs Django 1.5 or 1.6. Please install with\n\npip install django==1.6"
|
||||
return 2
|
||||
fi
|
||||
|
||||
if ! echo "import south; print [0,8,4] == map(int,south.__version__.split(\".\"))" | python 2>/dev/null | grep True >/dev/null; then
|
||||
echo -e "This program needs South 0.8.4. Please install with\n\npip install south==0.8.4"
|
||||
return 2
|
||||
fi
|
||||
|
||||
# read command line parameters
|
||||
|
||||
BBBASEDIR=`dirname ${BASH_SOURCE}`/..
|
||||
RUNNING=0
|
||||
|
||||
NOTOASTERUI=0
|
||||
WEBSERVER=1
|
||||
TOASTER_BRBE=""
|
||||
WEB_PORT="8000"
|
||||
|
||||
for param in $*; do
|
||||
case $param in
|
||||
noui )
|
||||
NOTOASTERUI=1
|
||||
;;
|
||||
noweb )
|
||||
WEBSERVER=0
|
||||
;;
|
||||
brbe=* )
|
||||
TOASTER_BRBE=$'\n'"TOASTER_BRBE=\""${param#*=}"\""
|
||||
;;
|
||||
webport=*)
|
||||
WEB_PORT="${param#*=}"
|
||||
esac
|
||||
done
|
||||
|
||||
|
||||
if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; then
|
||||
# We are called as standalone. We refuse to run in a build environment - we need the interactive mode for that.
|
||||
# Start just the web server, point the web browser to the interface, and start any Django services.
|
||||
|
|
@ -152,7 +189,7 @@ if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; th
|
|||
echo "Failed to start the web server, stopping" 1>&2;
|
||||
exit 1;
|
||||
fi
|
||||
xdg-open http://0.0.0.0:8000/ >/dev/null 2>&1 &
|
||||
xdg-open http://0.0.0.0:$WEB_PORT/ >/dev/null 2>&1 &
|
||||
trap trap_ctrlc SIGINT
|
||||
echo "Running. Stop with Ctrl-C"
|
||||
while [ $RUNNING -gt 0 ]; do
|
||||
|
|
@ -170,23 +207,6 @@ if [ -z "$BUILDDIR" ] || [ -z `which bitbake` ]; then
|
|||
fi
|
||||
|
||||
|
||||
|
||||
# Verify prerequisites
|
||||
|
||||
if ! echo "import django; print (1,) == django.VERSION[0:1] and django.VERSION[1:2][0] in (5,6)" | python 2>/dev/null | grep True >/dev/null; then
|
||||
echo -e "This program needs Django 1.5 or 1.6. Please install with\n\npip install django==1.6"
|
||||
return 2
|
||||
fi
|
||||
|
||||
if ! echo "import south; print [0,8,4] == map(int,south.__version__.split(\".\"))" | python 2>/dev/null | grep True >/dev/null; then
|
||||
echo -e "This program needs South 0.8.4. Please install with\n\npip install south==0.8.4"
|
||||
return 2
|
||||
fi
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Determine the action. If specified by arguments, fine, if not, toggle it
|
||||
if [ "x$1" == "xstart" ] || [ "x$1" == "xstop" ]; then
|
||||
CMD="$1"
|
||||
|
|
@ -198,22 +218,6 @@ else
|
|||
fi;
|
||||
fi
|
||||
|
||||
NOTOASTERUI=0
|
||||
WEBSERVER=1
|
||||
TOASTER_BRBE=""
|
||||
for param in $*; do
|
||||
case $param in
|
||||
noui )
|
||||
NOTOASTERUI=1
|
||||
;;
|
||||
noweb )
|
||||
WEBSERVER=0
|
||||
;;
|
||||
brbe=* )
|
||||
TOASTER_BRBE=$'\n'"TOASTER_BRBE=\""${param#*=}"\""
|
||||
esac
|
||||
done
|
||||
|
||||
echo "The system will $CMD."
|
||||
|
||||
# Make sure it's safe to run by checking bitbake lock
|
||||
|
|
@ -223,12 +227,18 @@ if [ -e $BUILDDIR/bitbake.lock ]; then
|
|||
(flock -n 200 ) 200<$BUILDDIR/bitbake.lock || lock=0
|
||||
fi
|
||||
|
||||
if [ ${CMD} == "start" ] && ( [ $lock -eq 0 ] || [ -e $BUILDDIR/.toastermain.pid ] ); then
|
||||
echo "Error: bitbake lock state error. File locks show that the system is on." 2>&1
|
||||
echo "If you see problems, stop and then start the system again." 2>&1
|
||||
if [ ${CMD} == "start" ] && [ $lock -eq 0 ]; then
|
||||
echo "Error: bitbake lock state error. File locks show that the system is on." 1>&2
|
||||
echo "Please wait for the current build to finish, stop and then start the system again." 1>&2
|
||||
return 3
|
||||
fi
|
||||
|
||||
if [ ${CMD} == "start" ] && [ -e $BUILDDIR/.toastermain.pid ] && kill -0 `cat $BUILDDIR/.toastermain.pid`; then
|
||||
echo "Error: bitbake appears to be dead, but the webserver is alive. Something fishy is going on." 1>&2
|
||||
echo "Cleaning up the web server at to start a clean slate."
|
||||
webserverKillAll
|
||||
fi
|
||||
|
||||
|
||||
# Execute the commands
|
||||
|
||||
|
|
@ -241,12 +251,12 @@ case $CMD in
|
|||
return 4
|
||||
fi
|
||||
unset BBSERVER
|
||||
bitbake --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:8200
|
||||
bitbake --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0
|
||||
if [ $? -ne 0 ]; then
|
||||
start_success=0
|
||||
echo "Bitbake server start failed"
|
||||
else
|
||||
export BBSERVER=0.0.0.0:8200
|
||||
export BBSERVER=0.0.0.0:-1
|
||||
if [ $NOTOASTERUI == 0 ]; then # we start the TOASTERUI only if not inhibited
|
||||
bitbake --observe-only -u toasterui >${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -117,6 +117,25 @@ class BuildEnvironmentController(object):
|
|||
self.be = be
|
||||
self.connection = None
|
||||
|
||||
@staticmethod
|
||||
def _updateBBLayers(bblayerconf, layerlist):
|
||||
conflines = open(bblayerconf, "r").readlines()
|
||||
|
||||
bblayerconffile = open(bblayerconf, "w")
|
||||
skip = 0
|
||||
for i in xrange(len(conflines)):
|
||||
if skip > 0:
|
||||
skip =- 1
|
||||
continue
|
||||
if conflines[i].startswith("# line added by toaster"):
|
||||
skip = 1
|
||||
else:
|
||||
bblayerconffile.write(conflines[i])
|
||||
|
||||
bblayerconffile.write("# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
|
||||
bblayerconffile.close()
|
||||
|
||||
|
||||
|
||||
def startBBServer(self, brbe):
|
||||
""" Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI.
|
||||
|
|
|
|||
|
|
@ -86,8 +86,12 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
raise
|
||||
|
||||
cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)
|
||||
print("DEBUG: executing ", cmd)
|
||||
print self._shellcmd(cmd)
|
||||
port = "-1"
|
||||
for i in self._shellcmd(cmd).split("\n"):
|
||||
if i.startswith("Bitbake server address"):
|
||||
port = i.split(" ")[-1]
|
||||
print "Found bitbake server port ", port
|
||||
|
||||
def _toaster_ui_started(filepath):
|
||||
if not os.path.exists(filepath):
|
||||
return False
|
||||
|
|
@ -105,7 +109,7 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
print("DEBUG: Started server")
|
||||
assert self.be.sourcedir and os.path.exists(self.be.builddir)
|
||||
self.be.bbaddress = "localhost"
|
||||
self.be.bbport = "8200"
|
||||
self.be.bbport = port
|
||||
self.be.bbstate = BuildEnvironment.SERVER_STARTED
|
||||
self.be.save()
|
||||
|
||||
|
|
@ -129,7 +133,7 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
gitrepos = {}
|
||||
gitrepos[bitbakes[0].giturl] = []
|
||||
gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
|
||||
|
||||
|
||||
for layer in layers:
|
||||
# we don't process local URLs
|
||||
if layer.giturl.startswith("file://"):
|
||||
|
|
@ -141,7 +145,8 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
commitid = gitrepos[giturl][0][2]
|
||||
for e in gitrepos[giturl]:
|
||||
if commitid != e[2]:
|
||||
raise BuildSetupException("More than one commit per git url, unsupported configuration")
|
||||
import pprint
|
||||
raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos))
|
||||
|
||||
|
||||
layerlist = []
|
||||
|
|
@ -170,6 +175,11 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
print "DEBUG: selected poky dir name", localdirname
|
||||
self.pokydirname = localdirname
|
||||
|
||||
# make sure we have a working bitbake
|
||||
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
|
||||
print "DEBUG: checking bitbake into the poky dirname %s " % self.pokydirname
|
||||
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
|
||||
|
||||
# verify our repositories
|
||||
for name, dirpath, commit in gitrepos[giturl]:
|
||||
localdirpath = os.path.join(localdirname, dirpath)
|
||||
|
|
@ -177,7 +187,7 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
|
||||
|
||||
if name != "bitbake":
|
||||
layerlist.append(localdirpath)
|
||||
layerlist.append(localdirpath.rstrip("/"))
|
||||
|
||||
print "DEBUG: current layer list ", layerlist
|
||||
|
||||
|
|
@ -190,21 +200,7 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
if not os.path.exists(bblayerconf):
|
||||
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
|
||||
|
||||
conflines = open(bblayerconf, "r").readlines()
|
||||
|
||||
bblayerconffile = open(bblayerconf, "w")
|
||||
skip = 0
|
||||
for i in xrange(len(conflines)):
|
||||
if skip > 0:
|
||||
skip =- 1
|
||||
continue
|
||||
if conflines[i].startswith("# line added by toaster"):
|
||||
skip = 1
|
||||
else:
|
||||
bblayerconffile.write(conflines[i])
|
||||
|
||||
bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
|
||||
bblayerconffile.close()
|
||||
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
|
||||
|
||||
self.islayerset = True
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -80,13 +80,18 @@ class SSHBEController(BuildEnvironmentController):
|
|||
def startBBServer(self, brbe):
|
||||
assert self.pokydirname and self._pathexists(self.pokydirname)
|
||||
assert self.islayerset
|
||||
print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe))
|
||||
# FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
|
||||
# but since they start async without any return, we just wait a bit
|
||||
print "Started server"
|
||||
cmd = self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe))
|
||||
|
||||
port = "-1"
|
||||
for i in cmd.split("\n"):
|
||||
if i.startswith("Bitbake server address"):
|
||||
port = i.split(" ")[-1]
|
||||
print "Found bitbake server port ", port
|
||||
|
||||
|
||||
assert self.be.sourcedir and self._pathexists(self.be.builddir)
|
||||
self.be.bbaddress = self.be.address.split("@")[-1]
|
||||
self.be.bbport = "8200"
|
||||
self.be.bbport = port
|
||||
self.be.bbstate = BuildEnvironment.SERVER_STARTED
|
||||
self.be.save()
|
||||
|
||||
|
|
@ -99,6 +104,19 @@ class SSHBEController(BuildEnvironmentController):
|
|||
self.be.save()
|
||||
print "Stopped server"
|
||||
|
||||
|
||||
def _copyFile(self, filepath1, filepath2):
|
||||
p = subprocess.Popen("scp '%s' '%s'" % (filepath1, filepath2), stdout=subprocess.PIPE, stderr = subprocess.PIPE, shell=True)
|
||||
(out, err) = p.communicate()
|
||||
if p.returncode:
|
||||
raise ShellCmdException(err)
|
||||
|
||||
def pullFile(self, local_filename, remote_filename):
|
||||
_copyFile(local_filename, "%s:%s" % (self.be.address, remote_filename))
|
||||
|
||||
def pushFile(self, local_filename, remote_filename):
|
||||
_copyFile("%s:%s" % (self.be.address, remote_filename), local_filename)
|
||||
|
||||
def setLayers(self, bitbakes, layers):
|
||||
""" a word of attention: by convention, the first layer for any build will be poky! """
|
||||
|
||||
|
|
@ -110,7 +128,7 @@ class SSHBEController(BuildEnvironmentController):
|
|||
gitrepos = {}
|
||||
gitrepos[bitbakes[0].giturl] = []
|
||||
gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
|
||||
|
||||
|
||||
for layer in layers:
|
||||
# we don't process local URLs
|
||||
if layer.giturl.startswith("file://"):
|
||||
|
|
@ -171,17 +189,15 @@ class SSHBEController(BuildEnvironmentController):
|
|||
if not self._pathexists(bblayerconf):
|
||||
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
|
||||
|
||||
conflines = open(bblayerconf, "r").readlines()
|
||||
import uuid
|
||||
local_bblayerconf = "/tmp/" + uuid.uuid4() + "-bblayer.conf"
|
||||
|
||||
bblayerconffile = open(bblayerconf, "w")
|
||||
for i in xrange(len(conflines)):
|
||||
if conflines[i].startswith("# line added by toaster"):
|
||||
i += 2
|
||||
else:
|
||||
bblayerconffile.write(conflines[i])
|
||||
self.pullFile(bblayerconf, local_bblayerconf)
|
||||
|
||||
bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
|
||||
bblayerconffile.close()
|
||||
BuildEnvironmentController._updateBBLayers(local_bblayerconf, layerlist)
|
||||
self.pushFile(local_bblayerconf, bblayerconf)
|
||||
|
||||
os.unlink(local_bblayerconf)
|
||||
|
||||
self.islayerset = True
|
||||
return True
|
||||
|
|
|
|||
Loading…
Reference in New Issue