diff --git a/bitbake/bin/toaster b/bitbake/bin/toaster index 34f65e4721..4c6ac5ac40 100755 --- a/bitbake/bin/toaster +++ b/bitbake/bin/toaster @@ -37,7 +37,7 @@ function webserverKillAll() kill -SIGTERM -$(< ${pidfile}) 2>/dev/null sleep 1; # Kill processes if they are still running - may happen in interactive shells - ps fux | grep "python.*manage.py" | awk '{print $2}' | xargs kill + ps fux | grep "python.*manage.py runserver" | awk '{print $2}' | xargs kill done; rm ${pidfile} fi @@ -69,13 +69,13 @@ function webserverStartAll() fi if [ $retval -eq 0 ]; then echo "Starting webserver" - python $BBBASEDIR/lib/toaster/manage.py runserver 0.0.0.0:8000 ${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid + python $BBBASEDIR/lib/toaster/manage.py runserver "0.0.0.0:$WEB_PORT" ${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid sleep 1 if ! cat "${BUILDDIR}/.toastermain.pid" | xargs -I{} kill -0 {} ; then retval=1 rm "${BUILDDIR}/.toastermain.pid" else - echo "Webserver address: 0.0.0.0:8000" + echo "Webserver address: http://0.0.0.0:$WEB_PORT/" fi fi return $retval @@ -103,7 +103,7 @@ function stop_system() kill $(< ${BUILDDIR}/.toasterui.pid ) 2>/dev/null rm ${BUILDDIR}/.toasterui.pid fi - BBSERVER=0.0.0.0:8200 bitbake -m + BBSERVER=0.0.0.0:-1 bitbake -m unset BBSERVER webserverKillAll # force stop any misbehaving bitbake server @@ -126,9 +126,46 @@ function notify_chldexit() { } + +# Verify prerequisites + +if ! echo "import django; print (1,) == django.VERSION[0:1] and django.VERSION[1:2][0] in (5,6)" | python 2>/dev/null | grep True >/dev/null; then + echo -e "This program needs Django 1.5 or 1.6. Please install with\n\npip install django==1.6" + return 2 +fi + +if ! echo "import south; print [0,8,4] == map(int,south.__version__.split(\".\"))" | python 2>/dev/null | grep True >/dev/null; then + echo -e "This program needs South 0.8.4. Please install with\n\npip install south==0.8.4" + return 2 +fi + +# read command line parameters + BBBASEDIR=`dirname ${BASH_SOURCE}`/.. RUNNING=0 +NOTOASTERUI=0 +WEBSERVER=1 +TOASTER_BRBE="" +WEB_PORT="8000" + +for param in $*; do + case $param in + noui ) + NOTOASTERUI=1 + ;; + noweb ) + WEBSERVER=0 + ;; + brbe=* ) + TOASTER_BRBE=$'\n'"TOASTER_BRBE=\""${param#*=}"\"" + ;; + webport=*) + WEB_PORT="${param#*=}" + esac +done + + if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; then # We are called as standalone. We refuse to run in a build environment - we need the interactive mode for that. # Start just the web server, point the web browser to the interface, and start any Django services. @@ -152,7 +189,7 @@ if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; th echo "Failed to start the web server, stopping" 1>&2; exit 1; fi - xdg-open http://0.0.0.0:8000/ >/dev/null 2>&1 & + xdg-open http://0.0.0.0:$WEB_PORT/ >/dev/null 2>&1 & trap trap_ctrlc SIGINT echo "Running. Stop with Ctrl-C" while [ $RUNNING -gt 0 ]; do @@ -170,23 +207,6 @@ if [ -z "$BUILDDIR" ] || [ -z `which bitbake` ]; then fi - -# Verify prerequisites - -if ! echo "import django; print (1,) == django.VERSION[0:1] and django.VERSION[1:2][0] in (5,6)" | python 2>/dev/null | grep True >/dev/null; then - echo -e "This program needs Django 1.5 or 1.6. Please install with\n\npip install django==1.6" - return 2 -fi - -if ! echo "import south; print [0,8,4] == map(int,south.__version__.split(\".\"))" | python 2>/dev/null | grep True >/dev/null; then - echo -e "This program needs South 0.8.4. Please install with\n\npip install south==0.8.4" - return 2 -fi - - - - - # Determine the action. If specified by arguments, fine, if not, toggle it if [ "x$1" == "xstart" ] || [ "x$1" == "xstop" ]; then CMD="$1" @@ -198,22 +218,6 @@ else fi; fi -NOTOASTERUI=0 -WEBSERVER=1 -TOASTER_BRBE="" -for param in $*; do - case $param in - noui ) - NOTOASTERUI=1 - ;; - noweb ) - WEBSERVER=0 - ;; - brbe=* ) - TOASTER_BRBE=$'\n'"TOASTER_BRBE=\""${param#*=}"\"" - esac -done - echo "The system will $CMD." # Make sure it's safe to run by checking bitbake lock @@ -223,12 +227,18 @@ if [ -e $BUILDDIR/bitbake.lock ]; then (flock -n 200 ) 200<$BUILDDIR/bitbake.lock || lock=0 fi -if [ ${CMD} == "start" ] && ( [ $lock -eq 0 ] || [ -e $BUILDDIR/.toastermain.pid ] ); then - echo "Error: bitbake lock state error. File locks show that the system is on." 2>&1 - echo "If you see problems, stop and then start the system again." 2>&1 +if [ ${CMD} == "start" ] && [ $lock -eq 0 ]; then + echo "Error: bitbake lock state error. File locks show that the system is on." 1>&2 + echo "Please wait for the current build to finish, stop and then start the system again." 1>&2 return 3 fi +if [ ${CMD} == "start" ] && [ -e $BUILDDIR/.toastermain.pid ] && kill -0 `cat $BUILDDIR/.toastermain.pid`; then + echo "Error: bitbake appears to be dead, but the webserver is alive. Something fishy is going on." 1>&2 + echo "Cleaning up the web server at to start a clean slate." + webserverKillAll +fi + # Execute the commands @@ -241,12 +251,12 @@ case $CMD in return 4 fi unset BBSERVER - bitbake --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:8200 + bitbake --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 if [ $? -ne 0 ]; then start_success=0 echo "Bitbake server start failed" else - export BBSERVER=0.0.0.0:8200 + export BBSERVER=0.0.0.0:-1 if [ $NOTOASTERUI == 0 ]; then # we start the TOASTERUI only if not inhibited bitbake --observe-only -u toasterui >${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid fi diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py index 6bb45d6388..7c27fe110e 100644 --- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py +++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -117,6 +117,25 @@ class BuildEnvironmentController(object): self.be = be self.connection = None + @staticmethod + def _updateBBLayers(bblayerconf, layerlist): + conflines = open(bblayerconf, "r").readlines() + + bblayerconffile = open(bblayerconf, "w") + skip = 0 + for i in xrange(len(conflines)): + if skip > 0: + skip =- 1 + continue + if conflines[i].startswith("# line added by toaster"): + skip = 1 + else: + bblayerconffile.write(conflines[i]) + + bblayerconffile.write("# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") + bblayerconffile.close() + + def startBBServer(self, brbe): """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI. diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index a272860ec0..ebf2b4f3bb 100644 --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -86,8 +86,12 @@ class LocalhostBEController(BuildEnvironmentController): raise cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe) - print("DEBUG: executing ", cmd) - print self._shellcmd(cmd) + port = "-1" + for i in self._shellcmd(cmd).split("\n"): + if i.startswith("Bitbake server address"): + port = i.split(" ")[-1] + print "Found bitbake server port ", port + def _toaster_ui_started(filepath): if not os.path.exists(filepath): return False @@ -105,7 +109,7 @@ class LocalhostBEController(BuildEnvironmentController): print("DEBUG: Started server") assert self.be.sourcedir and os.path.exists(self.be.builddir) self.be.bbaddress = "localhost" - self.be.bbport = "8200" + self.be.bbport = port self.be.bbstate = BuildEnvironment.SERVER_STARTED self.be.save() @@ -129,7 +133,7 @@ class LocalhostBEController(BuildEnvironmentController): gitrepos = {} gitrepos[bitbakes[0].giturl] = [] gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) - + for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): @@ -141,7 +145,8 @@ class LocalhostBEController(BuildEnvironmentController): commitid = gitrepos[giturl][0][2] for e in gitrepos[giturl]: if commitid != e[2]: - raise BuildSetupException("More than one commit per git url, unsupported configuration") + import pprint + raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos)) layerlist = [] @@ -170,6 +175,11 @@ class LocalhostBEController(BuildEnvironmentController): print "DEBUG: selected poky dir name", localdirname self.pokydirname = localdirname + # make sure we have a working bitbake + if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): + print "DEBUG: checking bitbake into the poky dirname %s " % self.pokydirname + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) + # verify our repositories for name, dirpath, commit in gitrepos[giturl]: localdirpath = os.path.join(localdirname, dirpath) @@ -177,7 +187,7 @@ class LocalhostBEController(BuildEnvironmentController): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": - layerlist.append(localdirpath) + layerlist.append(localdirpath.rstrip("/")) print "DEBUG: current layer list ", layerlist @@ -190,21 +200,7 @@ class LocalhostBEController(BuildEnvironmentController): if not os.path.exists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) - conflines = open(bblayerconf, "r").readlines() - - bblayerconffile = open(bblayerconf, "w") - skip = 0 - for i in xrange(len(conflines)): - if skip > 0: - skip =- 1 - continue - if conflines[i].startswith("# line added by toaster"): - skip = 1 - else: - bblayerconffile.write(conflines[i]) - - bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") - bblayerconffile.close() + BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) self.islayerset = True return True diff --git a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py index f9fd51e5f8..45e15392e5 100644 --- a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py @@ -80,13 +80,18 @@ class SSHBEController(BuildEnvironmentController): def startBBServer(self, brbe): assert self.pokydirname and self._pathexists(self.pokydirname) assert self.islayerset - print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)) - # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected - # but since they start async without any return, we just wait a bit - print "Started server" + cmd = self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)) + + port = "-1" + for i in cmd.split("\n"): + if i.startswith("Bitbake server address"): + port = i.split(" ")[-1] + print "Found bitbake server port ", port + + assert self.be.sourcedir and self._pathexists(self.be.builddir) self.be.bbaddress = self.be.address.split("@")[-1] - self.be.bbport = "8200" + self.be.bbport = port self.be.bbstate = BuildEnvironment.SERVER_STARTED self.be.save() @@ -99,6 +104,19 @@ class SSHBEController(BuildEnvironmentController): self.be.save() print "Stopped server" + + def _copyFile(self, filepath1, filepath2): + p = subprocess.Popen("scp '%s' '%s'" % (filepath1, filepath2), stdout=subprocess.PIPE, stderr = subprocess.PIPE, shell=True) + (out, err) = p.communicate() + if p.returncode: + raise ShellCmdException(err) + + def pullFile(self, local_filename, remote_filename): + _copyFile(local_filename, "%s:%s" % (self.be.address, remote_filename)) + + def pushFile(self, local_filename, remote_filename): + _copyFile("%s:%s" % (self.be.address, remote_filename), local_filename) + def setLayers(self, bitbakes, layers): """ a word of attention: by convention, the first layer for any build will be poky! """ @@ -110,7 +128,7 @@ class SSHBEController(BuildEnvironmentController): gitrepos = {} gitrepos[bitbakes[0].giturl] = [] gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) - + for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): @@ -171,17 +189,15 @@ class SSHBEController(BuildEnvironmentController): if not self._pathexists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) - conflines = open(bblayerconf, "r").readlines() + import uuid + local_bblayerconf = "/tmp/" + uuid.uuid4() + "-bblayer.conf" - bblayerconffile = open(bblayerconf, "w") - for i in xrange(len(conflines)): - if conflines[i].startswith("# line added by toaster"): - i += 2 - else: - bblayerconffile.write(conflines[i]) + self.pullFile(bblayerconf, local_bblayerconf) - bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") - bblayerconffile.close() + BuildEnvironmentController._updateBBLayers(local_bblayerconf, layerlist) + self.pushFile(local_bblayerconf, bblayerconf) + + os.unlink(local_bblayerconf) self.islayerset = True return True