2014-09-04 14:27:32 +00:00
#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# BitBake Toaster Implementation
#
# Copyright (C) 2014 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import sys
import re
from django . db import transaction
from django . db . models import Q
from bldcontrol . models import BuildEnvironment , BRLayer , BRVariable , BRTarget , BRBitbake
import subprocess
from toastermain import settings
2015-02-02 17:57:36 +00:00
from bbcontroller import BuildEnvironmentController , ShellCmdException , BuildSetupException
2014-09-04 14:27:32 +00:00
2014-01-20 09:39:34 +00:00
import logging
logger = logging . getLogger ( " toaster " )
2015-01-28 13:18:09 +00:00
from pprint import pprint , pformat
2014-01-20 09:39:34 +00:00
2014-09-04 14:27:32 +00:00
class LocalhostBEController ( BuildEnvironmentController ) :
""" Implementation of the BuildEnvironmentController for the localhost;
this controller manages the default build directory ,
the server setup and system start and stop for the localhost - type build environment
"""
def __init__ ( self , be ) :
super ( LocalhostBEController , self ) . __init__ ( be )
self . dburl = settings . getDATABASE_URL ( )
self . pokydirname = None
self . islayerset = False
def _shellcmd ( self , command , cwd = None ) :
if cwd is None :
cwd = self . be . sourcedir
2015-07-28 14:24:44 +00:00
logger . debug ( " lbc_shellcmmd: ( %s ) %s " % ( cwd , command ) )
2014-09-04 14:27:32 +00:00
p = subprocess . Popen ( command , cwd = cwd , shell = True , stdout = subprocess . PIPE , stderr = subprocess . PIPE )
( out , err ) = p . communicate ( )
2015-01-28 13:18:09 +00:00
p . wait ( )
2014-09-04 14:27:32 +00:00
if p . returncode :
if len ( err ) == 0 :
err = " command: %s \n %s " % ( command , out )
else :
err = " command: %s \n %s " % ( command , err )
2015-07-28 14:24:44 +00:00
logger . warn ( " localhostbecontroller: shellcmd error %s " % err )
2014-09-04 14:27:32 +00:00
raise ShellCmdException ( err )
else :
2015-07-28 14:24:44 +00:00
logger . debug ( " localhostbecontroller: shellcmd success " )
2014-09-04 14:27:32 +00:00
return out
def _setupBE ( self ) :
assert self . pokydirname and os . path . exists ( self . pokydirname )
2015-07-28 14:24:43 +00:00
path = self . be . builddir
if not path :
raise Exception ( " Invalid path creation specified. " )
if not os . path . exists ( path ) :
os . makedirs ( path , 0755 )
self . _shellcmd ( " bash -c \" source %s /oe-init-build-env %s \" " % ( self . pokydirname , path ) )
2015-03-12 11:30:43 +00:00
# delete the templateconf.cfg; it may come from an unsupported layer configuration
2015-07-28 14:24:43 +00:00
os . remove ( os . path . join ( path , " conf/templateconf.cfg " ) )
2014-09-04 14:27:32 +00:00
2015-02-02 12:13:58 +00:00
def writeConfFile ( self , file_name , variable_list = None , raw = None ) :
filepath = os . path . join ( self . be . builddir , file_name )
with open ( filepath , " w " ) as conffile :
if variable_list is not None :
for i in variable_list :
conffile . write ( " %s = \" %s \" \n " % ( i . name , i . value ) )
if raw is not None :
conffile . write ( raw )
2015-01-28 13:18:09 +00:00
def startBBServer ( self ) :
2014-09-04 14:27:32 +00:00
assert self . pokydirname and os . path . exists ( self . pokydirname )
assert self . islayerset
2014-10-13 16:10:39 +00:00
2015-01-28 13:18:09 +00:00
# find our own toasterui listener/bitbake
from toaster . bldcontrol . management . commands . loadconf import _reduce_canon_path
own_bitbake = _reduce_canon_path ( os . path . join ( os . path . dirname ( os . path . abspath ( __file__ ) ) , " ../../../bin/bitbake " ) )
assert os . path . exists ( own_bitbake ) and os . path . isfile ( own_bitbake )
logger . debug ( " localhostbecontroller: running the listener at %s " % own_bitbake )
2015-02-17 12:57:29 +00:00
toaster_ui_log_filepath = os . path . join ( self . be . builddir , " toaster_ui.log " )
# get the file length; we need to detect the _last_ start of the toaster UI, not the first
toaster_ui_log_filelength = 0
if os . path . exists ( toaster_ui_log_filepath ) :
2015-06-17 16:30:34 +00:00
with open ( toaster_ui_log_filepath , " w " ) as f :
2015-02-17 12:57:29 +00:00
f . seek ( 0 , 2 ) # jump to the end
toaster_ui_log_filelength = f . tell ( )
2015-01-28 13:18:09 +00:00
2015-05-20 14:41:24 +00:00
cmd = " bash -c \" source %s /oe-init-build-env %s 2>&1 >toaster_server.log && bitbake --read %s /conf/toaster-pre.conf --postread %s /conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 2>&1 >>toaster_server.log \" " % ( self . pokydirname , self . be . builddir , self . be . builddir , self . be . builddir )
2015-05-18 22:06:12 +00:00
2014-11-03 13:47:16 +00:00
port = " -1 "
2015-03-12 11:30:43 +00:00
logger . debug ( " localhostbecontroller: starting builder \n %s \n " % cmd )
2015-05-18 22:06:12 +00:00
2015-02-02 17:57:36 +00:00
cmdoutput = self . _shellcmd ( cmd )
2015-05-18 22:06:11 +00:00
with open ( self . be . builddir + " /toaster_server.log " , " r " ) as f :
for i in f . readlines ( ) :
if i . startswith ( " Bitbake server address " ) :
port = i . split ( " " ) [ - 1 ]
logger . debug ( " localhostbecontroller: Found bitbake server port %s " % port )
2014-11-03 13:47:16 +00:00
2015-05-18 22:06:12 +00:00
cmd = " bash -c \" source %s /oe-init-build-env-memres -1 %s && DATABASE_URL= %s %s --observe-only -u toasterui --remote-server=0.0.0.0:-1 -t xmlrpc \" " % ( self . pokydirname , self . be . builddir , self . dburl , own_bitbake )
with open ( toaster_ui_log_filepath , " a+ " ) as f :
p = subprocess . Popen ( cmd , cwd = self . be . builddir , shell = True , stdout = f , stderr = f )
2015-02-17 12:57:29 +00:00
def _toaster_ui_started ( filepath , filepos = 0 ) :
2014-10-13 16:10:39 +00:00
if not os . path . exists ( filepath ) :
return False
with open ( filepath , " r " ) as f :
2015-02-17 12:57:29 +00:00
f . seek ( filepos )
2014-10-13 16:10:39 +00:00
for line in f :
2015-07-28 14:24:41 +00:00
if line . startswith ( " NOTE: ToasterUI waiting for events " ) :
2014-10-13 16:10:39 +00:00
return True
return False
2015-02-02 17:57:36 +00:00
retries = 0
started = False
2015-09-09 02:39:44 +00:00
while not started and retries < 50 :
2015-02-17 12:57:29 +00:00
started = _toaster_ui_started ( toaster_ui_log_filepath , toaster_ui_log_filelength )
2014-10-13 16:10:39 +00:00
import time
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: Waiting bitbake server to start " )
2014-10-13 16:10:39 +00:00
time . sleep ( 0.5 )
2015-02-02 17:57:36 +00:00
retries + = 1
if not started :
2015-05-18 22:06:12 +00:00
toaster_ui_log = open ( os . path . join ( self . be . builddir , " toaster_ui.log " ) , " r " ) . read ( )
2015-03-12 11:30:43 +00:00
toaster_server_log = open ( os . path . join ( self . be . builddir , " toaster_server.log " ) , " r " ) . read ( )
2015-09-09 02:39:44 +00:00
raise BuildSetupException ( " localhostbecontroller: Bitbake server did not start in 25 seconds, aborting (Error: ' %s ' ' %s ' ) " % ( toaster_ui_log , toaster_server_log ) )
2014-10-13 16:10:39 +00:00
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: Started bitbake server " )
2015-01-28 13:18:09 +00:00
while port == " -1 " :
# the port specification is "autodetect"; read the bitbake.lock file
with open ( " %s /bitbake.lock " % self . be . builddir , " r " ) as f :
for line in f . readlines ( ) :
if " : " in line :
port = line . split ( " : " ) [ 1 ] . strip ( )
logger . debug ( " localhostbecontroller: Autodetected bitbake port %s " , port )
break
2014-09-04 14:27:32 +00:00
assert self . be . sourcedir and os . path . exists ( self . be . builddir )
self . be . bbaddress = " localhost "
2014-11-03 13:47:16 +00:00
self . be . bbport = port
2014-09-04 14:27:32 +00:00
self . be . bbstate = BuildEnvironment . SERVER_STARTED
self . be . save ( )
def stopBBServer ( self ) :
assert self . pokydirname and os . path . exists ( self . pokydirname )
assert self . islayerset
2014-01-20 09:39:34 +00:00
self . _shellcmd ( " bash -c \" source %s /oe-init-build-env %s && %s source toaster stop \" " %
2014-09-04 14:27:32 +00:00
( self . pokydirname , self . be . builddir , ( lambda : " " if self . be . bbtoken is None else " BBTOKEN= %s " % self . be . bbtoken ) ( ) ) )
self . be . bbstate = BuildEnvironment . SERVER_STOPPED
self . be . save ( )
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: Stopped bitbake server " )
2014-09-04 14:27:32 +00:00
2015-02-02 17:57:36 +00:00
def getGitCloneDirectory ( self , url , branch ) :
""" Utility that returns the last component of a git path as directory
"""
import re
components = re . split ( r ' [: \ . \ /] ' , url )
base = components [ - 2 ] if components [ - 1 ] == " git " else components [ - 1 ]
if branch != " HEAD " :
2015-03-12 11:30:43 +00:00
return " _ %s _ %s .toaster_cloned " % ( base , branch )
2015-02-02 17:57:36 +00:00
# word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases
# which _ALWAYS_ means the current poky checkout
from os . path import dirname as DN
local_checkout_path = DN ( DN ( DN ( DN ( DN ( os . path . abspath ( __file__ ) ) ) ) ) )
#logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
return local_checkout_path
2014-09-04 14:27:32 +00:00
def setLayers ( self , bitbakes , layers ) :
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self . be . sourcedir is not None
assert len ( bitbakes ) == 1
# set layers in the layersource
2015-01-28 13:18:09 +00:00
# 1. get a list of repos with branches, and map dirpaths for each layer
2014-09-04 14:27:32 +00:00
gitrepos = { }
2015-01-28 13:18:09 +00:00
gitrepos [ ( bitbakes [ 0 ] . giturl , bitbakes [ 0 ] . commit ) ] = [ ]
gitrepos [ ( bitbakes [ 0 ] . giturl , bitbakes [ 0 ] . commit ) ] . append ( ( " bitbake " , bitbakes [ 0 ] . dirpath ) )
2014-11-03 13:47:16 +00:00
2014-09-04 14:27:32 +00:00
for layer in layers :
# we don't process local URLs
if layer . giturl . startswith ( " file:// " ) :
continue
2015-01-28 13:18:09 +00:00
if not ( layer . giturl , layer . commit ) in gitrepos :
gitrepos [ ( layer . giturl , layer . commit ) ] = [ ]
gitrepos [ ( layer . giturl , layer . commit ) ] . append ( ( layer . name , layer . dirpath ) )
logger . debug ( " localhostbecontroller, our git repos are %s " % pformat ( gitrepos ) )
# 2. find checked-out git repos in the sourcedir directory that may help faster cloning
cached_layers = { }
for ldir in os . listdir ( self . be . sourcedir ) :
fldir = os . path . join ( self . be . sourcedir , ldir )
if os . path . isdir ( fldir ) :
try :
for line in self . _shellcmd ( " git remote -v " , fldir ) . split ( " \n " ) :
try :
remote = line . split ( " \t " ) [ 1 ] . split ( " " ) [ 0 ]
if remote not in cached_layers :
cached_layers [ remote ] = fldir
except IndexError :
pass
except ShellCmdException :
# ignore any errors in collecting git remotes
pass
2014-09-04 14:27:32 +00:00
layerlist = [ ]
2015-02-02 17:57:36 +00:00
2015-01-28 13:18:09 +00:00
# 3. checkout the repositories
for giturl , commit in gitrepos . keys ( ) :
2015-03-12 11:30:43 +00:00
localdirname = os . path . join ( self . be . sourcedir , self . getGitCloneDirectory ( giturl , commit ) )
2015-01-28 13:18:09 +00:00
logger . debug ( " localhostbecontroller: giturl %s : %s checking out in current directory %s " % ( giturl , commit , localdirname ) )
2014-09-04 14:27:32 +00:00
# make sure our directory is a git repository
if os . path . exists ( localdirname ) :
2015-02-02 17:57:36 +00:00
localremotes = self . _shellcmd ( " git remote -v " , localdirname )
if not giturl in localremotes :
raise BuildSetupException ( " Existing git repository at %s , but with different remotes ( ' %s ' , expected ' %s ' ). Toaster will not continue out of fear of damaging something. " % ( localdirname , " , " . join ( localremotes . split ( " \n " ) ) , giturl ) )
2014-09-04 14:27:32 +00:00
else :
2015-01-28 13:18:09 +00:00
if giturl in cached_layers :
logger . debug ( " localhostbecontroller git-copying %s to %s " % ( cached_layers [ giturl ] , localdirname ) )
self . _shellcmd ( " git clone \" %s \" \" %s \" " % ( cached_layers [ giturl ] , localdirname ) )
self . _shellcmd ( " git remote remove origin " , localdirname )
self . _shellcmd ( " git remote add origin \" %s \" " % giturl , localdirname )
else :
logger . debug ( " localhostbecontroller: cloning %s : %s in %s " % ( giturl , commit , localdirname ) )
self . _shellcmd ( " git clone \" %s \" --single-branch --branch \" %s \" \" %s \" " % ( giturl , commit , localdirname ) )
2014-09-04 14:27:32 +00:00
# branch magic name "HEAD" will inhibit checkout
if commit != " HEAD " :
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: checking out commit %s to %s " % ( commit , localdirname ) )
2015-02-13 10:22:35 +00:00
self . _shellcmd ( " git fetch --all && git checkout \" %s \" && git rebase \" origin/ %s \" " % ( commit , commit ) , localdirname )
2014-09-04 14:27:32 +00:00
# take the localdirname as poky dir if we can find the oe-init-build-env
if self . pokydirname is None and os . path . exists ( os . path . join ( localdirname , " oe-init-build-env " ) ) :
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: selected poky dir name %s " % localdirname )
2014-09-04 14:27:32 +00:00
self . pokydirname = localdirname
2014-11-03 13:47:16 +00:00
# make sure we have a working bitbake
if not os . path . exists ( os . path . join ( self . pokydirname , ' bitbake ' ) ) :
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: checking bitbake into the poky dirname %s " % self . pokydirname )
2014-11-03 13:47:16 +00:00
self . _shellcmd ( " git clone -b \" %s \" \" %s \" \" %s \" " % ( bitbakes [ 0 ] . commit , bitbakes [ 0 ] . giturl , os . path . join ( self . pokydirname , ' bitbake ' ) ) )
2014-09-04 14:27:32 +00:00
# verify our repositories
2015-01-28 13:18:09 +00:00
for name , dirpath in gitrepos [ ( giturl , commit ) ] :
2014-09-04 14:27:32 +00:00
localdirpath = os . path . join ( localdirname , dirpath )
2015-01-28 13:18:09 +00:00
logger . debug ( " localhostbecontroller: localdirpath expected ' %s ' " % localdirpath )
2014-09-04 14:27:32 +00:00
if not os . path . exists ( localdirpath ) :
raise BuildSetupException ( " Cannot find layer git path ' %s ' in checked out repository ' %s : %s ' . Aborting. " % ( localdirpath , giturl , commit ) )
if name != " bitbake " :
2014-11-03 13:47:16 +00:00
layerlist . append ( localdirpath . rstrip ( " / " ) )
2014-09-04 14:27:32 +00:00
2015-01-28 13:18:09 +00:00
logger . debug ( " localhostbecontroller: current layer list %s " % pformat ( layerlist ) )
2014-09-04 14:27:32 +00:00
2015-01-28 13:18:09 +00:00
# 4. configure the build environment, so we have a conf/bblayers.conf
2014-09-04 14:27:32 +00:00
assert self . pokydirname is not None
self . _setupBE ( )
2015-01-28 13:18:09 +00:00
# 5. update the bblayers.conf
2014-09-04 14:27:32 +00:00
bblayerconf = os . path . join ( self . be . builddir , " conf/bblayers.conf " )
if not os . path . exists ( bblayerconf ) :
raise BuildSetupException ( " BE is not consistent: bblayers.conf file missing at %s " % bblayerconf )
2014-11-03 13:47:16 +00:00
BuildEnvironmentController . _updateBBLayers ( bblayerconf , layerlist )
2014-09-04 14:27:32 +00:00
self . islayerset = True
return True
2015-02-10 16:25:17 +00:00
def readServerLogFile ( self ) :
return open ( os . path . join ( self . be . builddir , " toaster_server.log " ) , " r " ) . read ( )
2014-09-04 14:27:32 +00:00
def release ( self ) :
assert self . be . sourcedir and os . path . exists ( self . be . builddir )
import shutil
shutil . rmtree ( os . path . join ( self . be . sourcedir , " build " ) )
assert not os . path . exists ( self . be . builddir )
2015-06-02 11:57:03 +00:00
def triggerBuild ( self , bitbake , layers , variables , targets ) :
# set up the buid environment with the needed layers
self . setLayers ( bitbake , layers )
2015-06-17 11:27:48 +00:00
self . writeConfFile ( " conf/toaster-pre.conf " , variables )
2015-06-02 11:57:03 +00:00
self . writeConfFile ( " conf/toaster.conf " , raw = " INHERIT+= \" toaster buildhistory \" " )
# get the bb server running with the build req id and build env id
bbctrl = self . getBBController ( )
# trigger the build command
task = reduce ( lambda x , y : x if len ( y ) == 0 else y , map ( lambda y : y . task , targets ) )
if len ( task ) == 0 :
task = None
bbctrl . build ( list ( map ( lambda x : x . target , targets ) ) , task )
logger . debug ( " localhostbecontroller: Build launched, exiting. Follow build logs at %s /toaster_ui.log " % self . be . builddir )
# disconnect from the server
bbctrl . disconnect ( )