2014-09-04 14:27:32 +00:00
#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# BitBake Toaster Implementation
#
# Copyright (C) 2014 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import sys
import re
2015-09-29 04:45:13 +00:00
import shutil
2014-09-04 14:27:32 +00:00
from django . db import transaction
from django . db . models import Q
from bldcontrol . models import BuildEnvironment , BRLayer , BRVariable , BRTarget , BRBitbake
2016-11-24 11:19:58 +00:00
from orm . models import CustomImageRecipe , Layer , Layer_Version , ProjectLayer , ToasterSetting
2014-09-04 14:27:32 +00:00
import subprocess
from toastermain import settings
2016-05-10 12:17:26 +00:00
from bldcontrol . bbcontroller import BuildEnvironmentController , ShellCmdException , BuildSetupException , BitbakeController
2014-09-04 14:27:32 +00:00
2014-01-20 09:39:34 +00:00
import logging
logger = logging . getLogger ( " toaster " )
2015-01-28 13:18:09 +00:00
from pprint import pprint , pformat
2014-01-20 09:39:34 +00:00
2014-09-04 14:27:32 +00:00
class LocalhostBEController ( BuildEnvironmentController ) :
""" Implementation of the BuildEnvironmentController for the localhost;
this controller manages the default build directory ,
the server setup and system start and stop for the localhost - type build environment
"""
def __init__ ( self , be ) :
super ( LocalhostBEController , self ) . __init__ ( be )
self . pokydirname = None
self . islayerset = False
2016-04-06 16:46:23 +00:00
def _shellcmd ( self , command , cwd = None , nowait = False ) :
2014-09-04 14:27:32 +00:00
if cwd is None :
cwd = self . be . sourcedir
2015-07-28 14:24:44 +00:00
logger . debug ( " lbc_shellcmmd: ( %s ) %s " % ( cwd , command ) )
2014-09-04 14:27:32 +00:00
p = subprocess . Popen ( command , cwd = cwd , shell = True , stdout = subprocess . PIPE , stderr = subprocess . PIPE )
2016-04-06 16:46:23 +00:00
if nowait :
return
2014-09-04 14:27:32 +00:00
( out , err ) = p . communicate ( )
2015-01-28 13:18:09 +00:00
p . wait ( )
2014-09-04 14:27:32 +00:00
if p . returncode :
if len ( err ) == 0 :
err = " command: %s \n %s " % ( command , out )
else :
err = " command: %s \n %s " % ( command , err )
2016-05-09 13:01:12 +00:00
logger . warning ( " localhostbecontroller: shellcmd error %s " % err )
2014-09-04 14:27:32 +00:00
raise ShellCmdException ( err )
else :
2015-07-28 14:24:44 +00:00
logger . debug ( " localhostbecontroller: shellcmd success " )
2016-05-10 12:28:50 +00:00
return out . decode ( ' utf-8 ' )
2014-09-04 14:27:32 +00:00
2015-02-02 17:57:36 +00:00
def getGitCloneDirectory ( self , url , branch ) :
2015-09-23 22:34:57 +00:00
""" Construct unique clone directory name out of url and branch. """
2015-02-02 17:57:36 +00:00
if branch != " HEAD " :
2016-10-07 06:23:20 +00:00
return " _toaster_clones/_ %s _ %s " % ( re . sub ( ' [:/@+ % ] ' , ' _ ' , url ) , branch )
2015-02-02 17:57:36 +00:00
# word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases
# which _ALWAYS_ means the current poky checkout
from os . path import dirname as DN
local_checkout_path = DN ( DN ( DN ( DN ( DN ( os . path . abspath ( __file__ ) ) ) ) ) )
#logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
return local_checkout_path
2015-12-10 03:56:38 +00:00
def setLayers ( self , bitbake , layers , targets ) :
2014-09-04 14:27:32 +00:00
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self . be . sourcedir is not None
2016-07-20 08:58:57 +00:00
layerlist = [ ]
nongitlayerlist = [ ]
2014-09-04 14:27:32 +00:00
# set layers in the layersource
2015-01-28 13:18:09 +00:00
# 1. get a list of repos with branches, and map dirpaths for each layer
2014-09-04 14:27:32 +00:00
gitrepos = { }
2015-01-28 13:18:09 +00:00
2016-08-22 15:42:32 +00:00
# if we're using a remotely fetched version of bitbake add its git
# details to the list of repos to clone
if bitbake . giturl and bitbake . commit :
gitrepos [ ( bitbake . giturl , bitbake . commit ) ] = [ ]
gitrepos [ ( bitbake . giturl , bitbake . commit ) ] . append (
( " bitbake " , bitbake . dirpath ) )
2014-11-03 13:47:16 +00:00
2014-09-04 14:27:32 +00:00
for layer in layers :
2016-03-23 08:28:38 +00:00
# We don't need to git clone the layer for the CustomImageRecipe
# as it's generated by us layer on if needed
if CustomImageRecipe . LAYER_NAME in layer . name :
2014-09-04 14:27:32 +00:00
continue
2016-07-20 08:58:57 +00:00
# If we have local layers then we don't need clone them
# For local layers giturl will be empty
if not layer . giturl :
nongitlayerlist . append ( layer . layer_version . layer . local_source_dir )
continue
2015-01-28 13:18:09 +00:00
if not ( layer . giturl , layer . commit ) in gitrepos :
gitrepos [ ( layer . giturl , layer . commit ) ] = [ ]
gitrepos [ ( layer . giturl , layer . commit ) ] . append ( ( layer . name , layer . dirpath ) )
logger . debug ( " localhostbecontroller, our git repos are %s " % pformat ( gitrepos ) )
2015-10-09 10:28:58 +00:00
# 2. Note for future use if the current source directory is a
# checked-out git repos that could match a layer's vcs_url and therefore
# be used to speed up cloning (rather than fetching it again).
2015-01-28 13:18:09 +00:00
cached_layers = { }
2015-10-09 10:28:58 +00:00
try :
for remotes in self . _shellcmd ( " git remote -v " , self . be . sourcedir ) . split ( " \n " ) :
2015-01-28 13:18:09 +00:00
try :
2015-10-09 10:28:58 +00:00
remote = remotes . split ( " \t " ) [ 1 ] . split ( " " ) [ 0 ]
if remote not in cached_layers :
cached_layers [ remote ] = self . be . sourcedir
except IndexError :
2015-01-28 13:18:09 +00:00
pass
2015-10-09 10:28:58 +00:00
except ShellCmdException :
# ignore any errors in collecting git remotes this is an optional
# step
pass
logger . info ( " Using pre-checked out source for layer %s " , cached_layers )
2015-01-28 13:18:09 +00:00
# 3. checkout the repositories
for giturl , commit in gitrepos . keys ( ) :
2015-03-12 11:30:43 +00:00
localdirname = os . path . join ( self . be . sourcedir , self . getGitCloneDirectory ( giturl , commit ) )
2015-01-28 13:18:09 +00:00
logger . debug ( " localhostbecontroller: giturl %s : %s checking out in current directory %s " % ( giturl , commit , localdirname ) )
2014-09-04 14:27:32 +00:00
2016-08-22 15:42:36 +00:00
# see if our directory is a git repository
2014-09-04 14:27:32 +00:00
if os . path . exists ( localdirname ) :
2016-08-22 15:42:36 +00:00
try :
localremotes = self . _shellcmd ( " git remote -v " ,
localdirname )
2016-09-16 16:22:28 +00:00
if not giturl in localremotes and commit != ' HEAD ' :
2016-08-22 15:42:36 +00:00
raise BuildSetupException ( " Existing git repository at %s , but with different remotes ( ' %s ' , expected ' %s ' ). Toaster will not continue out of fear of damaging something. " % ( localdirname , " , " . join ( localremotes . split ( " \n " ) ) , giturl ) )
except ShellCmdException :
# our localdirname might not be a git repository
#- that's fine
pass
2014-09-04 14:27:32 +00:00
else :
2015-01-28 13:18:09 +00:00
if giturl in cached_layers :
logger . debug ( " localhostbecontroller git-copying %s to %s " % ( cached_layers [ giturl ] , localdirname ) )
self . _shellcmd ( " git clone \" %s \" \" %s \" " % ( cached_layers [ giturl ] , localdirname ) )
self . _shellcmd ( " git remote remove origin " , localdirname )
self . _shellcmd ( " git remote add origin \" %s \" " % giturl , localdirname )
else :
2015-09-23 22:34:53 +00:00
logger . debug ( " localhostbecontroller: cloning %s in %s " % ( giturl , localdirname ) )
self . _shellcmd ( ' git clone " %s " " %s " ' % ( giturl , localdirname ) )
2014-09-04 14:27:32 +00:00
# branch magic name "HEAD" will inhibit checkout
if commit != " HEAD " :
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: checking out commit %s to %s " % ( commit , localdirname ) )
2015-09-23 22:34:55 +00:00
ref = commit if re . match ( ' ^[a-fA-F0-9]+$ ' , commit ) else ' origin/ %s ' % commit
self . _shellcmd ( ' git fetch --all && git reset --hard " %s " ' % ref , localdirname )
2014-09-04 14:27:32 +00:00
# take the localdirname as poky dir if we can find the oe-init-build-env
if self . pokydirname is None and os . path . exists ( os . path . join ( localdirname , " oe-init-build-env " ) ) :
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: selected poky dir name %s " % localdirname )
2014-09-04 14:27:32 +00:00
self . pokydirname = localdirname
2014-11-03 13:47:16 +00:00
# make sure we have a working bitbake
if not os . path . exists ( os . path . join ( self . pokydirname , ' bitbake ' ) ) :
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: checking bitbake into the poky dirname %s " % self . pokydirname )
2015-12-10 03:56:38 +00:00
self . _shellcmd ( " git clone -b \" %s \" \" %s \" \" %s \" " % ( bitbake . commit , bitbake . giturl , os . path . join ( self . pokydirname , ' bitbake ' ) ) )
2014-11-03 13:47:16 +00:00
2014-09-04 14:27:32 +00:00
# verify our repositories
2015-01-28 13:18:09 +00:00
for name , dirpath in gitrepos [ ( giturl , commit ) ] :
2014-09-04 14:27:32 +00:00
localdirpath = os . path . join ( localdirname , dirpath )
2015-01-28 13:18:09 +00:00
logger . debug ( " localhostbecontroller: localdirpath expected ' %s ' " % localdirpath )
2014-09-04 14:27:32 +00:00
if not os . path . exists ( localdirpath ) :
raise BuildSetupException ( " Cannot find layer git path ' %s ' in checked out repository ' %s : %s ' . Aborting. " % ( localdirpath , giturl , commit ) )
if name != " bitbake " :
2014-11-03 13:47:16 +00:00
layerlist . append ( localdirpath . rstrip ( " / " ) )
2014-09-04 14:27:32 +00:00
2015-01-28 13:18:09 +00:00
logger . debug ( " localhostbecontroller: current layer list %s " % pformat ( layerlist ) )
2014-09-04 14:27:32 +00:00
2015-12-02 18:02:46 +00:00
# 5. create custom layer and add custom recipes to it
2015-09-29 04:45:13 +00:00
for target in targets :
try :
2016-11-24 11:20:03 +00:00
customrecipe = CustomImageRecipe . objects . get (
name = target . target ,
project = bitbake . req . project )
custom_layer_path = self . setup_custom_image_recipe (
customrecipe , layers )
if os . path . isdir ( custom_layer_path ) :
layerlist . append ( custom_layer_path )
2015-09-29 04:45:13 +00:00
except CustomImageRecipe . DoesNotExist :
2016-11-24 11:20:03 +00:00
continue # not a custom recipe, skip
2015-09-29 04:45:13 +00:00
2016-07-20 08:58:57 +00:00
layerlist . extend ( nongitlayerlist )
2016-11-24 11:20:03 +00:00
logger . debug ( " \n \n set layers gives this list \n %s " % ' ' . join ( layerlist ) )
self . islayerset = True
2016-04-06 16:46:21 +00:00
return layerlist
2014-09-04 14:27:32 +00:00
2016-11-24 11:20:03 +00:00
def setup_custom_image_recipe ( self , customrecipe , layers ) :
""" Set up toaster-custom-images layer and recipe files """
layerpath = os . path . join ( self . be . builddir ,
CustomImageRecipe . LAYER_NAME )
# create directory structure
for name in ( " conf " , " recipes " ) :
path = os . path . join ( layerpath , name )
if not os . path . isdir ( path ) :
os . makedirs ( path )
# create layer.conf
config = os . path . join ( layerpath , " conf " , " layer.conf " )
if not os . path . isfile ( config ) :
with open ( config , " w " ) as conf :
conf . write ( ' BBPATH .= " :$ {LAYERDIR} " \n BBFILES += " $ {LAYERDIR} /recipes/*.bb " \n ' )
# Update the Layer_Version dirpath that has our base_recipe in
# to be able to read the base recipe to then generate the
# custom recipe.
br_layer_base_recipe = layers . get (
layer_version = customrecipe . base_recipe . layer_version )
# If the layer is one that we've cloned we know where it lives
if br_layer_base_recipe . giturl and br_layer_base_recipe . commit :
layer_path = self . getGitCloneDirectory (
br_layer_base_recipe . giturl ,
br_layer_base_recipe . commit )
# Otherwise it's a local layer
elif br_layer_base_recipe . local_source_dir :
layer_path = br_layer_base_recipe . local_source_dir
else :
logger . error ( " Unable to workout the dir path for the custom "
" image recipe " )
br_layer_base_dirpath = os . path . join (
self . be . sourcedir ,
layer_path ,
customrecipe . base_recipe . layer_version . dirpath )
customrecipe . base_recipe . layer_version . dirpath = br_layer_base_dirpath
customrecipe . base_recipe . layer_version . save ( )
# create recipe
recipe_path = os . path . join ( layerpath , " recipes " , " %s .bb " %
customrecipe . name )
with open ( recipe_path , " w " ) as recipef :
recipef . write ( customrecipe . generate_recipe_file_contents ( ) )
# Update the layer and recipe objects
customrecipe . layer_version . dirpath = layerpath
customrecipe . layer_version . layer . local_source_dir = layerpath
customrecipe . layer_version . layer . save ( )
customrecipe . layer_version . save ( )
customrecipe . file_path = recipe_path
customrecipe . save ( )
return layerpath
2015-02-10 16:25:17 +00:00
def readServerLogFile ( self ) :
return open ( os . path . join ( self . be . builddir , " toaster_server.log " ) , " r " ) . read ( )
2015-06-02 11:57:03 +00:00
2016-04-06 16:46:20 +00:00
def triggerBuild ( self , bitbake , layers , variables , targets , brbe ) :
2016-04-06 16:46:22 +00:00
layers = self . setLayers ( bitbake , layers , targets )
# init build environment from the clone
builddir = ' %s -toaster- %d ' % ( self . be . builddir , bitbake . req . project . id )
oe_init = os . path . join ( self . pokydirname , ' oe-init-build-env ' )
# init build environment
2016-11-24 11:19:58 +00:00
try :
custom_script = ToasterSetting . objects . get ( name = " CUSTOM_BUILD_INIT_SCRIPT " ) . value
custom_script = custom_script . replace ( " % BUILDDIR % " , builddir )
self . _shellcmd ( " bash -c ' source %s ' " % ( custom_script ) )
except ToasterSetting . DoesNotExist :
self . _shellcmd ( " bash -c ' source %s %s ' " % ( oe_init , builddir ) ,
2016-04-06 16:46:27 +00:00
self . be . sourcedir )
2016-04-06 16:46:22 +00:00
# update bblayers.conf
2016-11-24 11:19:59 +00:00
bblconfpath = os . path . join ( builddir , " conf/toaster-bblayers.conf " )
2016-04-06 16:46:22 +00:00
with open ( bblconfpath , ' w ' ) as bblayers :
bblayers . write ( ' # line added by toaster build control \n '
' BBLAYERS = " %s " ' % ' ' . join ( layers ) )
2015-06-02 11:57:03 +00:00
2016-01-19 16:13:28 +00:00
# write configuration file
2016-04-06 16:46:22 +00:00
confpath = os . path . join ( builddir , ' conf/toaster.conf ' )
with open ( confpath , ' w ' ) as conf :
2016-01-19 16:13:28 +00:00
for var in variables :
conf . write ( ' %s = " %s " \n ' % ( var . name , var . value ) )
conf . write ( ' INHERIT+= " toaster buildhistory " ' )
2016-04-06 16:46:22 +00:00
# run bitbake server from the clone
bitbake = os . path . join ( self . pokydirname , ' bitbake ' , ' bin ' , ' bitbake ' )
2016-11-24 11:19:59 +00:00
toasterlayers = os . path . join ( builddir , " conf/toaster-bblayers.conf " )
self . _shellcmd ( ' bash -c \" source %s %s ; BITBAKE_UI= " knotty " %s --read %s --read %s '
2016-04-06 16:46:27 +00:00
' --server-only -t xmlrpc -B 0.0.0.0:0 \" ' % ( oe_init ,
2016-11-24 11:19:59 +00:00
builddir , bitbake , confpath , toasterlayers ) , self . be . sourcedir )
2016-04-06 16:46:22 +00:00
# read port number from bitbake.lock
self . be . bbport = " "
bblock = os . path . join ( builddir , ' bitbake.lock ' )
with open ( bblock ) as fplock :
for line in fplock :
if " : " in line :
self . be . bbport = line . split ( " : " ) [ - 1 ] . strip ( )
logger . debug ( " localhostbecontroller: bitbake port %s " , self . be . bbport )
break
if not self . be . bbport :
raise BuildSetupException ( " localhostbecontroller: can ' t read bitbake port from %s " % bblock )
self . be . bbaddress = " localhost "
self . be . bbstate = BuildEnvironment . SERVER_STARTED
self . be . lock = BuildEnvironment . LOCK_RUNNING
self . be . save ( )
bbtargets = ' '
for target in targets :
task = target . task
if task :
if not task . startswith ( ' do_ ' ) :
task = ' do_ ' + task
task = ' : %s ' % task
bbtargets + = ' %s %s ' % ( target . target , task )
2016-04-06 16:46:24 +00:00
# run build with local bitbake. stop the server after the build.
2016-04-06 16:46:22 +00:00
log = os . path . join ( builddir , ' toaster_ui.log ' )
2016-04-06 16:46:46 +00:00
local_bitbake = os . path . join ( os . path . dirname ( os . getenv ( ' BBBASEDIR ' ) ) ,
' bitbake ' )
2016-04-06 16:46:27 +00:00
self . _shellcmd ( [ ' bash -c \" (TOASTER_BRBE= " %s " BBSERVER= " 0.0.0.0:-1 " '
2016-04-06 16:46:46 +00:00
' %s %s -u toasterui --token= " " >> %s 2>&1; '
2016-05-12 13:32:53 +00:00
' BITBAKE_UI= " knotty " BBSERVER=0.0.0.0:-1 %s -m)& \" ' \
2016-04-06 16:46:46 +00:00
% ( brbe , local_bitbake , bbtargets , log , bitbake ) ] ,
builddir , nowait = True )
2016-04-06 16:46:22 +00:00
logger . debug ( ' localhostbecontroller: Build launched, exiting. '
' Follow build logs at %s ' % log )