2014-09-04 14:27:32 +00:00
#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# BitBake Toaster Implementation
#
# Copyright (C) 2014 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import sys
import re
2015-09-29 04:45:13 +00:00
import shutil
2014-09-04 14:27:32 +00:00
from django . db import transaction
from django . db . models import Q
from bldcontrol . models import BuildEnvironment , BRLayer , BRVariable , BRTarget , BRBitbake
2015-09-29 04:45:13 +00:00
from orm . models import CustomImageRecipe , Layer , Layer_Version , ProjectLayer
2014-09-04 14:27:32 +00:00
import subprocess
from toastermain import settings
2015-02-02 17:57:36 +00:00
from bbcontroller import BuildEnvironmentController , ShellCmdException , BuildSetupException
2014-09-04 14:27:32 +00:00
2014-01-20 09:39:34 +00:00
import logging
logger = logging . getLogger ( " toaster " )
2015-01-28 13:18:09 +00:00
from pprint import pprint , pformat
2014-01-20 09:39:34 +00:00
2014-09-04 14:27:32 +00:00
class LocalhostBEController ( BuildEnvironmentController ) :
""" Implementation of the BuildEnvironmentController for the localhost;
this controller manages the default build directory ,
the server setup and system start and stop for the localhost - type build environment
"""
def __init__ ( self , be ) :
super ( LocalhostBEController , self ) . __init__ ( be )
self . pokydirname = None
self . islayerset = False
def _shellcmd ( self , command , cwd = None ) :
if cwd is None :
cwd = self . be . sourcedir
2015-07-28 14:24:44 +00:00
logger . debug ( " lbc_shellcmmd: ( %s ) %s " % ( cwd , command ) )
2014-09-04 14:27:32 +00:00
p = subprocess . Popen ( command , cwd = cwd , shell = True , stdout = subprocess . PIPE , stderr = subprocess . PIPE )
( out , err ) = p . communicate ( )
2015-01-28 13:18:09 +00:00
p . wait ( )
2014-09-04 14:27:32 +00:00
if p . returncode :
if len ( err ) == 0 :
err = " command: %s \n %s " % ( command , out )
else :
err = " command: %s \n %s " % ( command , err )
2015-07-28 14:24:44 +00:00
logger . warn ( " localhostbecontroller: shellcmd error %s " % err )
2014-09-04 14:27:32 +00:00
raise ShellCmdException ( err )
else :
2015-07-28 14:24:44 +00:00
logger . debug ( " localhostbecontroller: shellcmd success " )
2014-09-04 14:27:32 +00:00
return out
2015-01-28 13:18:09 +00:00
def startBBServer ( self ) :
2014-09-04 14:27:32 +00:00
assert self . pokydirname and os . path . exists ( self . pokydirname )
assert self . islayerset
2014-10-13 16:10:39 +00:00
2015-01-28 13:18:09 +00:00
# find our own toasterui listener/bitbake
from toaster . bldcontrol . management . commands . loadconf import _reduce_canon_path
2015-12-02 18:02:47 +00:00
toaster = _reduce_canon_path ( os . path . join ( os . path . dirname ( os . path . abspath ( __file__ ) ) , " ../../../bin/toaster " ) )
assert os . path . exists ( toaster ) and os . path . isfile ( toaster )
# restart bitbake server and toastergui observer
self . _shellcmd ( " bash -c ' source %s restart-bitbake ' " % toaster , self . be . builddir )
logger . debug ( " localhostbecontroller: restarted bitbake server " )
# read port number from bitbake.lock
self . be . bbport = " "
bblock = os . path . join ( self . be . builddir , ' bitbake.lock ' )
if os . path . exists ( bblock ) :
with open ( bblock ) as fplock :
for line in fplock :
2015-01-28 13:18:09 +00:00
if " : " in line :
2015-12-02 18:02:47 +00:00
self . be . bbport = line . split ( " : " ) [ - 1 ] . strip ( )
logger . debug ( " localhostbecontroller: bitbake port %s " , self . be . bbport )
2015-01-28 13:18:09 +00:00
break
2015-12-02 18:02:47 +00:00
if not self . be . bbport :
raise BuildSetupException ( " localhostbecontroller: can ' t read bitbake port from %s " % bblock )
2014-09-04 14:27:32 +00:00
self . be . bbaddress = " localhost "
self . be . bbstate = BuildEnvironment . SERVER_STARTED
self . be . save ( )
2015-02-02 17:57:36 +00:00
def getGitCloneDirectory ( self , url , branch ) :
2015-09-23 22:34:57 +00:00
""" Construct unique clone directory name out of url and branch. """
2015-02-02 17:57:36 +00:00
if branch != " HEAD " :
2015-10-07 14:31:20 +00:00
return " _toaster_clones/_ %s _ %s " % ( re . sub ( ' [:/@ % ] ' , ' _ ' , url ) , branch )
2015-02-02 17:57:36 +00:00
# word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases
# which _ALWAYS_ means the current poky checkout
from os . path import dirname as DN
local_checkout_path = DN ( DN ( DN ( DN ( DN ( os . path . abspath ( __file__ ) ) ) ) ) )
#logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
return local_checkout_path
2015-12-10 03:56:38 +00:00
def setLayers ( self , bitbake , layers , targets ) :
2014-09-04 14:27:32 +00:00
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self . be . sourcedir is not None
# set layers in the layersource
2015-01-28 13:18:09 +00:00
# 1. get a list of repos with branches, and map dirpaths for each layer
2014-09-04 14:27:32 +00:00
gitrepos = { }
2015-01-28 13:18:09 +00:00
2015-12-10 03:56:38 +00:00
gitrepos [ ( bitbake . giturl , bitbake . commit ) ] = [ ]
gitrepos [ ( bitbake . giturl , bitbake . commit ) ] . append ( ( " bitbake " , bitbake . dirpath ) )
2014-11-03 13:47:16 +00:00
2014-09-04 14:27:32 +00:00
for layer in layers :
# we don't process local URLs
if layer . giturl . startswith ( " file:// " ) :
continue
2015-01-28 13:18:09 +00:00
if not ( layer . giturl , layer . commit ) in gitrepos :
gitrepos [ ( layer . giturl , layer . commit ) ] = [ ]
gitrepos [ ( layer . giturl , layer . commit ) ] . append ( ( layer . name , layer . dirpath ) )
logger . debug ( " localhostbecontroller, our git repos are %s " % pformat ( gitrepos ) )
2015-10-09 10:28:58 +00:00
# 2. Note for future use if the current source directory is a
# checked-out git repos that could match a layer's vcs_url and therefore
# be used to speed up cloning (rather than fetching it again).
2015-01-28 13:18:09 +00:00
cached_layers = { }
2015-10-09 10:28:58 +00:00
try :
for remotes in self . _shellcmd ( " git remote -v " , self . be . sourcedir ) . split ( " \n " ) :
2015-01-28 13:18:09 +00:00
try :
2015-10-09 10:28:58 +00:00
remote = remotes . split ( " \t " ) [ 1 ] . split ( " " ) [ 0 ]
if remote not in cached_layers :
cached_layers [ remote ] = self . be . sourcedir
except IndexError :
2015-01-28 13:18:09 +00:00
pass
2015-10-09 10:28:58 +00:00
except ShellCmdException :
# ignore any errors in collecting git remotes this is an optional
# step
pass
logger . info ( " Using pre-checked out source for layer %s " , cached_layers )
2015-01-28 13:18:09 +00:00
2014-09-04 14:27:32 +00:00
layerlist = [ ]
2015-02-02 17:57:36 +00:00
2015-01-28 13:18:09 +00:00
# 3. checkout the repositories
for giturl , commit in gitrepos . keys ( ) :
2015-03-12 11:30:43 +00:00
localdirname = os . path . join ( self . be . sourcedir , self . getGitCloneDirectory ( giturl , commit ) )
2015-01-28 13:18:09 +00:00
logger . debug ( " localhostbecontroller: giturl %s : %s checking out in current directory %s " % ( giturl , commit , localdirname ) )
2014-09-04 14:27:32 +00:00
# make sure our directory is a git repository
if os . path . exists ( localdirname ) :
2015-02-02 17:57:36 +00:00
localremotes = self . _shellcmd ( " git remote -v " , localdirname )
if not giturl in localremotes :
raise BuildSetupException ( " Existing git repository at %s , but with different remotes ( ' %s ' , expected ' %s ' ). Toaster will not continue out of fear of damaging something. " % ( localdirname , " , " . join ( localremotes . split ( " \n " ) ) , giturl ) )
2014-09-04 14:27:32 +00:00
else :
2015-01-28 13:18:09 +00:00
if giturl in cached_layers :
logger . debug ( " localhostbecontroller git-copying %s to %s " % ( cached_layers [ giturl ] , localdirname ) )
self . _shellcmd ( " git clone \" %s \" \" %s \" " % ( cached_layers [ giturl ] , localdirname ) )
self . _shellcmd ( " git remote remove origin " , localdirname )
self . _shellcmd ( " git remote add origin \" %s \" " % giturl , localdirname )
else :
2015-09-23 22:34:53 +00:00
logger . debug ( " localhostbecontroller: cloning %s in %s " % ( giturl , localdirname ) )
self . _shellcmd ( ' git clone " %s " " %s " ' % ( giturl , localdirname ) )
2014-09-04 14:27:32 +00:00
# branch magic name "HEAD" will inhibit checkout
if commit != " HEAD " :
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: checking out commit %s to %s " % ( commit , localdirname ) )
2015-09-23 22:34:55 +00:00
ref = commit if re . match ( ' ^[a-fA-F0-9]+$ ' , commit ) else ' origin/ %s ' % commit
self . _shellcmd ( ' git fetch --all && git reset --hard " %s " ' % ref , localdirname )
2014-09-04 14:27:32 +00:00
# take the localdirname as poky dir if we can find the oe-init-build-env
if self . pokydirname is None and os . path . exists ( os . path . join ( localdirname , " oe-init-build-env " ) ) :
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: selected poky dir name %s " % localdirname )
2014-09-04 14:27:32 +00:00
self . pokydirname = localdirname
2014-11-03 13:47:16 +00:00
# make sure we have a working bitbake
if not os . path . exists ( os . path . join ( self . pokydirname , ' bitbake ' ) ) :
2014-01-20 09:39:34 +00:00
logger . debug ( " localhostbecontroller: checking bitbake into the poky dirname %s " % self . pokydirname )
2015-12-10 03:56:38 +00:00
self . _shellcmd ( " git clone -b \" %s \" \" %s \" \" %s \" " % ( bitbake . commit , bitbake . giturl , os . path . join ( self . pokydirname , ' bitbake ' ) ) )
2014-11-03 13:47:16 +00:00
2014-09-04 14:27:32 +00:00
# verify our repositories
2015-01-28 13:18:09 +00:00
for name , dirpath in gitrepos [ ( giturl , commit ) ] :
2014-09-04 14:27:32 +00:00
localdirpath = os . path . join ( localdirname , dirpath )
2015-01-28 13:18:09 +00:00
logger . debug ( " localhostbecontroller: localdirpath expected ' %s ' " % localdirpath )
2014-09-04 14:27:32 +00:00
if not os . path . exists ( localdirpath ) :
raise BuildSetupException ( " Cannot find layer git path ' %s ' in checked out repository ' %s : %s ' . Aborting. " % ( localdirpath , giturl , commit ) )
if name != " bitbake " :
2014-11-03 13:47:16 +00:00
layerlist . append ( localdirpath . rstrip ( " / " ) )
2014-09-04 14:27:32 +00:00
2015-01-28 13:18:09 +00:00
logger . debug ( " localhostbecontroller: current layer list %s " % pformat ( layerlist ) )
2014-09-04 14:27:32 +00:00
2015-12-02 18:02:46 +00:00
# 4. update the bblayers.conf
2014-09-04 14:27:32 +00:00
bblayerconf = os . path . join ( self . be . builddir , " conf/bblayers.conf " )
if not os . path . exists ( bblayerconf ) :
raise BuildSetupException ( " BE is not consistent: bblayers.conf file missing at %s " % bblayerconf )
2015-12-02 18:02:46 +00:00
# 5. create custom layer and add custom recipes to it
2016-03-23 08:28:36 +00:00
layerpath = os . path . join ( self . be . sourcedir ,
" _meta-toaster-custom_ " ,
bitbake . req . project . name )
2015-09-29 04:45:13 +00:00
for target in targets :
try :
customrecipe = CustomImageRecipe . objects . get ( name = target . target ,
2015-12-10 03:56:38 +00:00
project = bitbake . req . project )
2015-09-29 04:45:13 +00:00
except CustomImageRecipe . DoesNotExist :
continue # not a custom recipe, skip
# create directory structure
for name in ( " conf " , " recipes " ) :
path = os . path . join ( layerpath , name )
if not os . path . isdir ( path ) :
os . makedirs ( path )
# create layer.oonf
config = os . path . join ( layerpath , " conf " , " layer.conf " )
if not os . path . isfile ( config ) :
with open ( config , " w " ) as conf :
conf . write ( ' BBPATH .= " :$ {LAYERDIR} " \n BBFILES += " $ {LAYERDIR} /recipes/*.bb " \n ' )
2015-12-14 18:45:02 +00:00
# Update the Layer_Version dirpath that has our base_recipe in
# to be able to read the base recipe to then generate the
# custom recipe.
br_layer_base_recipe = layers . get (
layer_version = customrecipe . base_recipe . layer_version )
br_layer_base_dirpath = \
os . path . join ( self . be . sourcedir ,
self . getGitCloneDirectory (
br_layer_base_recipe . giturl ,
br_layer_base_recipe . commit ) ,
customrecipe . base_recipe . layer_version . dirpath
)
customrecipe . base_recipe . layer_version . dirpath = \
br_layer_base_dirpath
customrecipe . base_recipe . layer_version . save ( )
2015-09-29 04:45:13 +00:00
# create recipe
2015-11-04 15:02:28 +00:00
recipe_path = \
os . path . join ( layerpath , " recipes " , " %s .bb " % target . target )
with open ( recipe_path , " w " ) as recipef :
recipef . write ( customrecipe . generate_recipe_file_contents ( ) )
# Update the layer and recipe objects
customrecipe . layer_version . dirpath = layerpath
customrecipe . layer_version . save ( )
customrecipe . file_path = recipe_path
customrecipe . save ( )
2015-09-29 04:45:13 +00:00
# create *Layer* objects needed for build machinery to work
2015-11-04 15:02:28 +00:00
BRLayer . objects . get_or_create ( req = target . req ,
name = layer . name ,
dirpath = layerpath ,
2015-09-29 04:45:13 +00:00
giturl = " file:// %s " % layerpath )
if os . path . isdir ( layerpath ) :
layerlist . append ( layerpath )
2014-11-03 13:47:16 +00:00
BuildEnvironmentController . _updateBBLayers ( bblayerconf , layerlist )
2014-09-04 14:27:32 +00:00
self . islayerset = True
return True
2015-02-10 16:25:17 +00:00
def readServerLogFile ( self ) :
return open ( os . path . join ( self . be . builddir , " toaster_server.log " ) , " r " ) . read ( )
2014-09-04 14:27:32 +00:00
def release ( self ) :
assert self . be . sourcedir and os . path . exists ( self . be . builddir )
import shutil
shutil . rmtree ( os . path . join ( self . be . sourcedir , " build " ) )
assert not os . path . exists ( self . be . builddir )
2015-06-02 11:57:03 +00:00
def triggerBuild ( self , bitbake , layers , variables , targets ) :
2015-12-10 03:56:38 +00:00
# set up the build environment with the needed layers
2015-09-29 04:45:13 +00:00
self . setLayers ( bitbake , layers , targets )
2015-06-02 11:57:03 +00:00
2016-01-19 16:13:28 +00:00
# write configuration file
filepath = os . path . join ( self . be . builddir , " conf/toaster.conf " )
with open ( filepath , ' w ' ) as conf :
for var in variables :
conf . write ( ' %s = " %s " \n ' % ( var . name , var . value ) )
conf . write ( ' INHERIT+= " toaster buildhistory " ' )
2015-06-02 11:57:03 +00:00
# get the bb server running with the build req id and build env id
bbctrl = self . getBBController ( )
2016-02-10 18:34:10 +00:00
# set variables; TOASTER_BRBE is not set on the server, as this
# causes events from command-line builds to be attached to the last
# Toaster-triggered build; instead, TOASTER_BRBE is fired as an event so
# that toasterui can set it on the buildinfohelper;
# see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021
2015-12-02 18:02:54 +00:00
for var in variables :
2015-12-09 14:25:48 +00:00
if var . name == ' TOASTER_BRBE ' :
bbctrl . triggerEvent ( ' bb.event.MetadataEvent( " SetBRBE " , " %s " ) ' \
% var . value )
2016-02-10 18:34:10 +00:00
else :
bbctrl . setVariable ( var . name , var . value )
2015-12-02 18:02:54 +00:00
# Add 'toaster' and 'buildhistory' to INHERIT variable
inherit = { item . strip ( ) for item in bbctrl . getVariable ( ' INHERIT ' ) . split ( ) }
inherit = inherit . union ( [ " toaster " , " buildhistory " ] )
bbctrl . setVariable ( ' INHERIT ' , ' ' . join ( inherit ) )
2015-06-02 11:57:03 +00:00
# trigger the build command
task = reduce ( lambda x , y : x if len ( y ) == 0 else y , map ( lambda y : y . task , targets ) )
if len ( task ) == 0 :
task = None
bbctrl . build ( list ( map ( lambda x : x . target , targets ) ) , task )
logger . debug ( " localhostbecontroller: Build launched, exiting. Follow build logs at %s /toaster_ui.log " % self . be . builddir )
# disconnect from the server
bbctrl . disconnect ( )