2014-12-19 11:41:50 +00:00
# Utility functions for reading and modifying recipes
#
# Some code borrowed from the OE layer index
#
2016-07-13 21:04:25 +00:00
# Copyright (C) 2013-2016 Intel Corporation
2014-12-19 11:41:50 +00:00
#
import sys
import os
import os . path
import tempfile
import textwrap
import difflib
2016-05-20 10:57:44 +00:00
from . import utils
2014-12-19 11:41:50 +00:00
import shutil
import re
2015-05-18 15:15:07 +00:00
import fnmatch
2016-07-13 21:04:25 +00:00
import glob
2014-12-19 11:41:50 +00:00
from collections import OrderedDict , defaultdict
# Help us to find places to insert values
2016-05-29 22:20:58 +00:00
recipe_progression = [ ' SUMMARY ' , ' DESCRIPTION ' , ' HOMEPAGE ' , ' BUGTRACKER ' , ' SECTION ' , ' LICENSE ' , ' LICENSE_FLAGS ' , ' LIC_FILES_CHKSUM ' , ' PROVIDES ' , ' DEPENDS ' , ' PR ' , ' PV ' , ' SRCREV ' , ' SRCPV ' , ' SRC_URI ' , ' S ' , ' do_fetch() ' , ' do_unpack() ' , ' do_patch() ' , ' EXTRA_OECONF ' , ' EXTRA_OECMAKE ' , ' EXTRA_OESCONS ' , ' do_configure() ' , ' EXTRA_OEMAKE ' , ' do_compile() ' , ' do_install() ' , ' do_populate_sysroot() ' , ' INITSCRIPT ' , ' USERADD ' , ' GROUPADD ' , ' PACKAGES ' , ' FILES ' , ' RDEPENDS ' , ' RRECOMMENDS ' , ' RSUGGESTS ' , ' RPROVIDES ' , ' RREPLACES ' , ' RCONFLICTS ' , ' ALLOW_EMPTY ' , ' populate_packages() ' , ' do_package() ' , ' do_deploy() ' ]
2014-12-19 11:41:50 +00:00
# Variables that sometimes are a bit long but shouldn't be wrapped
2015-05-18 11:04:55 +00:00
nowrap_vars = [ ' SUMMARY ' , ' HOMEPAGE ' , ' BUGTRACKER ' , ' SRC_URI[md5sum] ' , ' SRC_URI[sha256sum] ' ]
2014-12-19 11:41:50 +00:00
list_vars = [ ' SRC_URI ' , ' LIC_FILES_CHKSUM ' ]
meta_vars = [ ' SUMMARY ' , ' DESCRIPTION ' , ' HOMEPAGE ' , ' BUGTRACKER ' , ' SECTION ' ]
2016-08-16 17:00:13 +00:00
def pn_to_recipe ( cooker , pn , mc = ' ' ) :
2014-12-19 11:41:50 +00:00
""" Convert a recipe name (PN) to the path to the recipe file """
import bb . providers
2016-08-16 17:00:13 +00:00
if pn in cooker . recipecaches [ mc ] . pkg_pn :
best = bb . providers . findBestProvider ( pn , cooker . data , cooker . recipecaches [ mc ] , cooker . recipecaches [ mc ] . pkg_pn )
2014-12-19 11:41:50 +00:00
return best [ 3 ]
2016-08-16 17:00:13 +00:00
elif pn in cooker . recipecaches [ mc ] . providers :
filenames = cooker . recipecaches [ mc ] . providers [ pn ]
eligible , foundUnique = bb . providers . filterProviders ( filenames , pn , cooker . expanded_data , cooker . recipecaches [ mc ] )
2015-10-14 18:19:23 +00:00
filename = eligible [ 0 ]
return filename
2014-12-19 11:41:50 +00:00
else :
return None
def get_unavailable_reasons ( cooker , pn ) :
""" If a recipe could not be found, find out why if possible """
import bb . taskdata
taskdata = bb . taskdata . TaskData ( None , skiplist = cooker . skiplist )
return taskdata . get_reasons ( pn )
2016-08-16 17:00:13 +00:00
def parse_recipe ( cooker , fn , appendfiles ) :
2015-05-14 10:44:59 +00:00
"""
Parse an individual recipe file , optionally with a list of
bbappend files .
"""
2014-12-19 11:41:50 +00:00
import bb . cache
2016-08-16 17:00:13 +00:00
parser = bb . cache . NoCache ( cooker . databuilder )
envdata = parser . loadDataFull ( fn , appendfiles )
2014-12-19 11:41:50 +00:00
return envdata
2015-05-14 10:44:59 +00:00
def parse_recipe_simple ( cooker , pn , d , appends = True ) :
"""
Parse a recipe and optionally all bbappends that apply to it
in the current configuration .
"""
import bb . providers
recipefile = pn_to_recipe ( cooker , pn )
if not recipefile :
skipreasons = get_unavailable_reasons ( cooker , pn )
# We may as well re-use bb.providers.NoProvider here
if skipreasons :
raise bb . providers . NoProvider ( skipreasons )
else :
raise bb . providers . NoProvider ( ' Unable to find any recipe file matching %s ' % pn )
if appends :
appendfiles = cooker . collection . get_file_appends ( recipefile )
2015-09-22 16:21:26 +00:00
else :
appendfiles = None
2016-08-16 17:00:13 +00:00
return parse_recipe ( cooker , recipefile , appendfiles )
2015-05-14 10:44:59 +00:00
2014-12-19 11:41:50 +00:00
def get_var_files ( fn , varlist , d ) :
""" Find the file in which each of a list of variables is set.
Note : requires variable history to be enabled when parsing .
"""
varfiles = { }
for v in varlist :
2015-08-16 11:46:42 +00:00
history = d . varhistory . variable ( v )
2014-12-19 11:41:50 +00:00
files = [ ]
for event in history :
if ' file ' in event and not ' flag ' in event :
files . append ( event [ ' file ' ] )
if files :
actualfile = files [ - 1 ]
else :
actualfile = None
varfiles [ v ] = actualfile
return varfiles
2015-09-22 16:21:15 +00:00
def split_var_value ( value , assignment = True ) :
"""
Split a space - separated variable ' s value into a list of items,
taking into account that some of the items might be made up of
expressions containing spaces that should not be split .
Parameters :
value :
The string value to split
assignment :
True to assume that the value represents an assignment
statement , False otherwise . If True , and an assignment
statement is passed in the first item in
the returned list will be the part of the assignment
statement up to and including the opening quote character ,
and the last item will be the closing quote .
"""
inexpr = 0
lastchar = None
out = [ ]
buf = ' '
for char in value :
if char == ' { ' :
if lastchar == ' $ ' :
inexpr + = 1
elif char == ' } ' :
inexpr - = 1
elif assignment and char in ' " \' ' and inexpr == 0 :
if buf :
out . append ( buf )
out . append ( char )
char = ' '
buf = ' '
elif char . isspace ( ) and inexpr == 0 :
char = ' '
if buf :
out . append ( buf )
buf = ' '
buf + = char
lastchar = char
if buf :
out . append ( buf )
# Join together assignment statement and opening quote
outlist = out
if assignment :
assigfound = False
for idx , item in enumerate ( out ) :
if ' = ' in item :
assigfound = True
if assigfound :
if ' " ' in item or " ' " in item :
outlist = [ ' ' . join ( out [ : idx + 1 ] ) ]
outlist . extend ( out [ idx + 1 : ] )
break
return outlist
2016-05-29 22:20:57 +00:00
def patch_recipe_lines ( fromlines , values , trailing_newline = True ) :
2016-05-29 22:20:56 +00:00
""" Update or insert variable values into lines from a recipe.
2014-12-19 11:41:50 +00:00
Note that some manual inspection / intervention may be required
since this cannot handle all situations .
"""
2015-05-18 11:04:55 +00:00
import bb . utils
2016-05-29 22:20:57 +00:00
if trailing_newline :
newline = ' \n '
else :
newline = ' '
2015-05-18 11:04:55 +00:00
recipe_progression_res = [ ]
recipe_progression_restrs = [ ]
for item in recipe_progression :
if item . endswith ( ' () ' ) :
key = item [ : - 2 ]
else :
key = item
restr = ' %s (_[a-zA-Z0-9-_$() {} ]+| \ [[^ \ ]]* \ ])? ' % key
if item . endswith ( ' () ' ) :
recipe_progression_restrs . append ( restr + ' () ' )
else :
recipe_progression_restrs . append ( restr )
recipe_progression_res . append ( re . compile ( ' ^ %s $ ' % restr ) )
def get_recipe_pos ( variable ) :
for i , p in enumerate ( recipe_progression_res ) :
if p . match ( variable ) :
return i
return - 1
2014-12-19 11:41:50 +00:00
remainingnames = { }
for k in values . keys ( ) :
2015-05-18 11:04:55 +00:00
remainingnames [ k ] = get_recipe_pos ( k )
2016-05-18 18:39:44 +00:00
remainingnames = OrderedDict ( sorted ( remainingnames . items ( ) , key = lambda x : x [ 1 ] ) )
2014-12-19 11:41:50 +00:00
2015-05-18 11:04:55 +00:00
modifying = False
def outputvalue ( name , lines , rewindcomments = False ) :
if values [ name ] is None :
return
2016-05-29 22:20:57 +00:00
rawtext = ' %s = " %s " %s ' % ( name , values [ name ] , newline )
2015-05-18 11:04:55 +00:00
addlines = [ ]
if name in nowrap_vars :
addlines . append ( rawtext )
elif name in list_vars :
splitvalue = split_var_value ( values [ name ] , assignment = False )
if len ( splitvalue ) > 1 :
linesplit = ' \\ \n ' + ( ' ' * ( len ( name ) + 4 ) )
2016-05-29 22:20:57 +00:00
addlines . append ( ' %s = " %s %s " %s ' % ( name , linesplit . join ( splitvalue ) , linesplit , newline ) )
2014-12-19 11:41:50 +00:00
else :
2015-05-18 11:04:55 +00:00
addlines . append ( rawtext )
else :
wrapped = textwrap . wrap ( rawtext )
for wrapline in wrapped [ : - 1 ] :
2016-05-29 22:20:57 +00:00
addlines . append ( ' %s \\ %s ' % ( wrapline , newline ) )
addlines . append ( ' %s %s ' % ( wrapped [ - 1 ] , newline ) )
2015-05-18 11:04:55 +00:00
if rewindcomments :
# Ensure we insert the lines before any leading comments
# (that we'd want to ensure remain leading the next value)
for i , ln in reversed ( list ( enumerate ( lines ) ) ) :
2016-05-29 22:20:57 +00:00
if not ln . startswith ( ' # ' ) :
2015-05-18 11:04:55 +00:00
lines [ i + 1 : i + 1 ] = addlines
break
else :
lines . extend ( addlines )
else :
lines . extend ( addlines )
existingnames = [ ]
def patch_recipe_varfunc ( varname , origvalue , op , newlines ) :
if modifying :
# Insert anything that should come before this variable
pos = get_recipe_pos ( varname )
2016-05-18 18:39:44 +00:00
for k in list ( remainingnames ) :
2015-05-18 11:04:55 +00:00
if remainingnames [ k ] > - 1 and pos > = remainingnames [ k ] and not k in existingnames :
outputvalue ( k , newlines , rewindcomments = True )
del remainingnames [ k ]
# Now change this variable, if it needs to be changed
2016-04-14 08:24:18 +00:00
if varname in existingnames and op in [ ' += ' , ' = ' , ' =+ ' ] :
if varname in remainingnames :
outputvalue ( varname , newlines )
del remainingnames [ varname ]
2015-05-18 11:04:55 +00:00
return None , None , 0 , True
else :
if varname in values :
existingnames . append ( varname )
return origvalue , None , 0 , True
# First run - establish which values we want to set are already in the file
varlist = [ re . escape ( item ) for item in values . keys ( ) ]
2016-05-29 22:20:56 +00:00
bb . utils . edit_metadata ( fromlines , varlist , patch_recipe_varfunc )
2015-05-18 11:04:55 +00:00
# Second run - actually set everything
modifying = True
varlist . extend ( recipe_progression_restrs )
changed , tolines = bb . utils . edit_metadata ( fromlines , varlist , patch_recipe_varfunc , match_overrides = True )
if remainingnames :
2016-07-25 08:47:17 +00:00
if tolines and tolines [ - 1 ] . strip ( ) != ' ' :
2015-05-18 11:04:55 +00:00
tolines . append ( ' \n ' )
for k in remainingnames . keys ( ) :
outputvalue ( k , tolines )
2016-05-29 22:20:56 +00:00
return changed , tolines
def patch_recipe_file ( fn , values , patch = False , relpath = ' ' ) :
""" Update or insert variable values into a recipe file (assuming you
have already identified the exact file you want to update . )
Note that some manual inspection / intervention may be required
since this cannot handle all situations .
"""
with open ( fn , ' r ' ) as f :
fromlines = f . readlines ( )
_ , tolines = patch_recipe_lines ( fromlines , values )
2014-12-19 11:41:50 +00:00
if patch :
relfn = os . path . relpath ( fn , relpath )
diff = difflib . unified_diff ( fromlines , tolines , ' a/ %s ' % relfn , ' b/ %s ' % relfn )
return diff
else :
with open ( fn , ' w ' ) as f :
f . writelines ( tolines )
return None
2015-05-18 11:04:55 +00:00
2014-12-19 11:41:50 +00:00
def localise_file_vars ( fn , varfiles , varlist ) :
""" Given a list of variables and variable history (fetched with get_var_files())
find where each variable should be set / changed . This handles for example where a
recipe includes an inc file where variables might be changed - in most cases
we want to update the inc file when changing the variable value rather than adding
it to the recipe itself .
"""
fndir = os . path . dirname ( fn ) + os . sep
first_meta_file = None
for v in meta_vars :
f = varfiles . get ( v , None )
if f :
actualdir = os . path . dirname ( f ) + os . sep
if actualdir . startswith ( fndir ) :
first_meta_file = f
break
filevars = defaultdict ( list )
for v in varlist :
f = varfiles [ v ]
# Only return files that are in the same directory as the recipe or in some directory below there
# (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
# in if we were going to set a value specific to this recipe)
if f :
actualfile = f
else :
# Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
if first_meta_file :
actualfile = first_meta_file
else :
actualfile = fn
actualdir = os . path . dirname ( actualfile ) + os . sep
if not actualdir . startswith ( fndir ) :
actualfile = fn
filevars [ actualfile ] . append ( v )
return filevars
def patch_recipe ( d , fn , varvalues , patch = False , relpath = ' ' ) :
""" Modify a list of variable values in the specified recipe. Handles inc files if
used by the recipe .
"""
varlist = varvalues . keys ( )
varfiles = get_var_files ( fn , varlist , d )
locs = localise_file_vars ( fn , varfiles , varlist )
patches = [ ]
2016-05-20 10:49:34 +00:00
for f , v in locs . items ( ) :
2014-12-19 11:41:50 +00:00
vals = { k : varvalues [ k ] for k in v }
patchdata = patch_recipe_file ( f , vals , patch , relpath )
if patch :
patches . append ( patchdata )
if patch :
return patches
else :
return None
def copy_recipe_files ( d , tgt_dir , whole_dir = False , download = True ) :
""" Copy (local) recipe files, including both files included via include/require,
and files referred to in the SRC_URI variable . """
import bb . fetch2
import oe . path
# FIXME need a warning if the unexpanded SRC_URI value contains variable references
uris = ( d . getVar ( ' SRC_URI ' , True ) or " " ) . split ( )
fetch = bb . fetch2 . Fetch ( uris , d )
if download :
fetch . download ( )
# Copy local files to target directory and gather any remote files
bb_dir = os . path . dirname ( d . getVar ( ' FILE ' , True ) ) + os . sep
remotes = [ ]
2016-07-13 21:04:24 +00:00
copied = [ ]
2014-12-19 11:41:50 +00:00
includes = [ path for path in d . getVar ( ' BBINCLUDED ' , True ) . split ( ) if
path . startswith ( bb_dir ) and os . path . exists ( path ) ]
for path in fetch . localpaths ( ) + includes :
# Only import files that are under the meta directory
if path . startswith ( bb_dir ) :
if not whole_dir :
relpath = os . path . relpath ( path , bb_dir )
subdir = os . path . join ( tgt_dir , os . path . dirname ( relpath ) )
if not os . path . exists ( subdir ) :
os . makedirs ( subdir )
shutil . copy2 ( path , os . path . join ( tgt_dir , relpath ) )
2016-07-13 21:04:24 +00:00
copied . append ( relpath )
2014-12-19 11:41:50 +00:00
else :
remotes . append ( path )
# Simply copy whole meta dir, if requested
if whole_dir :
shutil . copytree ( bb_dir , tgt_dir )
2016-07-13 21:04:24 +00:00
return copied , remotes
2014-12-19 11:41:50 +00:00
2015-04-23 12:38:52 +00:00
def get_recipe_local_files ( d , patches = False ) :
""" Get a list of local files in SRC_URI within a recipe. """
uris = ( d . getVar ( ' SRC_URI ' , True ) or " " ) . split ( )
fetch = bb . fetch2 . Fetch ( uris , d )
ret = { }
for uri in uris :
if fetch . ud [ uri ] . type == ' file ' :
if ( not patches and
2016-11-10 01:45:17 +00:00
bb . utils . exec_flat_python_func ( ' patch_path ' , uri , fetch , ' ' , expand = False ) ) :
2015-04-23 12:38:52 +00:00
continue
# Skip files that are referenced by absolute path
2016-08-31 23:38:46 +00:00
fname = fetch . ud [ uri ] . basepath
if os . path . isabs ( fname ) :
continue
# Handle subdir=
subdir = fetch . ud [ uri ] . parm . get ( ' subdir ' , ' ' )
if subdir :
if os . path . isabs ( subdir ) :
continue
fname = os . path . join ( subdir , fname )
ret [ fname ] = fetch . localpath ( uri )
2015-04-23 12:38:52 +00:00
return ret
2014-12-19 11:41:50 +00:00
def get_recipe_patches ( d ) :
""" Get a list of the patches included in SRC_URI within a recipe. """
patchfiles = [ ]
# Execute src_patches() defined in patch.bbclass - this works since that class
# is inherited globally
2016-11-10 01:45:17 +00:00
patches = bb . utils . exec_flat_python_func ( ' src_patches ' , d , expand = False )
2014-12-19 11:41:50 +00:00
for patch in patches :
_ , _ , local , _ , _ , parm = bb . fetch . decodeurl ( patch )
patchfiles . append ( local )
return patchfiles
2015-05-18 15:15:07 +00:00
def get_recipe_patched_files ( d ) :
"""
Get the list of patches for a recipe along with the files each patch modifies .
Params :
d : the datastore for the recipe
Returns :
a dict mapping patch file path to a list of tuples of changed files and
change mode ( ' A ' for add , ' D ' for delete or ' M ' for modify )
"""
import oe . patch
# Execute src_patches() defined in patch.bbclass - this works since that class
# is inherited globally
2016-11-10 01:45:17 +00:00
patches = bb . utils . exec_flat_python_func ( ' src_patches ' , d , expand = False )
2015-05-18 15:15:07 +00:00
patchedfiles = { }
for patch in patches :
_ , _ , patchfile , _ , _ , parm = bb . fetch . decodeurl ( patch )
striplevel = int ( parm [ ' striplevel ' ] )
patchedfiles [ patchfile ] = oe . patch . PatchSet . getPatchedFiles ( patchfile , striplevel , os . path . join ( d . getVar ( ' S ' , True ) , parm . get ( ' patchdir ' , ' ' ) ) )
return patchedfiles
2014-12-19 11:41:50 +00:00
def validate_pn ( pn ) :
""" Perform validation on a recipe name (PN) for a new recipe. """
reserved_names = [ ' forcevariable ' , ' append ' , ' prepend ' , ' remove ' ]
2016-09-18 20:08:12 +00:00
if not re . match ( ' ^[0-9a-z-.+]+$ ' , pn ) :
return ' Recipe name " %s " is invalid: only characters 0-9, a-z, -, + and . are allowed ' % pn
2014-12-19 11:41:50 +00:00
elif pn in reserved_names :
return ' Recipe name " %s " is invalid: is a reserved keyword ' % pn
elif pn . startswith ( ' pn- ' ) :
return ' Recipe name " %s " is invalid: names starting with " pn- " are reserved ' % pn
2015-11-19 01:15:55 +00:00
elif pn . endswith ( ( ' .bb ' , ' .bbappend ' , ' .bbclass ' , ' .inc ' , ' .conf ' ) ) :
return ' Recipe name " %s " is invalid: should be just a name, not a file name ' % pn
2014-12-19 11:41:50 +00:00
return ' '
2015-05-18 15:15:07 +00:00
2016-07-13 21:04:25 +00:00
def get_bbfile_path ( d , destdir , extrapathhint = None ) :
"""
Determine the correct path for a recipe within a layer
Parameters :
d : Recipe - specific datastore
destdir : destination directory . Can be the path to the base of the layer or a
partial path somewhere within the layer .
extrapathhint : a path relative to the base of the layer to try
"""
import bb . cookerdata
destdir = os . path . abspath ( destdir )
destlayerdir = find_layerdir ( destdir )
# Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
confdata = d . createCopy ( )
confdata . setVar ( ' BBFILES ' , ' ' )
confdata . setVar ( ' LAYERDIR ' , destlayerdir )
destlayerconf = os . path . join ( destlayerdir , " conf " , " layer.conf " )
confdata = bb . cookerdata . parse_config_file ( destlayerconf , confdata )
pn = d . getVar ( ' PN ' , True )
bbfilespecs = ( confdata . getVar ( ' BBFILES ' , True ) or ' ' ) . split ( )
if destdir == destlayerdir :
for bbfilespec in bbfilespecs :
if not bbfilespec . endswith ( ' .bbappend ' ) :
for match in glob . glob ( bbfilespec ) :
splitext = os . path . splitext ( os . path . basename ( match ) )
if splitext [ 1 ] == ' .bb ' :
mpn = splitext [ 0 ] . split ( ' _ ' ) [ 0 ]
if mpn == pn :
return os . path . dirname ( match )
# Try to make up a path that matches BBFILES
# this is a little crude, but better than nothing
bpn = d . getVar ( ' BPN ' , True )
recipefn = os . path . basename ( d . getVar ( ' FILE ' , True ) )
pathoptions = [ destdir ]
if extrapathhint :
pathoptions . append ( os . path . join ( destdir , extrapathhint ) )
if destdir == destlayerdir :
pathoptions . append ( os . path . join ( destdir , ' recipes- %s ' % bpn , bpn ) )
pathoptions . append ( os . path . join ( destdir , ' recipes ' , bpn ) )
pathoptions . append ( os . path . join ( destdir , bpn ) )
elif not destdir . endswith ( ( ' / ' + pn , ' / ' + bpn ) ) :
pathoptions . append ( os . path . join ( destdir , bpn ) )
closepath = ' '
for pathoption in pathoptions :
bbfilepath = os . path . join ( pathoption , ' test.bb ' )
for bbfilespec in bbfilespecs :
if fnmatch . fnmatchcase ( bbfilepath , bbfilespec ) :
return pathoption
return None
2015-05-18 15:15:07 +00:00
def get_bbappend_path ( d , destlayerdir , wildcardver = False ) :
""" Determine how a bbappend for a recipe should be named and located within another layer """
import bb . cookerdata
destlayerdir = os . path . abspath ( destlayerdir )
recipefile = d . getVar ( ' FILE ' , True )
recipefn = os . path . splitext ( os . path . basename ( recipefile ) ) [ 0 ]
if wildcardver and ' _ ' in recipefn :
recipefn = recipefn . split ( ' _ ' , 1 ) [ 0 ] + ' _ % '
appendfn = recipefn + ' .bbappend '
# Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
confdata = d . createCopy ( )
confdata . setVar ( ' BBFILES ' , ' ' )
confdata . setVar ( ' LAYERDIR ' , destlayerdir )
destlayerconf = os . path . join ( destlayerdir , " conf " , " layer.conf " )
confdata = bb . cookerdata . parse_config_file ( destlayerconf , confdata )
origlayerdir = find_layerdir ( recipefile )
if not origlayerdir :
return ( None , False )
# Now join this to the path where the bbappend is going and check if it is covered by BBFILES
appendpath = os . path . join ( destlayerdir , os . path . relpath ( os . path . dirname ( recipefile ) , origlayerdir ) , appendfn )
closepath = ' '
pathok = True
for bbfilespec in confdata . getVar ( ' BBFILES ' , True ) . split ( ) :
if fnmatch . fnmatchcase ( appendpath , bbfilespec ) :
# Our append path works, we're done
break
elif bbfilespec . startswith ( destlayerdir ) and fnmatch . fnmatchcase ( ' test.bbappend ' , os . path . basename ( bbfilespec ) ) :
# Try to find the longest matching path
if len ( bbfilespec ) > len ( closepath ) :
closepath = bbfilespec
else :
# Unfortunately the bbappend layer and the original recipe's layer don't have the same structure
if closepath :
# bbappend layer's layer.conf at least has a spec that picks up .bbappend files
# Now we just need to substitute out any wildcards
appendsubdir = os . path . relpath ( os . path . dirname ( closepath ) , destlayerdir )
if ' recipes-* ' in appendsubdir :
# Try to copy this part from the original recipe path
res = re . search ( ' /recipes-[^/]+/ ' , recipefile )
if res :
appendsubdir = appendsubdir . replace ( ' /recipes-*/ ' , res . group ( 0 ) )
# This is crude, but we have to do something
appendsubdir = appendsubdir . replace ( ' * ' , recipefn . split ( ' _ ' ) [ 0 ] )
appendsubdir = appendsubdir . replace ( ' ? ' , ' a ' )
appendpath = os . path . join ( destlayerdir , appendsubdir , appendfn )
else :
pathok = False
return ( appendpath , pathok )
def bbappend_recipe ( rd , destlayerdir , srcfiles , install = None , wildcardver = False , machine = None , extralines = None , removevalues = None ) :
"""
Writes a bbappend file for a recipe
Parameters :
rd : data dictionary for the recipe
destlayerdir : base directory of the layer to place the bbappend in
( subdirectory path from there will be determined automatically )
srcfiles : dict of source files to add to SRC_URI , where the value
is the full path to the file to be added , and the value is the
original filename as it would appear in SRC_URI or None if it
isn ' t already present. You may pass None for this parameter if
you simply want to specify your own content via the extralines
parameter .
install : dict mapping entries in srcfiles to a tuple of two elements :
install path ( * without * $ { D } prefix ) and permission value ( as a
string , e . g . ' 0644 ' ) .
wildcardver : True to use a % wildcard in the bbappend filename , or
False to make the bbappend specific to the recipe version .
machine :
If specified , make the changes in the bbappend specific to this
machine . This will also cause PACKAGE_ARCH = " $ {MACHINE_ARCH} "
to be added to the bbappend .
extralines :
Extra lines to add to the bbappend . This may be a dict of name
value pairs , or simply a list of the lines .
removevalues :
Variable values to remove - a dict of names / values .
"""
if not removevalues :
removevalues = { }
# Determine how the bbappend should be named
appendpath , pathok = get_bbappend_path ( rd , destlayerdir , wildcardver )
if not appendpath :
bb . error ( ' Unable to determine layer directory containing %s ' % recipefile )
return ( None , None )
if not pathok :
bb . warn ( ' Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied. ' % ( os . path . join ( destlayerdir , ' conf ' , ' layer.conf ' ) , os . path . dirname ( appendpath ) ) )
appenddir = os . path . dirname ( appendpath )
bb . utils . mkdirhier ( appenddir )
# FIXME check if the bbappend doesn't get overridden by a higher priority layer?
layerdirs = [ os . path . abspath ( layerdir ) for layerdir in rd . getVar ( ' BBLAYERS ' , True ) . split ( ) ]
if not os . path . abspath ( destlayerdir ) in layerdirs :
bb . warn ( ' Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active ' )
bbappendlines = [ ]
if extralines :
if isinstance ( extralines , dict ) :
2016-05-20 10:49:34 +00:00
for name , value in extralines . items ( ) :
2015-05-18 15:15:07 +00:00
bbappendlines . append ( ( name , ' = ' , value ) )
else :
# Do our best to split it
for line in extralines :
if line [ - 1 ] == ' \n ' :
line = line [ : - 1 ]
2015-06-24 22:17:45 +00:00
splitline = line . split ( None , 2 )
2015-05-18 15:15:07 +00:00
if len ( splitline ) == 3 :
bbappendlines . append ( tuple ( splitline ) )
else :
raise Exception ( ' Invalid extralines value passed ' )
def popline ( varname ) :
2016-05-20 10:56:05 +00:00
for i in range ( 0 , len ( bbappendlines ) ) :
2015-05-18 15:15:07 +00:00
if bbappendlines [ i ] [ 0 ] == varname :
line = bbappendlines . pop ( i )
return line
return None
def appendline ( varname , op , value ) :
2016-05-20 10:56:05 +00:00
for i in range ( 0 , len ( bbappendlines ) ) :
2015-05-18 15:15:07 +00:00
item = bbappendlines [ i ]
if item [ 0 ] == varname :
bbappendlines [ i ] = ( item [ 0 ] , item [ 1 ] , item [ 2 ] + ' ' + value )
break
else :
bbappendlines . append ( ( varname , op , value ) )
destsubdir = rd . getVar ( ' PN ' , True )
if srcfiles :
bbappendlines . append ( ( ' FILESEXTRAPATHS_prepend ' , ' := ' , ' $ {THISDIR} /$ {PN} : ' ) )
appendoverride = ' '
if machine :
bbappendlines . append ( ( ' PACKAGE_ARCH ' , ' = ' , ' $ {MACHINE_ARCH} ' ) )
appendoverride = ' _ %s ' % machine
copyfiles = { }
if srcfiles :
instfunclines = [ ]
2016-05-20 10:49:34 +00:00
for newfile , origsrcfile in srcfiles . items ( ) :
2015-05-18 15:15:07 +00:00
srcfile = origsrcfile
srcurientry = None
if not srcfile :
srcfile = os . path . basename ( newfile )
srcurientry = ' file:// %s ' % srcfile
# Double-check it's not there already
# FIXME do we care if the entry is added by another bbappend that might go away?
if not srcurientry in rd . getVar ( ' SRC_URI ' , True ) . split ( ) :
if machine :
appendline ( ' SRC_URI_append %s ' % appendoverride , ' = ' , ' ' + srcurientry )
else :
appendline ( ' SRC_URI ' , ' += ' , srcurientry )
copyfiles [ newfile ] = srcfile
if install :
institem = install . pop ( newfile , None )
if institem :
( destpath , perms ) = institem
instdestpath = replace_dir_vars ( destpath , rd )
instdirline = ' install -d $ {D} %s ' % os . path . dirname ( instdestpath )
if not instdirline in instfunclines :
instfunclines . append ( instdirline )
instfunclines . append ( ' install -m %s $ {WORKDIR} / %s $ {D} %s ' % ( perms , os . path . basename ( srcfile ) , instdestpath ) )
if instfunclines :
bbappendlines . append ( ( ' do_install_append %s () ' % appendoverride , ' ' , instfunclines ) )
bb . note ( ' Writing append file %s ' % appendpath )
if os . path . exists ( appendpath ) :
# Work around lack of nonlocal in python 2
extvars = { ' destsubdir ' : destsubdir }
def appendfile_varfunc ( varname , origvalue , op , newlines ) :
if varname == ' FILESEXTRAPATHS_prepend ' :
if origvalue . startswith ( ' $ {THISDIR} / ' ) :
popline ( ' FILESEXTRAPATHS_prepend ' )
extvars [ ' destsubdir ' ] = rd . expand ( origvalue . split ( ' $ {THISDIR} / ' , 1 ) [ 1 ] . rstrip ( ' : ' ) )
elif varname == ' PACKAGE_ARCH ' :
if machine :
popline ( ' PACKAGE_ARCH ' )
return ( machine , None , 4 , False )
elif varname . startswith ( ' do_install_append ' ) :
func = popline ( varname )
if func :
instfunclines = [ line . strip ( ) for line in origvalue . strip ( ' \n ' ) . splitlines ( ) ]
for line in func [ 2 ] :
if not line in instfunclines :
instfunclines . append ( line )
return ( instfunclines , None , 4 , False )
else :
2015-09-22 16:21:15 +00:00
splitval = split_var_value ( origvalue , assignment = False )
2015-05-18 15:15:07 +00:00
changed = False
removevar = varname
if varname in [ ' SRC_URI ' , ' SRC_URI_append %s ' % appendoverride ] :
removevar = ' SRC_URI '
line = popline ( varname )
if line :
if line [ 2 ] not in splitval :
splitval . append ( line [ 2 ] )
changed = True
else :
line = popline ( varname )
if line :
splitval = [ line [ 2 ] ]
changed = True
if removevar in removevalues :
remove = removevalues [ removevar ]
2016-05-20 10:57:44 +00:00
if isinstance ( remove , str ) :
2015-05-18 15:15:07 +00:00
if remove in splitval :
splitval . remove ( remove )
changed = True
else :
for removeitem in remove :
if removeitem in splitval :
splitval . remove ( removeitem )
changed = True
if changed :
newvalue = splitval
if len ( newvalue ) == 1 :
# Ensure it's written out as one line
if ' _append ' in varname :
newvalue = ' ' + newvalue [ 0 ]
else :
newvalue = newvalue [ 0 ]
if not newvalue and ( op in [ ' += ' , ' .= ' ] or ' _append ' in varname ) :
# There's no point appending nothing
newvalue = None
if varname . endswith ( ' () ' ) :
indent = 4
else :
indent = - 1
return ( newvalue , None , indent , True )
return ( origvalue , None , 4 , False )
varnames = [ item [ 0 ] for item in bbappendlines ]
if removevalues :
2016-05-20 10:53:11 +00:00
varnames . extend ( list ( removevalues . keys ( ) ) )
2015-05-18 15:15:07 +00:00
with open ( appendpath , ' r ' ) as f :
( updated , newlines ) = bb . utils . edit_metadata ( f , varnames , appendfile_varfunc )
destsubdir = extvars [ ' destsubdir ' ]
else :
updated = False
newlines = [ ]
if bbappendlines :
for line in bbappendlines :
if line [ 0 ] . endswith ( ' () ' ) :
newlines . append ( ' %s { \n %s \n } \n ' % ( line [ 0 ] , ' \n ' . join ( line [ 2 ] ) ) )
else :
newlines . append ( ' %s %s " %s " \n \n ' % line )
updated = True
if updated :
with open ( appendpath , ' w ' ) as f :
f . writelines ( newlines )
if copyfiles :
if machine :
destsubdir = os . path . join ( destsubdir , machine )
2016-05-20 10:49:34 +00:00
for newfile , srcfile in copyfiles . items ( ) :
2015-05-18 15:15:07 +00:00
filedest = os . path . join ( appenddir , destsubdir , os . path . basename ( srcfile ) )
if os . path . abspath ( newfile ) != os . path . abspath ( filedest ) :
2016-11-02 03:48:24 +00:00
if newfile . startswith ( tempfile . gettempdir ( ) ) :
newfiledisp = os . path . basename ( newfile )
else :
newfiledisp = newfile
bb . note ( ' Copying %s to %s ' % ( newfiledisp , filedest ) )
2015-05-18 15:15:07 +00:00
bb . utils . mkdirhier ( os . path . dirname ( filedest ) )
shutil . copyfile ( newfile , filedest )
return ( appendpath , os . path . join ( appenddir , destsubdir ) )
def find_layerdir ( fn ) :
2016-07-13 21:04:21 +00:00
""" Figure out the path to the base of the layer containing a file (e.g. a recipe) """
pth = fn
2015-05-18 15:15:07 +00:00
layerdir = ' '
while pth :
if os . path . exists ( os . path . join ( pth , ' conf ' , ' layer.conf ' ) ) :
layerdir = pth
break
pth = os . path . dirname ( pth )
2016-07-13 21:04:21 +00:00
if pth == ' / ' :
return None
2015-05-18 15:15:07 +00:00
return layerdir
def replace_dir_vars ( path , d ) :
""" Replace common directory paths with appropriate variable references (e.g. /etc becomes $ {sysconfdir} ) """
dirvars = { }
2015-07-14 21:56:00 +00:00
# Sort by length so we get the variables we're interested in first
2016-05-20 10:53:11 +00:00
for var in sorted ( list ( d . keys ( ) ) , key = len ) :
2015-05-18 15:15:07 +00:00
if var . endswith ( ' dir ' ) and var . lower ( ) == var :
value = d . getVar ( var , True )
2015-07-10 13:11:16 +00:00
if value . startswith ( ' / ' ) and not ' \n ' in value and value not in dirvars :
2015-05-18 15:15:07 +00:00
dirvars [ value ] = var
2016-05-20 10:53:11 +00:00
for dirpath in sorted ( list ( dirvars . keys ( ) ) , reverse = True ) :
2015-05-18 15:15:07 +00:00
path = path . replace ( dirpath , ' $ { %s } ' % dirvars [ dirpath ] )
return path
2015-06-01 21:04:29 +00:00
def get_recipe_pv_without_srcpv ( pv , uri_type ) :
2015-06-01 21:45:25 +00:00
"""
Get PV without SRCPV common in SCM ' s for now only
support git .
Returns tuple with pv , prefix and suffix .
"""
pfx = ' '
sfx = ' '
if uri_type == ' git ' :
2015-08-03 16:29:54 +00:00
git_regex = re . compile ( " (?P<pfx>v?)(?P<ver>[^ \ +]*)((?P<sfx> \ +(git)?r?(AUTOINC \ +))(?P<rev>.*))? " )
2015-06-01 21:45:25 +00:00
m = git_regex . match ( pv )
if m :
pv = m . group ( ' ver ' )
pfx = m . group ( ' pfx ' )
sfx = m . group ( ' sfx ' )
2015-07-15 00:43:57 +00:00
else :
2015-08-03 16:29:54 +00:00
regex = re . compile ( " (?P<pfx>(v|r)?)(?P<ver>.*) " )
2015-07-15 00:43:57 +00:00
m = regex . match ( pv )
if m :
pv = m . group ( ' ver ' )
pfx = m . group ( ' pfx ' )
2015-06-01 21:45:25 +00:00
return ( pv , pfx , sfx )
def get_recipe_upstream_version ( rd ) :
"""
Get upstream version of recipe using bb . fetch2 methods with support for
http , https , ftp and git .
bb . fetch2 exceptions can be raised ,
FetchError when don ' t have network access or upstream site don ' t response .
NoMethodError when uri latest_versionstring method isn ' t implemented.
Returns a dictonary with version , type and datetime .
Type can be A for Automatic , M for Manual and U for Unknown .
"""
from bb . fetch2 import decodeurl
from datetime import datetime
ru = { }
ru [ ' version ' ] = ' '
ru [ ' type ' ] = ' U '
ru [ ' datetime ' ] = ' '
2015-09-25 11:41:37 +00:00
pv = rd . getVar ( ' PV ' , True )
2015-07-09 23:15:43 +00:00
# XXX: If don't have SRC_URI means that don't have upstream sources so
2015-09-25 11:41:37 +00:00
# returns the current recipe version, so that upstream version check
# declares a match.
2015-07-09 23:15:43 +00:00
src_uris = rd . getVar ( ' SRC_URI ' , True )
if not src_uris :
2015-09-25 11:41:37 +00:00
ru [ ' version ' ] = pv
2015-07-09 23:15:43 +00:00
ru [ ' type ' ] = ' M '
ru [ ' datetime ' ] = datetime . now ( )
return ru
2015-06-01 21:45:25 +00:00
# XXX: we suppose that the first entry points to the upstream sources
2015-07-09 23:15:43 +00:00
src_uri = src_uris . split ( ) [ 0 ]
2015-06-01 21:45:25 +00:00
uri_type , _ , _ , _ , _ , _ = decodeurl ( src_uri )
manual_upstream_version = rd . getVar ( " RECIPE_UPSTREAM_VERSION " , True )
if manual_upstream_version :
# manual tracking of upstream version.
ru [ ' version ' ] = manual_upstream_version
ru [ ' type ' ] = ' M '
manual_upstream_date = rd . getVar ( " CHECK_DATE " , True )
if manual_upstream_date :
date = datetime . strptime ( manual_upstream_date , " % b %d , % Y " )
else :
date = datetime . now ( )
ru [ ' datetime ' ] = date
elif uri_type == " file " :
# files are always up-to-date
ru [ ' version ' ] = pv
ru [ ' type ' ] = ' A '
ru [ ' datetime ' ] = datetime . now ( )
else :
ud = bb . fetch2 . FetchData ( src_uri , rd )
pupver = ud . method . latest_versionstring ( ud , rd )
2015-07-15 00:43:56 +00:00
( upversion , revision ) = pupver
2015-06-01 21:45:25 +00:00
2015-07-15 00:43:56 +00:00
# format git version version+gitAUTOINC+HASH
2015-06-01 21:45:25 +00:00
if uri_type == ' git ' :
2015-06-01 21:04:29 +00:00
( pv , pfx , sfx ) = get_recipe_pv_without_srcpv ( pv , uri_type )
2015-06-01 21:45:25 +00:00
2015-07-15 00:43:56 +00:00
# if contains revision but not upversion use current pv
if upversion == ' ' and revision :
upversion = pv
2015-06-01 21:45:25 +00:00
2015-07-15 00:43:56 +00:00
if upversion :
tmp = upversion
upversion = ' '
2015-07-15 00:43:55 +00:00
if pfx :
2015-07-15 00:43:56 +00:00
upversion = pfx + tmp
else :
upversion = tmp
2015-07-15 00:43:55 +00:00
if sfx :
2015-07-15 00:43:56 +00:00
upversion = upversion + sfx + revision [ : 10 ]
2015-06-01 21:45:25 +00:00
2015-07-15 00:43:56 +00:00
if upversion :
ru [ ' version ' ] = upversion
2015-06-01 21:45:25 +00:00
ru [ ' type ' ] = ' A '
ru [ ' datetime ' ] = datetime . now ( )
return ru