361 lines
16 KiB
Python
361 lines
16 KiB
Python
# Recipe creation tool - append plugin
|
|
#
|
|
# Copyright (C) 2015 Intel Corporation
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
import sys
|
|
import os
|
|
import argparse
|
|
import glob
|
|
import fnmatch
|
|
import re
|
|
import subprocess
|
|
import logging
|
|
import stat
|
|
import shutil
|
|
import scriptutils
|
|
import errno
|
|
from collections import defaultdict
|
|
|
|
logger = logging.getLogger('recipetool')
|
|
|
|
tinfoil = None
|
|
|
|
def plugin_init(pluginlist):
|
|
# Don't need to do anything here right now, but plugins must have this function defined
|
|
pass
|
|
|
|
def tinfoil_init(instance):
|
|
global tinfoil
|
|
tinfoil = instance
|
|
|
|
|
|
# FIXME guessing when we don't have pkgdata?
|
|
# FIXME mode to create patch rather than directly substitute
|
|
|
|
class InvalidTargetFileError(Exception):
|
|
pass
|
|
|
|
def find_target_file(targetpath, d, pkglist=None):
|
|
"""Find the recipe installing the specified target path, optionally limited to a select list of packages"""
|
|
import json
|
|
|
|
pkgdata_dir = d.getVar('PKGDATA_DIR', True)
|
|
|
|
# The mix between /etc and ${sysconfdir} here may look odd, but it is just
|
|
# being consistent with usage elsewhere
|
|
invalidtargets = {'${sysconfdir}/version': '${sysconfdir}/version is written out at image creation time',
|
|
'/etc/timestamp': '/etc/timestamp is written out at image creation time',
|
|
'/dev/*': '/dev is handled by udev (or equivalent) and the kernel (devtmpfs)',
|
|
'/etc/passwd': '/etc/passwd should be managed through the useradd and extrausers classes',
|
|
'/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
|
|
'/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
|
|
'/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
|
|
'${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',}
|
|
|
|
for pthspec, message in invalidtargets.iteritems():
|
|
if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
|
|
raise InvalidTargetFileError(d.expand(message))
|
|
|
|
targetpath_re = re.compile(r'\s+(\$D)?%s(\s|$)' % targetpath)
|
|
|
|
recipes = defaultdict(list)
|
|
for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
|
|
if pkglist:
|
|
filelist = pkglist
|
|
else:
|
|
filelist = files
|
|
for fn in filelist:
|
|
pkgdatafile = os.path.join(root, fn)
|
|
if pkglist and not os.path.exists(pkgdatafile):
|
|
continue
|
|
with open(pkgdatafile, 'r') as f:
|
|
pn = ''
|
|
# This does assume that PN comes before other values, but that's a fairly safe assumption
|
|
for line in f:
|
|
if line.startswith('PN:'):
|
|
pn = line.split(':', 1)[1].strip()
|
|
elif line.startswith('FILES_INFO:'):
|
|
val = line.split(':', 1)[1].strip()
|
|
dictval = json.loads(val)
|
|
for fullpth in dictval.keys():
|
|
if fnmatch.fnmatchcase(fullpth, targetpath):
|
|
recipes[targetpath].append(pn)
|
|
elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'):
|
|
scriptval = line.split(':', 1)[1].strip().decode('string_escape')
|
|
if 'update-alternatives --install %s ' % targetpath in scriptval:
|
|
recipes[targetpath].append('?%s' % pn)
|
|
elif targetpath_re.search(scriptval):
|
|
recipes[targetpath].append('!%s' % pn)
|
|
return recipes
|
|
|
|
def _get_recipe_file(cooker, pn):
|
|
import oe.recipeutils
|
|
recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
|
|
if not recipefile:
|
|
skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
|
|
if skipreasons:
|
|
logger.error('\n'.join(skipreasons))
|
|
else:
|
|
logger.error("Unable to find any recipe file matching %s" % pn)
|
|
return recipefile
|
|
|
|
def _parse_recipe(pn, tinfoil):
|
|
import oe.recipeutils
|
|
recipefile = _get_recipe_file(tinfoil.cooker, pn)
|
|
if not recipefile:
|
|
# Error already logged
|
|
return None
|
|
append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
|
|
rd = oe.recipeutils.parse_recipe(recipefile, append_files,
|
|
tinfoil.config_data)
|
|
return rd
|
|
|
|
def determine_file_source(targetpath, rd):
|
|
"""Assuming we know a file came from a specific recipe, figure out exactly where it came from"""
|
|
import oe.recipeutils
|
|
|
|
# See if it's in do_install for the recipe
|
|
workdir = rd.getVar('WORKDIR', True)
|
|
src_uri = rd.getVar('SRC_URI', True)
|
|
srcfile = ''
|
|
modpatches = []
|
|
elements = check_do_install(rd, targetpath)
|
|
if elements:
|
|
logger.debug('do_install line:\n%s' % ' '.join(elements))
|
|
srcpath = get_source_path(elements)
|
|
logger.debug('source path: %s' % srcpath)
|
|
if not srcpath.startswith('/'):
|
|
# Handle non-absolute path
|
|
srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs', True).split()[-1], srcpath))
|
|
if srcpath.startswith(workdir):
|
|
# OK, now we have the source file name, look for it in SRC_URI
|
|
workdirfile = os.path.relpath(srcpath, workdir)
|
|
# FIXME this is where we ought to have some code in the fetcher, because this is naive
|
|
for item in src_uri.split():
|
|
localpath = bb.fetch2.localpath(item, rd)
|
|
# Source path specified in do_install might be a glob
|
|
if fnmatch.fnmatch(os.path.basename(localpath), workdirfile):
|
|
srcfile = 'file://%s' % localpath
|
|
elif '/' in workdirfile:
|
|
if item == 'file://%s' % workdirfile:
|
|
srcfile = 'file://%s' % localpath
|
|
|
|
# Check patches
|
|
srcpatches = []
|
|
patchedfiles = oe.recipeutils.get_recipe_patched_files(rd)
|
|
for patch, filelist in patchedfiles.iteritems():
|
|
for fileitem in filelist:
|
|
if fileitem[0] == srcpath:
|
|
srcpatches.append((patch, fileitem[1]))
|
|
if srcpatches:
|
|
addpatch = None
|
|
for patch in srcpatches:
|
|
if patch[1] == 'A':
|
|
addpatch = patch[0]
|
|
else:
|
|
modpatches.append(patch[0])
|
|
if addpatch:
|
|
srcfile = 'patch://%s' % addpatch
|
|
|
|
return (srcfile, elements, modpatches)
|
|
|
|
def get_source_path(cmdelements):
|
|
"""Find the source path specified within a command"""
|
|
command = cmdelements[0]
|
|
if command in ['install', 'cp']:
|
|
helptext = subprocess.check_output('LC_ALL=C %s --help' % command, shell=True)
|
|
argopts = ''
|
|
argopt_line_re = re.compile('^-([a-zA-Z0-9]), --[a-z-]+=')
|
|
for line in helptext.splitlines():
|
|
line = line.lstrip()
|
|
res = argopt_line_re.search(line)
|
|
if res:
|
|
argopts += res.group(1)
|
|
if not argopts:
|
|
# Fallback
|
|
if command == 'install':
|
|
argopts = 'gmoSt'
|
|
elif command == 'cp':
|
|
argopts = 't'
|
|
else:
|
|
raise Exception('No fallback arguments for command %s' % command)
|
|
|
|
skipnext = False
|
|
for elem in cmdelements[1:-1]:
|
|
if elem.startswith('-'):
|
|
if len(elem) > 1 and elem[1] in argopts:
|
|
skipnext = True
|
|
continue
|
|
if skipnext:
|
|
skipnext = False
|
|
continue
|
|
return elem
|
|
else:
|
|
raise Exception('get_source_path: no handling for command "%s"')
|
|
|
|
def get_func_deps(func, d):
|
|
"""Find the function dependencies of a shell function"""
|
|
deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
|
|
deps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
|
|
funcdeps = []
|
|
for dep in deps:
|
|
if d.getVarFlag(dep, 'func', True):
|
|
funcdeps.append(dep)
|
|
return funcdeps
|
|
|
|
def check_do_install(rd, targetpath):
|
|
"""Look at do_install for a command that installs/copies the specified target path"""
|
|
instpath = os.path.abspath(os.path.join(rd.getVar('D', True), targetpath.lstrip('/')))
|
|
do_install = rd.getVar('do_install', True)
|
|
# Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose)
|
|
deps = get_func_deps('do_install', rd)
|
|
for dep in deps:
|
|
do_install = do_install.replace(dep, rd.getVar(dep, True))
|
|
|
|
# Look backwards through do_install as we want to catch where a later line (perhaps
|
|
# from a bbappend) is writing over the top
|
|
for line in reversed(do_install.splitlines()):
|
|
line = line.strip()
|
|
if (line.startswith('install ') and ' -m' in line) or line.startswith('cp '):
|
|
elements = line.split()
|
|
destpath = os.path.abspath(elements[-1])
|
|
if destpath == instpath:
|
|
return elements
|
|
elif destpath.rstrip('/') == os.path.dirname(instpath):
|
|
# FIXME this doesn't take recursive copy into account; unsure if it's practical to do so
|
|
srcpath = get_source_path(elements)
|
|
if fnmatch.fnmatchcase(os.path.basename(instpath), os.path.basename(srcpath)):
|
|
return elements
|
|
return None
|
|
|
|
|
|
def appendfile(args):
|
|
import oe.recipeutils
|
|
|
|
if not args.targetpath.startswith('/'):
|
|
logger.error('Target path should start with /')
|
|
return 2
|
|
|
|
if os.path.isdir(args.newfile):
|
|
logger.error('Specified new file "%s" is a directory' % args.newfile)
|
|
return 2
|
|
|
|
if not os.path.exists(args.destlayer):
|
|
logger.error('Destination layer directory "%s" does not exist' % args.destlayer)
|
|
return 2
|
|
if not os.path.exists(os.path.join(args.destlayer, 'conf', 'layer.conf')):
|
|
logger.error('conf/layer.conf not found in destination layer "%s"' % args.destlayer)
|
|
return 2
|
|
|
|
stdout = ''
|
|
try:
|
|
(stdout, _) = bb.process.run('LANG=C file -E -b %s' % args.newfile, shell=True)
|
|
except bb.process.ExecutionError as err:
|
|
logger.debug('file command returned error: %s' % err)
|
|
pass
|
|
if stdout:
|
|
logger.debug('file command output: %s' % stdout.rstrip())
|
|
if ('executable' in stdout and not 'shell script' in stdout) or 'shared object' in stdout:
|
|
logger.warn('This file looks like it is a binary or otherwise the output of compilation. If it is, you should consider building it properly instead of substituting a binary file directly.')
|
|
|
|
if args.recipe:
|
|
recipes = {args.targetpath: [args.recipe],}
|
|
else:
|
|
try:
|
|
recipes = find_target_file(args.targetpath, tinfoil.config_data)
|
|
except InvalidTargetFileError as e:
|
|
logger.error('%s cannot be handled by this tool: %s' % (args.targetpath, e))
|
|
return 1
|
|
if not recipes:
|
|
logger.error('Unable to find any package producing path %s - this may be because the recipe packaging it has not been built yet' % args.targetpath)
|
|
return 1
|
|
|
|
alternative_pns = []
|
|
postinst_pns = []
|
|
|
|
selectpn = None
|
|
for targetpath, pnlist in recipes.iteritems():
|
|
for pn in pnlist:
|
|
if pn.startswith('?'):
|
|
alternative_pns.append(pn[1:])
|
|
elif pn.startswith('!'):
|
|
postinst_pns.append(pn[1:])
|
|
else:
|
|
selectpn = pn
|
|
|
|
if not selectpn and len(alternative_pns) == 1:
|
|
selectpn = alternative_pns[0]
|
|
logger.error('File %s is an alternative possibly provided by recipe %s but seemingly no other, selecting it by default - you should double check other recipes' % (args.targetpath, selectpn))
|
|
|
|
if selectpn:
|
|
logger.debug('Selecting recipe %s for file %s' % (selectpn, args.targetpath))
|
|
if postinst_pns:
|
|
logger.warn('%s be modified by postinstall scripts for the following recipes:\n %s\nThis may or may not be an issue depending on what modifications these postinstall scripts make.' % (args.targetpath, '\n '.join(postinst_pns)))
|
|
rd = _parse_recipe(selectpn, tinfoil)
|
|
if not rd:
|
|
# Error message already shown
|
|
return 1
|
|
sourcefile, instelements, modpatches = determine_file_source(args.targetpath, rd)
|
|
sourcepath = None
|
|
if sourcefile:
|
|
sourcetype, sourcepath = sourcefile.split('://', 1)
|
|
logger.debug('Original source file is %s (%s)' % (sourcepath, sourcetype))
|
|
if sourcetype == 'patch':
|
|
logger.warn('File %s is added by the patch %s - you may need to remove or replace this patch in order to replace the file.' % (args.targetpath, sourcepath))
|
|
sourcepath = None
|
|
else:
|
|
logger.debug('Unable to determine source file, proceeding anyway')
|
|
if modpatches:
|
|
logger.warn('File %s is modified by the following patches:\n %s' % (args.targetpath, '\n '.join(modpatches)))
|
|
|
|
if instelements and sourcepath:
|
|
install = None
|
|
else:
|
|
# Auto-determine permissions
|
|
# Check destination
|
|
binpaths = '${bindir}:${sbindir}:${base_bindir}:${base_sbindir}:${libexecdir}:${sysconfdir}/init.d'
|
|
perms = '0644'
|
|
if os.path.abspath(os.path.dirname(args.targetpath)) in rd.expand(binpaths).split(':'):
|
|
# File is going into a directory normally reserved for executables, so it should be executable
|
|
perms = '0755'
|
|
else:
|
|
# Check source
|
|
st = os.stat(args.newfile)
|
|
if st.st_mode & stat.S_IXUSR:
|
|
perms = '0755'
|
|
install = {args.newfile: (args.targetpath, perms)}
|
|
oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine)
|
|
return 0
|
|
else:
|
|
if alternative_pns:
|
|
logger.error('File %s is an alternative possibly provided by the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(alternative_pns)))
|
|
elif postinst_pns:
|
|
logger.error('File %s may be written out in a pre/postinstall script of the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(postinst_pns)))
|
|
return 3
|
|
|
|
|
|
def register_command(subparsers):
|
|
parser_appendfile = subparsers.add_parser('appendfile',
|
|
help='Create a bbappend to replace a file',
|
|
description='')
|
|
parser_appendfile.add_argument('destlayer', help='Destination layer to write the bbappend to')
|
|
parser_appendfile.add_argument('targetpath', help='Path within the image to the file to be replaced')
|
|
parser_appendfile.add_argument('newfile', help='Custom file to replace it with')
|
|
parser_appendfile.add_argument('-r', '--recipe', help='Override recipe to apply to (default is to find which recipe already packages it)')
|
|
parser_appendfile.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE')
|
|
parser_appendfile.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
|
|
parser_appendfile.set_defaults(func=appendfile, parserecipes=True)
|