devtool: add build-sdk subcommand
Add a build-sdk command which is only available within the extensible SDK that builds a derivative extensible SDK. The idea is recipes in the workspace become a part of the new SDK - for example, this allows taking a vendor provided SDK, adding a few libs and then producing a new SDK with those included. When normally building the extensible SDK, the workspace is excluded; here we need to copy into the new SDK (renaming it in the process); the recipes' task signatures become locked and thus the sources are no longer needed, so they are removed along with the workspace bbappends which would interfere with the locked signatures. Additionally we need to just copy the configuration files (i.e. local.conf and auto.conf) rather than filtering and appending to them since that work has already been done when constructing the original SDK. The extra sstate artifacts from workspace recipes are also determined and copied into the new SDK in minimal mode (on the assumption that you won't set up a new sstate mirror). This reuses some code from build-image, so that needed to be generalised to allow that. Implements [YOCTO #8892]. (From OE-Core rev: 59e207ff6dd4b50a8905e14bc9292cf2794f4e7a) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Paul Eggleton
Richard Purdie
parent
41eb36dc4c
commit
25d9c4e02a
|
|
@ -89,7 +89,14 @@ python copy_buildsystem () {
|
|||
# Copy in all metadata layers + bitbake (as repositories)
|
||||
buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d)
|
||||
baseoutpath = d.getVar('SDK_OUTPUT', True) + '/' + d.getVar('SDKPATH', True)
|
||||
layers_copied = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers')
|
||||
|
||||
# Determine if we're building a derivative extensible SDK (from devtool build-sdk)
|
||||
derivative = (d.getVar('SDK_DERIVATIVE', True) or '') == '1'
|
||||
if derivative:
|
||||
workspace_name = 'orig-workspace'
|
||||
else:
|
||||
workspace_name = None
|
||||
layers_copied = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name)
|
||||
|
||||
sdkbblayers = []
|
||||
corebase = os.path.basename(d.getVar('COREBASE', True))
|
||||
|
|
@ -158,6 +165,10 @@ python copy_buildsystem () {
|
|||
f.write(' "\n')
|
||||
|
||||
# Create local.conf
|
||||
builddir = d.getVar('TOPDIR', True)
|
||||
if derivative:
|
||||
shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf')
|
||||
else:
|
||||
local_conf_whitelist = (d.getVar('SDK_LOCAL_CONF_WHITELIST', True) or '').split()
|
||||
local_conf_blacklist = (d.getVar('SDK_LOCAL_CONF_BLACKLIST', True) or '').split()
|
||||
def handle_var(varname, origvalue, op, newlines):
|
||||
|
|
@ -167,7 +178,6 @@ python copy_buildsystem () {
|
|||
else:
|
||||
return origvalue, op, 0, True
|
||||
varlist = ['[^#=+ ]*']
|
||||
builddir = d.getVar('TOPDIR', True)
|
||||
with open(builddir + '/conf/local.conf', 'r') as f:
|
||||
oldlines = f.readlines()
|
||||
(updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
|
||||
|
|
@ -216,6 +226,9 @@ python copy_buildsystem () {
|
|||
f.write('require conf/locked-sigs.inc\n')
|
||||
|
||||
if os.path.exists(builddir + '/conf/auto.conf'):
|
||||
if derivative:
|
||||
shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf')
|
||||
else:
|
||||
with open(builddir + '/conf/auto.conf', 'r') as f:
|
||||
oldlines = f.readlines()
|
||||
(updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
|
||||
|
|
@ -253,7 +266,24 @@ python copy_buildsystem () {
|
|||
lockedsigs_pruned,
|
||||
lockedsigs_copy)
|
||||
|
||||
if d.getVar('SDK_EXT_TYPE', True) != 'minimal':
|
||||
if d.getVar('SDK_EXT_TYPE', True) == 'minimal':
|
||||
if derivative:
|
||||
# Assume the user is not going to set up an additional sstate
|
||||
# mirror, thus we need to copy the additional artifacts (from
|
||||
# workspace recipes) into the derivative SDK
|
||||
lockedsigs_orig = d.getVar('TOPDIR', True) + '/conf/locked-sigs.inc'
|
||||
if os.path.exists(lockedsigs_orig):
|
||||
lockedsigs_extra = d.getVar('WORKDIR', True) + '/locked-sigs-extra.inc'
|
||||
oe.copy_buildsystem.merge_lockedsigs(None,
|
||||
lockedsigs_orig,
|
||||
lockedsigs_pruned,
|
||||
None,
|
||||
lockedsigs_extra)
|
||||
oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra,
|
||||
d.getVar('SSTATE_DIR', True),
|
||||
sstate_out, d,
|
||||
fixedlsbstring)
|
||||
else:
|
||||
oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned,
|
||||
d.getVar('SSTATE_DIR', True),
|
||||
sstate_out, d,
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class BuildSystem(object):
|
|||
self.layerdirs = d.getVar('BBLAYERS', True).split()
|
||||
self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE', True) or "").split()
|
||||
|
||||
def copy_bitbake_and_layers(self, destdir):
|
||||
def copy_bitbake_and_layers(self, destdir, workspace_name=None):
|
||||
# Copy in all metadata layers + bitbake (as repositories)
|
||||
layers_copied = []
|
||||
bb.utils.mkdirhier(destdir)
|
||||
|
|
@ -34,6 +34,14 @@ class BuildSystem(object):
|
|||
if layer_exclude in layers:
|
||||
layers.remove(layer_exclude)
|
||||
|
||||
workspace_newname = workspace_name
|
||||
if workspace_newname:
|
||||
layernames = [os.path.basename(layer) for layer in layers]
|
||||
extranum = 0
|
||||
while workspace_newname in layernames:
|
||||
extranum += 1
|
||||
workspace_newname = '%s-%d' % (workspace_name, extranum)
|
||||
|
||||
corebase_files = self.d.getVar('COREBASE_FILES', True).split()
|
||||
corebase_files = [corebase + '/' +x for x in corebase_files]
|
||||
# Make sure bitbake goes in
|
||||
|
|
@ -42,9 +50,15 @@ class BuildSystem(object):
|
|||
|
||||
for layer in layers:
|
||||
layerconf = os.path.join(layer, 'conf', 'layer.conf')
|
||||
layernewname = os.path.basename(layer)
|
||||
workspace = False
|
||||
if os.path.exists(layerconf):
|
||||
with open(layerconf, 'r') as f:
|
||||
if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"):
|
||||
if workspace_newname:
|
||||
layernewname = workspace_newname
|
||||
workspace = True
|
||||
else:
|
||||
bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context))
|
||||
continue
|
||||
|
||||
|
|
@ -53,7 +67,7 @@ class BuildSystem(object):
|
|||
layerdestpath = destdir
|
||||
if corebase == os.path.dirname(layer):
|
||||
layerdestpath += '/' + os.path.basename(corebase)
|
||||
layerdestpath += '/' + os.path.basename(layer)
|
||||
layerdestpath += '/' + layernewname
|
||||
|
||||
layer_relative = os.path.relpath(layerdestpath,
|
||||
destdir)
|
||||
|
|
@ -73,6 +87,38 @@ class BuildSystem(object):
|
|||
else:
|
||||
_smart_copy(layer, layerdestpath)
|
||||
|
||||
if workspace:
|
||||
# Make some adjustments original workspace layer
|
||||
# Drop sources (recipe tasks will be locked, so we don't need them)
|
||||
srcdir = os.path.join(layerdestpath, 'sources')
|
||||
if os.path.isdir(srcdir):
|
||||
shutil.rmtree(srcdir)
|
||||
# Drop all bbappends except the one for the image the SDK is being built for
|
||||
# (because of externalsrc, the workspace bbappends will interfere with the
|
||||
# locked signatures if present, and we don't need them anyway)
|
||||
image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE', True)))[0] + '.bbappend'
|
||||
appenddir = os.path.join(layerdestpath, 'appends')
|
||||
if os.path.isdir(appenddir):
|
||||
for fn in os.listdir(appenddir):
|
||||
if fn == image_bbappend:
|
||||
continue
|
||||
else:
|
||||
os.remove(os.path.join(appenddir, fn))
|
||||
# Drop README
|
||||
readme = os.path.join(layerdestpath, 'README')
|
||||
if os.path.exists(readme):
|
||||
os.remove(readme)
|
||||
# Filter out comments in layer.conf and change layer name
|
||||
layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf')
|
||||
with open(layerconf, 'r') as f:
|
||||
origlines = f.readlines()
|
||||
with open(layerconf, 'w') as f:
|
||||
for line in origlines:
|
||||
if line.startswith('#'):
|
||||
continue
|
||||
line = line.replace('workspacelayer', workspace_newname)
|
||||
f.write(line)
|
||||
|
||||
return layers_copied
|
||||
|
||||
def generate_locked_sigs(sigfile, d):
|
||||
|
|
@ -123,7 +169,7 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu
|
|||
if line.endswith('\\\n'):
|
||||
if not line in merged[invalue]:
|
||||
target, task = line.strip().split(':')[:2]
|
||||
if task in copy_tasks:
|
||||
if not copy_tasks or task in copy_tasks:
|
||||
tocopy[invalue].append(line)
|
||||
merged[invalue].append(line)
|
||||
else:
|
||||
|
|
@ -150,6 +196,7 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu
|
|||
f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes))
|
||||
|
||||
write_sigs_file(copy_output, tocopy.keys(), tocopy)
|
||||
if merged_output:
|
||||
write_sigs_file(merged_output, arch_order, merged)
|
||||
|
||||
def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring=""):
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@ from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, Devtool
|
|||
|
||||
logger = logging.getLogger('devtool')
|
||||
|
||||
class TargetNotImageError(Exception):
|
||||
pass
|
||||
|
||||
def _get_packages(tinfoil, workspace, config):
|
||||
"""Get list of packages from recipes in the workspace."""
|
||||
result = []
|
||||
|
|
@ -51,6 +54,24 @@ def build_image(args, config, basepath, workspace):
|
|||
if not image:
|
||||
raise DevtoolError('Unable to determine image to build, please specify one')
|
||||
|
||||
try:
|
||||
if args.add_packages:
|
||||
add_packages = args.add_packages.split(',')
|
||||
else:
|
||||
add_packages = None
|
||||
result, outputdir = build_image_task(config, basepath, workspace, image, add_packages)
|
||||
except TargetNotImageError:
|
||||
if auto_image:
|
||||
raise DevtoolError('Unable to determine image to build, please specify one')
|
||||
else:
|
||||
raise DevtoolError('Specified recipe %s is not an image recipe' % image)
|
||||
|
||||
if result == 0:
|
||||
logger.info('Successfully built %s. You can find output files in %s'
|
||||
% (image, outputdir))
|
||||
return result
|
||||
|
||||
def build_image_task(config, basepath, workspace, image, add_packages=None, task=None, extra_append=None):
|
||||
appendfile = os.path.join(config.workspace_path, 'appends',
|
||||
'%s.bbappend' % image)
|
||||
|
||||
|
|
@ -63,46 +84,60 @@ def build_image(args, config, basepath, workspace):
|
|||
rd = parse_recipe(config, tinfoil, image, True)
|
||||
if not rd:
|
||||
# Error already shown
|
||||
return 1
|
||||
return (1, None)
|
||||
if not bb.data.inherits_class('image', rd):
|
||||
if auto_image:
|
||||
raise DevtoolError('Unable to determine image to build, please specify one')
|
||||
else:
|
||||
raise DevtoolError('Specified recipe %s is not an image recipe' % image)
|
||||
raise TargetNotImageError()
|
||||
|
||||
outputdir = None
|
||||
try:
|
||||
if workspace or args.add_packages:
|
||||
if args.add_packages:
|
||||
packages = args.add_packages.split(',')
|
||||
if workspace or add_packages:
|
||||
if add_packages:
|
||||
packages = add_packages
|
||||
else:
|
||||
packages = _get_packages(tinfoil, workspace, config)
|
||||
if packages:
|
||||
else:
|
||||
packages = None
|
||||
if not task:
|
||||
if not packages and not add_packages and workspace:
|
||||
logger.warning('No recipes in workspace, building image %s unmodified', image)
|
||||
elif not packages:
|
||||
logger.warning('No packages to add, building image %s unmodified', image)
|
||||
|
||||
if packages or extra_append:
|
||||
bb.utils.mkdirhier(os.path.dirname(appendfile))
|
||||
with open(appendfile, 'w') as afile:
|
||||
if packages:
|
||||
# include packages from workspace recipes into the image
|
||||
afile.write('IMAGE_INSTALL_append = " %s"\n' % ' '.join(packages))
|
||||
if not task:
|
||||
logger.info('Building image %s with the following '
|
||||
'additional packages: %s', image, ' '.join(packages))
|
||||
else:
|
||||
logger.warning('No packages to add, building image %s unmodified', image)
|
||||
else:
|
||||
logger.warning('No recipes in workspace, building image %s unmodified', image)
|
||||
if extra_append:
|
||||
for line in extra_append:
|
||||
afile.write('%s\n' % line)
|
||||
|
||||
deploy_dir_image = tinfoil.config_data.getVar('DEPLOY_DIR_IMAGE', True)
|
||||
if task in ['populate_sdk', 'populate_sdk_ext']:
|
||||
outputdir = rd.getVar('SDK_DEPLOY', True)
|
||||
else:
|
||||
outputdir = rd.getVar('DEPLOY_DIR_IMAGE', True)
|
||||
|
||||
tinfoil.shutdown()
|
||||
|
||||
# run bitbake to build image
|
||||
options = ''
|
||||
if task:
|
||||
options += '-c %s' % task
|
||||
|
||||
# run bitbake to build image (or specified task)
|
||||
try:
|
||||
exec_build_env_command(config.init_path, basepath,
|
||||
'bitbake %s' % image, watch=True)
|
||||
'bitbake %s %s' % (options, image), watch=True)
|
||||
except ExecutionError as err:
|
||||
return err.exitcode
|
||||
return (err.exitcode, None)
|
||||
finally:
|
||||
if os.path.isfile(appendfile):
|
||||
os.unlink(appendfile)
|
||||
return (0, outputdir)
|
||||
|
||||
logger.info('Successfully built %s. You can find output files in %s'
|
||||
% (image, deploy_dir_image))
|
||||
|
||||
def register_commands(subparsers, context):
|
||||
"""Register devtool subcommands from the build-image plugin"""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
# Development tool - build-sdk command plugin
|
||||
#
|
||||
# Copyright (C) 2015-2016 Intel Corporation
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import logging
|
||||
import glob
|
||||
import shutil
|
||||
import errno
|
||||
import sys
|
||||
import tempfile
|
||||
from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
|
||||
from devtool import build_image
|
||||
|
||||
logger = logging.getLogger('devtool')
|
||||
|
||||
|
||||
def build_sdk(args, config, basepath, workspace):
|
||||
"""Entry point for the devtool build-sdk command"""
|
||||
|
||||
sdk_targets = config.get('SDK', 'sdk_targets', '').split()
|
||||
if sdk_targets:
|
||||
image = sdk_targets[0]
|
||||
else:
|
||||
raise DevtoolError('Unable to determine image to build SDK for')
|
||||
|
||||
extra_append = ['SDK_DERIVATIVE = "1"']
|
||||
try:
|
||||
result, outputdir = build_image.build_image_task(config,
|
||||
basepath,
|
||||
workspace,
|
||||
image,
|
||||
task='populate_sdk_ext',
|
||||
extra_append=extra_append)
|
||||
except build_image.TargetNotImageError:
|
||||
raise DevtoolError('Unable to determine image to build SDK for')
|
||||
|
||||
if result == 0:
|
||||
logger.info('Successfully built SDK. You can find output files in %s'
|
||||
% outputdir)
|
||||
return result
|
||||
|
||||
|
||||
def register_commands(subparsers, context):
|
||||
"""Register devtool subcommands"""
|
||||
if context.fixed_setup:
|
||||
parser_build_sdk = subparsers.add_parser('build-sdk',
|
||||
help='Build a derivative SDK of this one',
|
||||
description='Builds an extensible SDK based upon this one and the items in your workspace',
|
||||
group='advanced')
|
||||
parser_build_sdk.set_defaults(func=build_sdk)
|
||||
Loading…
Reference in New Issue