355 lines
13 KiB
Python
355 lines
13 KiB
Python
|
# Development tool - upgrade command plugin
|
||
|
#
|
||
|
# Copyright (C) 2014-2015 Intel Corporation
|
||
|
#
|
||
|
# This program is free software; you can redistribute it and/or modify
|
||
|
# it under the terms of the GNU General Public License version 2 as
|
||
|
# published by the Free Software Foundation.
|
||
|
#
|
||
|
# This program is distributed in the hope that it will be useful,
|
||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
# GNU General Public License for more details.
|
||
|
#
|
||
|
# You should have received a copy of the GNU General Public License along
|
||
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||
|
#
|
||
|
"""Devtool upgrade plugin"""
|
||
|
|
||
|
import os
|
||
|
import sys
|
||
|
import re
|
||
|
import shutil
|
||
|
import tempfile
|
||
|
import logging
|
||
|
import argparse
|
||
|
import scriptutils
|
||
|
import errno
|
||
|
import bb
|
||
|
import oe.recipeutils
|
||
|
from devtool import standard
|
||
|
from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe
|
||
|
|
||
|
logger = logging.getLogger('devtool')
|
||
|
|
||
|
def plugin_init(pluginlist):
|
||
|
"""Plugin initialization"""
|
||
|
pass
|
||
|
|
||
|
def _run(cmd, cwd=''):
|
||
|
logger.debug("Running command %s> %s" % (cwd,cmd))
|
||
|
return bb.process.run('%s' % cmd, cwd=cwd)
|
||
|
|
||
|
def _get_srctree(tmpdir):
|
||
|
srctree = tmpdir
|
||
|
dirs = os.listdir(tmpdir)
|
||
|
if len(dirs) == 1:
|
||
|
srctree = os.path.join(tmpdir, dirs[0])
|
||
|
return srctree
|
||
|
|
||
|
def _copy_source_code(orig, dest):
|
||
|
for path in standard._ls_tree(orig):
|
||
|
dest_dir = os.path.join(dest, os.path.dirname(path))
|
||
|
bb.utils.mkdirhier(dest_dir)
|
||
|
dest_path = os.path.join(dest, path)
|
||
|
os.rename(os.path.join(orig, path), dest_path)
|
||
|
|
||
|
def _get_checksums(rf):
|
||
|
import re
|
||
|
checksums = {}
|
||
|
with open(rf) as f:
|
||
|
for line in f:
|
||
|
for cs in ['md5sum', 'sha256sum']:
|
||
|
m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
|
||
|
if m:
|
||
|
checksums[cs] = m.group(1)
|
||
|
return checksums
|
||
|
|
||
|
def _replace_checksums(rf, md5, sha256):
|
||
|
if not md5 and not sha256:
|
||
|
return
|
||
|
checksums = {'md5sum':md5, 'sha256sum':sha256}
|
||
|
with open(rf + ".tmp", "w+") as tmprf:
|
||
|
with open(rf) as f:
|
||
|
for line in f:
|
||
|
m = None
|
||
|
for cs in checksums.keys():
|
||
|
m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
|
||
|
if m:
|
||
|
if checksums[cs]:
|
||
|
oldcheck = m.group(1)
|
||
|
newcheck = checksums[cs]
|
||
|
line = line.replace(oldcheck, newcheck)
|
||
|
break
|
||
|
tmprf.write(line)
|
||
|
os.rename(rf + ".tmp", rf)
|
||
|
|
||
|
|
||
|
def _remove_patch_dirs(recipefolder):
|
||
|
for root, dirs, files in os.walk(recipefolder):
|
||
|
for d in dirs:
|
||
|
shutil.rmtree(os.path.join(root,d))
|
||
|
|
||
|
def _recipe_contains(rf, var):
|
||
|
import re
|
||
|
found = False
|
||
|
with open(rf) as f:
|
||
|
for line in f:
|
||
|
if re.match("^%s.*=.*" % var, line):
|
||
|
found = True
|
||
|
break
|
||
|
return found
|
||
|
|
||
|
def _rename_recipe_dirs(oldpv, newpv, path):
|
||
|
for root, dirs, files in os.walk(path):
|
||
|
for olddir in dirs:
|
||
|
if olddir.find(oldpv) != -1:
|
||
|
newdir = olddir.replace(oldpv, newpv)
|
||
|
if olddir != newdir:
|
||
|
_run('mv %s %s' % (olddir, newdir))
|
||
|
|
||
|
def _rename_recipe_file(bpn, oldpv, newpv, path):
|
||
|
oldrecipe = "%s_%s.bb" % (bpn, oldpv)
|
||
|
newrecipe = "%s_%s.bb" % (bpn, newpv)
|
||
|
if os.path.isfile(os.path.join(path, oldrecipe)):
|
||
|
if oldrecipe != newrecipe:
|
||
|
_run('mv %s %s' % (oldrecipe, newrecipe), cwd=path)
|
||
|
else:
|
||
|
recipe = "%s_git.bb" % bpn
|
||
|
if os.path.isfile(os.path.join(path, recipe)):
|
||
|
newrecipe = recipe
|
||
|
raise DevtoolError("Original recipe not found on workspace")
|
||
|
return os.path.join(path, newrecipe)
|
||
|
|
||
|
def _rename_recipe_files(bpn, oldpv, newpv, path):
|
||
|
_rename_recipe_dirs(oldpv, newpv, path)
|
||
|
return _rename_recipe_file(bpn, oldpv, newpv, path)
|
||
|
|
||
|
def _use_external_build(same_dir, no_same_dir, d):
|
||
|
b_is_s = True
|
||
|
if no_same_dir:
|
||
|
logger.info('using separate build directory since --no-same-dir specified')
|
||
|
b_is_s = False
|
||
|
elif same_dir:
|
||
|
logger.info('using source tree as build directory since --same-dir specified')
|
||
|
elif bb.data.inherits_class('autotools-brokensep', d):
|
||
|
logger.info('using source tree as build directory since original recipe inherits autotools-brokensep')
|
||
|
elif d.getVar('B', True) == os.path.abspath(d.getVar('S', True)):
|
||
|
logger.info('using source tree as build directory since that is the default for this recipe')
|
||
|
else:
|
||
|
b_is_s = False
|
||
|
return b_is_s
|
||
|
|
||
|
def _write_append(rc, srctree, same_dir, no_same_dir, rev, workspace, d):
|
||
|
"""Writes an append file"""
|
||
|
if not os.path.exists(rc):
|
||
|
raise DevtoolError("bbappend not created because %s does not exist" % rc)
|
||
|
|
||
|
appendpath = os.path.join(workspace, 'appends')
|
||
|
if not os.path.exists(appendpath):
|
||
|
bb.utils.mkdirhier(appendpath)
|
||
|
|
||
|
brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename
|
||
|
|
||
|
srctree = os.path.abspath(srctree)
|
||
|
pn = d.getVar('PN',True)
|
||
|
af = os.path.join(appendpath, '%s.bbappend' % brf)
|
||
|
with open(af, 'w') as f:
|
||
|
f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
|
||
|
f.write('inherit externalsrc\n')
|
||
|
f.write(('# NOTE: We use pn- overrides here to avoid affecting'
|
||
|
'multiple variants in the case where the recipe uses BBCLASSEXTEND\n'))
|
||
|
f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree))
|
||
|
if _use_external_build(same_dir, no_same_dir, d):
|
||
|
f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree))
|
||
|
if rev:
|
||
|
f.write('\n# initial_rev: %s\n' % rev)
|
||
|
return af
|
||
|
|
||
|
def _cleanup_on_error(rf, srctree):
|
||
|
rfp = os.path.split(rf)[0] # recipe folder
|
||
|
rfpp = os.path.split(rfp)[0] # recipes folder
|
||
|
if os.path.exists(rfp):
|
||
|
shutil.rmtree(b)
|
||
|
if not len(os.listdir(rfpp)):
|
||
|
os.rmdir(rfpp)
|
||
|
srctree = os.path.abspath(srctree)
|
||
|
if os.path.exists(srctree):
|
||
|
shutil.rmtree(srctree)
|
||
|
|
||
|
def _upgrade_error(e, rf, srctree):
|
||
|
if rf:
|
||
|
cleanup_on_error(rf, srctree)
|
||
|
logger.error(e)
|
||
|
raise DevtoolError(e)
|
||
|
|
||
|
def _get_uri(rd):
|
||
|
srcuris = rd.getVar('SRC_URI', True).split()
|
||
|
if not len(srcuris):
|
||
|
raise DevtoolError('SRC_URI not found on recipe')
|
||
|
srcuri = srcuris[0] # it is assumed, URI is at first position
|
||
|
srcrev = '${AUTOREV}'
|
||
|
if '://' in srcuri:
|
||
|
# Fetch a URL
|
||
|
rev_re = re.compile(';rev=([^;]+)')
|
||
|
res = rev_re.search(srcuri)
|
||
|
if res:
|
||
|
srcrev = res.group(1)
|
||
|
srcuri = rev_re.sub('', srcuri)
|
||
|
return srcuri, srcrev
|
||
|
|
||
|
def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd):
|
||
|
"""Extract sources of a recipe with a new version"""
|
||
|
|
||
|
def __run(cmd):
|
||
|
"""Simple wrapper which calls _run with srctree as cwd"""
|
||
|
return _run(cmd, srctree)
|
||
|
|
||
|
crd = rd.createCopy()
|
||
|
|
||
|
pv = crd.getVar('PV', True)
|
||
|
crd.setVar('PV', newpv)
|
||
|
|
||
|
tmpsrctree = None
|
||
|
uri, rev = _get_uri(crd)
|
||
|
if srcrev:
|
||
|
rev = srcrev
|
||
|
if uri.startswith('git://'):
|
||
|
__run('git checkout %s' % rev)
|
||
|
__run('git tag -f devtool-base-new')
|
||
|
md5 = None
|
||
|
sha256 = None
|
||
|
else:
|
||
|
__run('git checkout -b devtool-%s' % newpv)
|
||
|
|
||
|
tmpdir = tempfile.mkdtemp(prefix='devtool')
|
||
|
try:
|
||
|
md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev)
|
||
|
except bb.fetch2.FetchError as e:
|
||
|
raise DevtoolError(e)
|
||
|
|
||
|
tmpsrctree = _get_srctree(tmpdir)
|
||
|
|
||
|
scrtree = os.path.abspath(srctree)
|
||
|
|
||
|
_copy_source_code(tmpsrctree, srctree)
|
||
|
|
||
|
(stdout,_) = __run('git ls-files --modified --others --exclude-standard')
|
||
|
for f in stdout.splitlines():
|
||
|
__run('git add "%s"' % f)
|
||
|
|
||
|
__run('git commit -q -m "Commit of upstream changes at version %s" --allow-empty' % newpv)
|
||
|
__run('git tag -f devtool-base-%s' % newpv)
|
||
|
|
||
|
(stdout, _) = __run('git rev-parse HEAD')
|
||
|
rev = stdout.rstrip()
|
||
|
|
||
|
if no_patch:
|
||
|
patches = oe.recipeutils.get_recipe_patches(crd)
|
||
|
if len(patches):
|
||
|
logger.warn('By user choice, the following patches will NOT be applied')
|
||
|
for patch in patches:
|
||
|
logger.warn("%s" % os.path.basename(patch))
|
||
|
else:
|
||
|
try:
|
||
|
__run('git checkout devtool-patched -b %s' % branch)
|
||
|
__run('git rebase %s' % rev)
|
||
|
if uri.startswith('git://'):
|
||
|
suffix = 'new'
|
||
|
else:
|
||
|
suffix = newpv
|
||
|
__run('git tag -f devtool-patched-%s' % suffix)
|
||
|
except bb.process.ExecutionError as e:
|
||
|
logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
|
||
|
|
||
|
if tmpsrctree:
|
||
|
if keep_temp:
|
||
|
logger.info('Preserving temporary directory %s' % tmpsrctree)
|
||
|
else:
|
||
|
shutil.rmtree(tmpsrctree)
|
||
|
|
||
|
return (rev, md5, sha256)
|
||
|
|
||
|
def _create_new_recipe(newpv, md5, sha256, workspace, rd):
|
||
|
"""Creates the new recipe under workspace"""
|
||
|
crd = rd.createCopy()
|
||
|
|
||
|
bpn = crd.getVar('BPN', True)
|
||
|
path = os.path.join(workspace, 'recipes', bpn)
|
||
|
bb.utils.mkdirhier(path)
|
||
|
oe.recipeutils.copy_recipe_files(crd, path)
|
||
|
|
||
|
oldpv = crd.getVar('PV', True)
|
||
|
if not newpv:
|
||
|
newpv = oldpv
|
||
|
fullpath = _rename_recipe_files(bpn, oldpv, newpv, path)
|
||
|
|
||
|
if _recipe_contains(fullpath, 'PV') and newpv != oldpv:
|
||
|
oe.recipeutils.patch_recipe(d, fullpath, {'PV':newpv})
|
||
|
|
||
|
if md5 and sha256:
|
||
|
# Unfortunately, oe.recipeutils.patch_recipe cannot update flags.
|
||
|
# once the latter feature is implemented, we should call patch_recipe
|
||
|
# instead of the following function
|
||
|
_replace_checksums(fullpath, md5, sha256)
|
||
|
|
||
|
return fullpath
|
||
|
|
||
|
def upgrade(args, config, basepath, workspace):
|
||
|
"""Entry point for the devtool 'upgrade' subcommand"""
|
||
|
|
||
|
if args.recipename in workspace:
|
||
|
raise DevtoolError("recipe %s is already in your workspace" % args.recipename)
|
||
|
if not args.version and not args.srcrev:
|
||
|
raise DevtoolError("You must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option")
|
||
|
|
||
|
reason = oe.recipeutils.validate_pn(args.recipename)
|
||
|
if reason:
|
||
|
raise DevtoolError(reason)
|
||
|
|
||
|
tinfoil = setup_tinfoil()
|
||
|
|
||
|
rd = parse_recipe(config, tinfoil, args.recipename, True)
|
||
|
if not rd:
|
||
|
return 1
|
||
|
|
||
|
standard._check_compatible_recipe(args.recipename, rd)
|
||
|
if rd.getVar('PV', True) == args.version and rd.getVar('SRCREV', True) == args.srcrev:
|
||
|
raise DevtoolError("Current and upgrade versions are the same version" % version)
|
||
|
|
||
|
rf = None
|
||
|
try:
|
||
|
rev1 = standard._extract_source(args.srctree, False, 'devtool-orig', rd)
|
||
|
rev2, md5, sha256 = _extract_new_source(args.version, args.srctree, args.no_patch,
|
||
|
args.srcrev, args.branch, args.keep_temp,
|
||
|
tinfoil, rd)
|
||
|
rf = _create_new_recipe(args.version, md5, sha256, config.workspace_path, rd)
|
||
|
except bb.process.CmdError as e:
|
||
|
_upgrade_error(e, rf, args.srctree)
|
||
|
except DevtoolError as e:
|
||
|
_upgrade_error(e, rf, args.srctree)
|
||
|
standard._add_md5(config, args.recipename, os.path.dirname(rf))
|
||
|
|
||
|
af = _write_append(rf, args.srctree, args.same_dir, args.no_same_dir, rev2,
|
||
|
config.workspace_path, rd)
|
||
|
standard._add_md5(config, args.recipename, af)
|
||
|
logger.info('Upgraded source extracted to %s' % args.srctree)
|
||
|
return 0
|
||
|
|
||
|
def register_commands(subparsers, context):
|
||
|
"""Register devtool subcommands from this plugin"""
|
||
|
parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe',
|
||
|
description='Upgrades an existing recipe to a new upstream version')
|
||
|
parser_upgrade.add_argument('recipename', help='Name for recipe to extract the source for')
|
||
|
parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree')
|
||
|
parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)')
|
||
|
parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (if fetching from an SCM such as git)')
|
||
|
parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
|
||
|
parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
|
||
|
group = parser_upgrade.add_mutually_exclusive_group()
|
||
|
group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
|
||
|
group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
|
||
|
parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
|
||
|
parser_upgrade.set_defaults(func=upgrade)
|