generic-poky/meta/classes/packagefeed-stability.bbclass

# Class to avoid copying packages into the feed if they haven't materially changed
#
# Copyright (C) 2015 Intel Corporation
# Released under the MIT license (see COPYING.MIT for details)
#
# This class effectively intercepts packages as they are written out by
# do_package_write_*, causing them to be written into a different
# directory where we can compare them to whatever older packages might
# be in the "real" package feed directory, and avoid copying the new
# package to the feed if it has not materially changed. The idea is to
# avoid unnecessary churn in the packages when dependencies trigger task
# reexecution (and thus repackaging). Enabling the class is simple:
#
# INHERIT += "packagefeed-stability"
#
# Caveats:
# 1) Latest PR values in the build system may not match those in packages
#    seen on the target (naturally)
# 2) If you rebuild from sstate without the existing package feed present,
#    you will lose the "state" of the package feed i.e. the preserved old
#    package versions. Not the end of the world, but would negate the
#    entire purpose of this class.
#
# Note that running -c cleanall on a recipe will purposely delete the old
# package files so they will definitely be copied the next time.

python() {
    if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d):
        return
    # Package backend agnostic intercept
    # This assumes that the package_write task is called package_write_<pkgtype>
    # and that the directory in which packages should be written is
    # pointed to by the variable DEPLOY_DIR_<PKGTYPE>
    for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
        if pkgclass.startswith('package_'):
            pkgtype = pkgclass.split('_', 1)[1]
            pkgwritefunc = 'do_package_write_%s' % pkgtype
            sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False)
            deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper()
            deploydirvarref = '${' + deploydirvar + '}'
            pkgcomparefunc = 'do_package_compare_%s' % pkgtype

            if bb.data.inherits_class('image', d):
                d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc)

            if bb.data.inherits_class('populate_sdk_base', d):
                d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc)

            if bb.data.inherits_class('populate_sdk_ext', d):
                d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc)

            d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc)

            if d.getVarFlag(pkgwritefunc, 'noexec', True) or not d.getVarFlag(pkgwritefunc, 'task', True):
                # Packaging is disabled for this recipe, we shouldn't do anything
                continue

            if deploydirvarref in sstate_outputdirs:
                # Set intermediate output directory
                d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deploydirvarref + '-prediff'))

            d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False))
            d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False))
            d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot')
            bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d)
}

# This isn't the real task function - it's a template that we use in the
# anonymous python code above
fakeroot python do_package_compare () {
    currenttask = d.getVar('BB_CURRENTTASK', True)
    pkgtype = currenttask.rsplit('_', 1)[1]
    package_compare_impl(pkgtype, d)
}

def package_compare_impl(pkgtype, d):
    import errno
    import fnmatch
    import glob
    import subprocess
    import oe.sstatesig

    pn = d.getVar('PN', True)
    deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
    prepath = deploydir + '-prediff/'

    # Find out PKGR values are
    pkgdatadir = d.getVar('PKGDATA_DIR', True)
    packages = []
    try:
        with open(os.path.join(pkgdatadir, pn), 'r') as f:
            for line in f:
                if line.startswith('PACKAGES:'):
                    packages = line.split(':', 1)[1].split()
                    break
    except IOError as e:
        if e.errno == errno.ENOENT:
            pass

    if not packages:
        bb.debug(2, '%s: no packages, nothing to do' % pn)
        return

    pkgrvalues = {}
    rpkgnames = {}
    rdepends = {}
    pkgvvalues = {}
    for pkg in packages:
        with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f:
            for line in f:
                if line.startswith('PKGR:'):
                    pkgrvalues[pkg] = line.split(':', 1)[1].strip()
                if line.startswith('PKGV:'):
                    pkgvvalues[pkg] = line.split(':', 1)[1].strip()
                elif line.startswith('PKG_%s:' % pkg):
                    rpkgnames[pkg] = line.split(':', 1)[1].strip()
                elif line.startswith('RDEPENDS_%s:' % pkg):
                    rdepends[pkg] = line.split(':', 1)[1].strip()

    # Prepare a list of the runtime package names for packages that were
    # actually produced
    rpkglist = []
    for pkg, rpkg in rpkgnames.items():
        if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')):
            rpkglist.append((rpkg, pkg))
    rpkglist.sort(key=lambda x: len(x[0]), reverse=True)

    pvu = d.getVar('PV', False)
    if '$' + '{SRCPV}' in pvu:
        pvprefix = pvu.split('$' + '{SRCPV}', 1)[0]
    else:
        pvprefix = None

    pkgwritetask = 'package_write_%s' % pkgtype
    files = []
    copypkgs = []
    manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d)
    with open(manifest, 'r') as f:
        for line in f:
            if line.startswith(prepath):
                srcpath = line.rstrip()
                if os.path.isfile(srcpath):
                    destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath))

                    # This is crude but should work assuming the output
                    # package file name starts with the package name
                    # and rpkglist is sorted by length (descending)
                    pkgbasename = os.path.basename(destpath)
                    pkgname = None
                    for rpkg, pkg in rpkglist:
                        if pkgbasename.startswith(rpkg):
                            pkgr = pkgrvalues[pkg]
                            destpathspec = destpath.replace(pkgr, '*')
                            if pvprefix:
                                pkgv = pkgvvalues[pkg]
                                if pkgv.startswith(pvprefix):
                                    pkgvsuffix = pkgv[len(pvprefix):]
                                    if '+' in pkgvsuffix:
                                        newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1]
                                        destpathspec = destpathspec.replace(pkgv, newpkgv)
                            pkgname = pkg
                            break
                    else:
                        bb.warn('Unable to map %s back to package' % pkgbasename)
                        destpathspec = destpath

                    oldfiles = glob.glob(destpathspec)
                    oldfile = None
                    docopy = True
                    if oldfiles:
                        oldfile = oldfiles[-1]
                        result = subprocess.call(['pkg-diff.sh', oldfile, srcpath])
                        if result == 0:
                            docopy = False

                    files.append((pkgname, pkgbasename, srcpath, oldfile, destpath))
                    bb.debug(2, '%s: package %s %s' % (pn, files[-1], docopy))
                    if docopy:
                        copypkgs.append(pkgname)

    # Ensure that dependencies on specific versions (such as -dev on the
    # main package) are copied in lock-step
    changed = True
    while changed:
        rpkgdict = {x[0]: x[1] for x in rpkglist}
        changed = False
        for pkgname, pkgbasename, srcpath, oldfile, destpath in files:
            rdeps = rdepends.get(pkgname, None)
            if not rdeps:
                continue
            rdepvers = bb.utils.explode_dep_versions2(rdeps)
            for rdep, versions in rdepvers.items():
                dep = rpkgdict.get(rdep, None)
                for version in versions:
                    if version and version.startswith('= '):
                        if dep in copypkgs and not pkgname in copypkgs:
                            bb.debug(2, '%s: copying %s because it has a fixed version dependency on %s and that package is going to be copied' % (pn, pkgname, dep))
                            changed = True
                            copypkgs.append(pkgname)
                        elif pkgname in copypkgs and not dep in copypkgs:
                            bb.debug(2, '%s: copying %s because %s has a fixed version dependency on it and that package is going to be copied' % (pn, dep, pkgname))
                            changed = True
                            copypkgs.append(dep)

    # Read in old manifest so we can delete any packages we aren't going to replace or preserve
    pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
    try:
        with open(pcmanifest, 'r') as f:
            knownfiles = [x[3] for x in files if x[3]]
            for line in f:
                fn = line.rstrip()
                if fn:
                    if fn in knownfiles:
                        knownfiles.remove(fn)
                    else:
                        try:
                            os.remove(fn)
                            bb.warn('Removed old package %s' % fn)
                        except OSError as e:
                            if e.errno == errno.ENOENT:
                                pass
    except IOError as e:
        if e.errno == errno.ENOENT:
            pass

    # Create new manifest
    with open(pcmanifest, 'w') as f:
        for pkgname, pkgbasename, srcpath, oldfile, destpath in files:
            if pkgname in copypkgs:
                bb.warn('Copying %s' % pkgbasename)
                destdir = os.path.dirname(destpath)
                bb.utils.mkdirhier(destdir)
                if oldfile:
                    try:
                        os.remove(oldfile)
                    except OSError as e:
                        if e.errno == errno.ENOENT:
                            pass
                if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev):
                    # Use a hard link to save space
                    os.link(srcpath, destpath)
                else:
                    shutil.copyfile(srcpath, destpath)
                f.write('%s\n' % destpath)
            else:
                bb.warn('Not copying %s' % pkgbasename)
                f.write('%s\n' % oldfile)


do_cleanall_append() {
    import errno
    for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
        if pkgclass.startswith('package_'):
            pkgtype = pkgclass.split('_', 1)[1]
            deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
            prepath = deploydir + '-prediff'
            pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
            try:
                with open(pcmanifest, 'r') as f:
                    for line in f:
                        fn = line.rstrip()
                        if fn:
                            try:
                                os.remove(fn)
                            except OSError as e:
                                if e.errno == errno.ENOENT:
                                    pass
                os.remove(pcmanifest)
            except IOError as e:
                if e.errno == errno.ENOENT:
                    pass
}