2011-11-30 16:48:47 +00:00
#
# Records history of build output in order to detect regressions
#
2011-12-01 17:26:44 +00:00
# Based in part on testlab.bbclass and packagehistory.bbclass
2011-11-30 16:48:47 +00:00
#
2014-02-11 16:10:11 +00:00
# Copyright (C) 2011-2014 Intel Corporation
2011-11-30 16:48:47 +00:00
# Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org>
#
2013-03-22 19:53:52 +00:00
BUILDHISTORY_FEATURES ?= "image package sdk"
2013-04-27 15:15:49 +00:00
BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
2011-11-30 16:48:47 +00:00
BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
2011-12-01 17:26:44 +00:00
BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
2013-06-07 16:35:18 +00:00
BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}/${IMAGE_BASENAME}"
2013-05-09 15:57:48 +00:00
BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group"
2011-11-30 16:48:47 +00:00
BUILDHISTORY_COMMIT ?= "0"
BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>"
BUILDHISTORY_PUSH_REPO ?= ""
2014-09-09 10:41:37 +00:00
SSTATEPOSTINSTFUNCS_append = " buildhistory_emit_pkghistory"
2014-03-03 15:45:57 +00:00
# We want to avoid influence the signatures of sstate tasks - first the function itself:
sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory"
# then the value added to SSTATEPOSTINSTFUNCS:
SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory"
2012-03-16 14:56:37 +00:00
2015-11-25 09:30:30 +00:00
# All items excepts those listed here will be removed from a recipe's
# build history directory by buildhistory_emit_pkghistory(). This is
# necessary because some of these items (package directories, files that
# we no longer emit) might be obsolete.
#
# When extending build history, derive your class from buildhistory.bbclass
# and extend this list here with the additional files created by the derived
# class.
BUILDHISTORY_PRESERVE = "latest latest_srcrev"
2011-12-01 17:26:44 +00:00
#
2015-11-05 15:48:18 +00:00
# Write out metadata about this package for comparison when writing future packages
2011-12-01 17:26:44 +00:00
#
python buildhistory_emit_pkghistory() {
2013-12-02 18:50:49 +00:00
if not d.getVar('BB_CURRENTTASK', True) in ['packagedata', 'packagedata_setscene']:
return 0
2012-07-19 11:32:20 +00:00
if not "package" in (d.getVar('BUILDHISTORY_FEATURES', True) or "").split():
return 0
2013-12-02 18:50:49 +00:00
import re
import json
import errno
2012-07-19 11:32:20 +00:00
pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True)
class RecipeInfo:
def __init__(self, name):
self.name = name
self.pe = "0"
self.pv = "0"
self.pr = "r0"
self.depends = ""
self.packages = ""
2013-02-27 15:37:24 +00:00
self.srcrev = ""
2012-07-19 11:32:20 +00:00
class PackageInfo:
def __init__(self, name):
self.name = name
self.pe = "0"
self.pv = "0"
self.pr = "r0"
2012-08-02 09:23:05 +00:00
# pkg/pkge/pkgv/pkgr should be empty because we want to be able to default them
self.pkg = ""
self.pkge = ""
self.pkgv = ""
self.pkgr = ""
2012-07-19 11:32:20 +00:00
self.size = 0
self.depends = ""
2013-02-08 09:18:20 +00:00
self.rprovides = ""
2012-07-19 11:32:20 +00:00
self.rdepends = ""
self.rrecommends = ""
2013-02-08 09:18:20 +00:00
self.rsuggests = ""
self.rreplaces = ""
self.rconflicts = ""
2012-07-19 11:32:20 +00:00
self.files = ""
self.filelist = ""
2012-08-02 09:23:04 +00:00
# Variables that need to be written to their own separate file
self.filevars = dict.fromkeys(['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'])
2012-07-19 11:32:20 +00:00
# Should check PACKAGES here to see if anything removed
def readPackageInfo(pkg, histfile):
pkginfo = PackageInfo(pkg)
2012-08-02 09:23:05 +00:00
with open(histfile, "r") as f:
2012-07-19 11:32:20 +00:00
for line in f:
2015-10-04 07:38:16 +00:00
lns = line.split('=', 1)
2012-07-19 11:32:20 +00:00
name = lns[0].strip()
value = lns[1].strip(" \t\r\n").strip('"')
if name == "PE":
pkginfo.pe = value
elif name == "PV":
pkginfo.pv = value
elif name == "PR":
pkginfo.pr = value
2012-08-02 09:23:05 +00:00
elif name == "PKG":
pkginfo.pkg = value
elif name == "PKGE":
pkginfo.pkge = value
elif name == "PKGV":
pkginfo.pkgv = value
elif name == "PKGR":
pkginfo.pkgr = value
2013-02-08 09:18:20 +00:00
elif name == "RPROVIDES":
pkginfo.rprovides = value
2012-07-19 11:32:20 +00:00
elif name == "RDEPENDS":
pkginfo.rdepends = value
elif name == "RRECOMMENDS":
pkginfo.rrecommends = value
2013-02-08 09:18:20 +00:00
elif name == "RSUGGESTS":
pkginfo.rsuggests = value
elif name == "RREPLACES":
pkginfo.rreplaces = value
elif name == "RCONFLICTS":
pkginfo.rconflicts = value
2012-07-19 11:32:20 +00:00
elif name == "PKGSIZE":
pkginfo.size = long(value)
elif name == "FILES":
pkginfo.files = value
elif name == "FILELIST":
pkginfo.filelist = value
2012-08-02 09:23:05 +00:00
# Apply defaults
if not pkginfo.pkg:
pkginfo.pkg = pkginfo.name
if not pkginfo.pkge:
pkginfo.pkge = pkginfo.pe
if not pkginfo.pkgv:
pkginfo.pkgv = pkginfo.pv
if not pkginfo.pkgr:
pkginfo.pkgr = pkginfo.pr
2012-07-19 11:32:20 +00:00
return pkginfo
def getlastpkgversion(pkg):
try:
histfile = os.path.join(pkghistdir, pkg, "latest")
return readPackageInfo(pkg, histfile)
except EnvironmentError:
return None
def sortpkglist(string):
pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+ [^ )]+\))?', string, 0)
pkglist = [p.group(0) for p in pkgiter]
pkglist.sort()
return ' '.join(pkglist)
def sortlist(string):
items = string.split(' ')
items.sort()
return ' '.join(items)
pn = d.getVar('PN', True)
pe = d.getVar('PE', True) or "0"
pv = d.getVar('PV', True)
pr = d.getVar('PR', True)
2013-02-27 15:37:24 +00:00
2013-12-02 18:50:49 +00:00
pkgdata_dir = d.getVar('PKGDATA_DIR', True)
packages = ""
try:
with open(os.path.join(pkgdata_dir, pn)) as f:
for line in f.readlines():
if line.startswith('PACKAGES: '):
2014-11-05 19:21:58 +00:00
packages = oe.utils.squashspaces(line.split(': ', 1)[1])
2013-12-02 18:50:49 +00:00
break
except IOError as e:
if e.errno == errno.ENOENT:
# Probably a -cross recipe, just ignore
return 0
else:
raise
2012-07-19 11:32:20 +00:00
2012-08-02 09:23:02 +00:00
packagelist = packages.split()
2015-11-25 09:30:30 +00:00
preserve = d.getVar('BUILDHISTORY_PRESERVE', True).split()
2012-08-02 09:23:02 +00:00
if not os.path.exists(pkghistdir):
2013-08-06 22:09:49 +00:00
bb.utils.mkdirhier(pkghistdir)
2012-08-02 09:23:02 +00:00
else:
# Remove files for packages that no longer exist
for item in os.listdir(pkghistdir):
2015-11-25 09:30:30 +00:00
if item not in preserve:
2012-08-02 09:23:02 +00:00
if item not in packagelist:
2015-08-21 12:55:00 +00:00
itempath = os.path.join(pkghistdir, item)
if os.path.isdir(itempath):
for subfile in os.listdir(itempath):
os.unlink(os.path.join(itempath, subfile))
os.rmdir(itempath)
else:
os.unlink(itempath)
2012-08-02 09:23:02 +00:00
2012-07-19 11:32:20 +00:00
rcpinfo = RecipeInfo(pn)
rcpinfo.pe = pe
rcpinfo.pv = pv
rcpinfo.pr = pr
2014-11-05 19:21:58 +00:00
rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS', True) or ""))
2012-07-19 11:32:20 +00:00
rcpinfo.packages = packages
write_recipehistory(rcpinfo, d)
pkgdest = d.getVar('PKGDEST', True)
2012-08-02 09:23:02 +00:00
for pkg in packagelist:
2013-12-02 18:50:49 +00:00
pkgdata = {}
with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
for line in f.readlines():
item = line.rstrip('\n').split(': ', 1)
2013-12-05 11:11:47 +00:00
key = item[0]
if key.endswith('_' + pkg):
key = key[:-len(pkg)-1]
2014-01-25 21:25:05 +00:00
pkgdata[key] = item[1].decode('utf-8').decode('string_escape')
2013-12-02 18:50:49 +00:00
pkge = pkgdata.get('PKGE', '0')
pkgv = pkgdata['PKGV']
pkgr = pkgdata['PKGR']
2012-07-19 11:32:20 +00:00
#
# Find out what the last version was
# Make sure the version did not decrease
#
2013-08-06 22:08:08 +00:00
lastversion = getlastpkgversion(pkg)
if lastversion:
last_pkge = lastversion.pkge
last_pkgv = lastversion.pkgv
last_pkgr = lastversion.pkgr
r = bb.utils.vercmp((pkge, pkgv, pkgr), (last_pkge, last_pkgv, last_pkgr))
if r < 0:
msg = "Package version for package %s went backwards which would break package feeds from (%s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr)
package_qa_handle_error("version-going-backwards", msg, d)
2012-07-19 11:32:20 +00:00
pkginfo = PackageInfo(pkg)
2012-08-02 09:23:05 +00:00
# Apparently the version can be different on a per-package basis (see Python)
2013-12-02 18:50:49 +00:00
pkginfo.pe = pkgdata.get('PE', '0')
pkginfo.pv = pkgdata['PV']
pkginfo.pr = pkgdata['PR']
2013-12-05 11:11:47 +00:00
pkginfo.pkg = pkgdata['PKG']
2012-08-02 09:23:05 +00:00
pkginfo.pkge = pkge
pkginfo.pkgv = pkgv
pkginfo.pkgr = pkgr
2014-11-05 19:21:58 +00:00
pkginfo.rprovides = sortpkglist(oe.utils.squashspaces(pkgdata.get('RPROVIDES', "")))
pkginfo.rdepends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RDEPENDS', "")))
pkginfo.rrecommends = sortpkglist(oe.utils.squashspaces(pkgdata.get('RRECOMMENDS', "")))
pkginfo.rsuggests = sortpkglist(oe.utils.squashspaces(pkgdata.get('RSUGGESTS', "")))
pkginfo.rreplaces = sortpkglist(oe.utils.squashspaces(pkgdata.get('RREPLACES', "")))
pkginfo.rconflicts = sortpkglist(oe.utils.squashspaces(pkgdata.get('RCONFLICTS', "")))
pkginfo.files = oe.utils.squashspaces(pkgdata.get('FILES', ""))
2012-08-02 09:23:04 +00:00
for filevar in pkginfo.filevars:
2013-12-05 11:11:47 +00:00
pkginfo.filevars[filevar] = pkgdata.get(filevar, "")
2012-07-19 11:32:20 +00:00
# Gather information about packaged files
2013-12-02 18:50:49 +00:00
val = pkgdata.get('FILES_INFO', '')
dictval = json.loads(val)
filelist = dictval.keys()
2012-07-19 11:32:20 +00:00
filelist.sort()
pkginfo.filelist = " ".join(filelist)
2013-12-05 11:11:47 +00:00
pkginfo.size = int(pkgdata['PKGSIZE'])
2013-12-02 18:50:49 +00:00
2012-07-19 11:32:20 +00:00
write_pkghistory(pkginfo, d)
2015-06-25 13:46:47 +00:00
# Create files-in-<package-name>.txt files containing a list of files of each recipe's package
bb.build.exec_func("buildhistory_list_pkg_files", d)
2011-12-01 17:26:44 +00:00
}
def write_recipehistory(rcpinfo, d):
2014-02-06 11:02:55 +00:00
import codecs
2012-07-19 11:32:20 +00:00
bb.debug(2, "Writing recipe history")
2011-12-01 17:26:44 +00:00
2012-07-19 11:32:20 +00:00
pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True)
2011-12-01 17:26:44 +00:00
2012-08-02 09:23:01 +00:00
infofile = os.path.join(pkghistdir, "latest")
2014-02-06 11:02:55 +00:00
with codecs.open(infofile, "w", encoding='utf8') as f:
2012-07-19 11:32:20 +00:00
if rcpinfo.pe != "0":
2014-02-06 11:02:55 +00:00
f.write(u"PE = %s\n" % rcpinfo.pe)
f.write(u"PV = %s\n" % rcpinfo.pv)
f.write(u"PR = %s\n" % rcpinfo.pr)
f.write(u"DEPENDS = %s\n" % rcpinfo.depends)
f.write(u"PACKAGES = %s\n" % rcpinfo.packages)
2011-12-01 17:26:44 +00:00
def write_pkghistory(pkginfo, d):
2014-02-06 11:02:55 +00:00
import codecs
2012-08-02 09:23:01 +00:00
bb.debug(2, "Writing package history for package %s" % pkginfo.name)
2012-07-19 11:32:20 +00:00
pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True)
2012-08-02 09:23:01 +00:00
pkgpath = os.path.join(pkghistdir, pkginfo.name)
if not os.path.exists(pkgpath):
2013-08-06 22:09:49 +00:00
bb.utils.mkdirhier(pkgpath)
2012-07-19 11:32:20 +00:00
2012-08-02 09:23:01 +00:00
infofile = os.path.join(pkgpath, "latest")
2014-02-06 11:02:55 +00:00
with codecs.open(infofile, "w", encoding='utf8') as f:
2012-07-19 11:32:20 +00:00
if pkginfo.pe != "0":
2014-02-06 11:02:55 +00:00
f.write(u"PE = %s\n" % pkginfo.pe)
f.write(u"PV = %s\n" % pkginfo.pv)
f.write(u"PR = %s\n" % pkginfo.pr)
2012-08-02 09:23:05 +00:00
pkgvars = {}
pkgvars['PKG'] = pkginfo.pkg if pkginfo.pkg != pkginfo.name else ''
pkgvars['PKGE'] = pkginfo.pkge if pkginfo.pkge != pkginfo.pe else ''
pkgvars['PKGV'] = pkginfo.pkgv if pkginfo.pkgv != pkginfo.pv else ''
pkgvars['PKGR'] = pkginfo.pkgr if pkginfo.pkgr != pkginfo.pr else ''
for pkgvar in pkgvars:
val = pkgvars[pkgvar]
if val:
2014-02-06 11:02:55 +00:00
f.write(u"%s = %s\n" % (pkgvar, val))
2012-08-02 09:23:05 +00:00
2014-02-06 11:02:55 +00:00
f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides)
f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends)
f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends)
2013-02-08 09:18:20 +00:00
if pkginfo.rsuggests:
2014-02-06 11:02:55 +00:00
f.write(u"RSUGGESTS = %s\n" % pkginfo.rsuggests)
2013-02-08 09:18:20 +00:00
if pkginfo.rreplaces:
2014-02-06 11:02:55 +00:00
f.write(u"RREPLACES = %s\n" % pkginfo.rreplaces)
2013-02-08 09:18:20 +00:00
if pkginfo.rconflicts:
2014-02-06 11:02:55 +00:00
f.write(u"RCONFLICTS = %s\n" % pkginfo.rconflicts)
f.write(u"PKGSIZE = %d\n" % pkginfo.size)
f.write(u"FILES = %s\n" % pkginfo.files)
f.write(u"FILELIST = %s\n" % pkginfo.filelist)
2012-08-02 09:23:04 +00:00
for filevar in pkginfo.filevars:
filevarpath = os.path.join(pkgpath, "latest.%s" % filevar)
val = pkginfo.filevars[filevar]
if val:
2014-02-06 11:02:55 +00:00
with codecs.open(filevarpath, "w", encoding='utf8') as f:
2012-08-02 09:23:04 +00:00
f.write(val)
else:
if os.path.exists(filevarpath):
os.unlink(filevarpath)
2011-12-01 17:26:44 +00:00
2014-03-18 11:19:20 +00:00
#
# rootfs_type can be: image, sdk_target, sdk_host
#
def buildhistory_list_installed(d, rootfs_type="image"):
from oe.rootfs import image_list_installed_packages
from oe.sdk import sdk_list_installed_packages
2014-02-19 10:13:09 +00:00
2014-03-18 11:19:20 +00:00
process_list = [('file', 'bh_installed_pkgs.txt'),\
('deps', 'bh_installed_pkgs_deps.txt')]
2014-02-19 10:13:09 +00:00
2014-03-18 11:19:20 +00:00
for output_type, output_file in process_list:
output_file_full = os.path.join(d.getVar('WORKDIR', True), output_file)
2014-03-05 12:02:51 +00:00
2014-03-18 11:19:20 +00:00
with open(output_file_full, 'w') as output:
if rootfs_type == "image":
output.write(image_list_installed_packages(d, output_type))
else:
output.write(sdk_list_installed_packages(d, rootfs_type == "sdk_target", output_type))
2014-03-05 12:02:51 +00:00
2014-03-18 11:19:20 +00:00
python buildhistory_list_installed_image() {
buildhistory_list_installed(d)
2014-02-19 10:13:09 +00:00
}
2014-03-18 11:19:20 +00:00
python buildhistory_list_installed_sdk_target() {
buildhistory_list_installed(d, "sdk_target")
}
python buildhistory_list_installed_sdk_host() {
buildhistory_list_installed(d, "sdk_host")
}
2011-12-01 17:26:44 +00:00
2013-03-22 19:53:52 +00:00
buildhistory_get_installed() {
mkdir -p $1
2011-11-30 16:48:47 +00:00
# Get list of installed packages
2013-03-22 19:53:52 +00:00
pkgcache="$1/installed-packages.tmp"
2014-02-19 10:13:09 +00:00
cat ${WORKDIR}/bh_installed_pkgs.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs.txt
2012-07-23 06:59:11 +00:00
2013-03-22 19:53:52 +00:00
cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
2013-06-19 16:53:06 +00:00
if [ -s $pkgcache ] ; then
cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt
else
printf "" > $1/installed-packages.txt
fi
2012-07-23 06:59:11 +00:00
# Produce dependency graph
2013-08-15 17:04:38 +00:00
# First, quote each name to handle characters that cause issues for dot
2014-05-06 12:17:47 +00:00
sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps.txt > $1/depends.tmp && \
2014-03-05 12:02:51 +00:00
rm ${WORKDIR}/bh_installed_pkgs_deps.txt
2012-07-23 06:59:11 +00:00
# Change delimiter from pipe to -> and set style for recommend lines
2013-08-15 17:04:38 +00:00
sed -i -e 's:|: -> :' -e 's:"\[REC\]":[style=dotted]:' -e 's:$:;:' $1/depends.tmp
2012-07-23 06:59:11 +00:00
# Add header, sorted and de-duped contents and footer and then delete the temp file
2013-03-22 19:53:52 +00:00
printf "digraph depends {\n node [shape=plaintext]\n" > $1/depends.dot
cat $1/depends.tmp | sort | uniq >> $1/depends.dot
echo "}" >> $1/depends.dot
rm $1/depends.tmp
2011-11-30 16:48:47 +00:00
2012-07-23 06:59:11 +00:00
# Produce installed package sizes list
2013-03-22 19:53:52 +00:00
printf "" > $1/installed-package-sizes.tmp
2013-08-15 17:04:35 +00:00
cat $pkgcache | while read pkg pkgfile pkgarch
2012-07-23 06:59:11 +00:00
do
2015-02-05 15:08:22 +00:00
size=`oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" ${pkg}_${pkgarch}`
2013-12-05 01:08:49 +00:00
if [ "$size" != "" ] ; then
echo "$size $pkg" >> $1/installed-package-sizes.tmp
fi
2012-07-23 06:59:11 +00:00
done
2013-03-22 19:53:52 +00:00
cat $1/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > $1/installed-package-sizes.txt
rm $1/installed-package-sizes.tmp
2011-11-30 16:48:47 +00:00
2012-07-23 06:59:11 +00:00
# We're now done with the cache, delete it
rm $pkgcache
2013-03-22 19:53:52 +00:00
if [ "$2" != "sdk" ] ; then
# Produce some cut-down graphs (for readability)
2013-08-15 17:04:38 +00:00
grep -v kernel_image $1/depends.dot | grep -v kernel-2 | grep -v kernel-3 > $1/depends-nokernel.dot
2013-03-22 19:53:52 +00:00
grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot
2013-08-15 17:04:38 +00:00
grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot
grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot
2013-03-22 19:53:52 +00:00
fi
2013-01-18 12:41:48 +00:00
# add complementary package information
if [ -e ${WORKDIR}/complementary_pkgs.txt ]; then
2013-03-22 19:53:52 +00:00
cp ${WORKDIR}/complementary_pkgs.txt $1
2013-01-18 12:41:48 +00:00
fi
2011-11-30 16:48:47 +00:00
}
2013-03-22 19:53:52 +00:00
buildhistory_get_image_installed() {
# Anything requiring the use of the packaging system should be done in here
# in case the packaging files are going to be removed for this image
2014-04-24 18:59:20 +00:00
if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
2012-03-16 11:37:22 +00:00
return
fi
2013-03-22 19:53:52 +00:00
buildhistory_get_installed ${BUILDHISTORY_DIR_IMAGE}
}
buildhistory_get_sdk_installed() {
# Anything requiring the use of the packaging system should be done in here
# in case the packaging files are going to be removed for this SDK
2014-04-24 18:59:20 +00:00
if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
2013-03-22 19:53:52 +00:00
return
fi
buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk
}
2014-02-13 13:49:14 +00:00
buildhistory_get_sdk_installed_host() {
buildhistory_get_sdk_installed host
}
buildhistory_get_sdk_installed_target() {
buildhistory_get_sdk_installed target
}
2013-03-22 19:53:52 +00:00
buildhistory_list_files() {
# List the files in the specified directory, but exclude date/time etc.
2011-11-30 16:48:47 +00:00
# This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo
2015-08-26 13:21:40 +00:00
if [ "$3" = "fakeroot" ] ; then
( cd $1 && ${FAKEROOTENV} ${FAKEROOTCMD} find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 )
else
( cd $1 && find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 )
fi
2013-03-22 19:53:52 +00:00
}
2015-06-25 13:46:47 +00:00
buildhistory_list_pkg_files() {
2015-08-20 10:55:35 +00:00
# Create individual files-in-package for each recipe's package
for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do
2015-08-21 09:50:57 +00:00
pkgname=$(basename $pkgdir)
outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname"
outfile="$outfolder/files-in-package.txt"
# Make sure the output folder exists so we can create the file
if [ ! -d $outfolder ] ; then
bbdebug 2 "Folder $outfolder does not exist, file $outfile not created"
2015-08-20 10:55:35 +00:00
continue
fi
2015-08-26 13:21:40 +00:00
buildhistory_list_files $pkgdir $outfile fakeroot
2015-08-20 10:55:35 +00:00
done
2015-06-25 13:46:47 +00:00
}
2013-03-22 19:53:52 +00:00
buildhistory_get_imageinfo() {
2014-04-24 18:59:20 +00:00
if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
2013-03-22 19:53:52 +00:00
return
fi
buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt
2011-11-30 16:48:47 +00:00
2013-05-09 15:57:48 +00:00
# Collect files requested in BUILDHISTORY_IMAGE_FILES
rm -rf ${BUILDHISTORY_DIR_IMAGE}/image-files
for f in ${BUILDHISTORY_IMAGE_FILES}; do
if [ -f ${IMAGE_ROOTFS}/$f ] ; then
mkdir -p ${BUILDHISTORY_DIR_IMAGE}/image-files/`dirname $f`
cp ${IMAGE_ROOTFS}/$f ${BUILDHISTORY_DIR_IMAGE}/image-files/$f
fi
done
2012-01-19 10:32:09 +00:00
# Record some machine-readable meta-information about the image
2012-09-20 09:56:59 +00:00
printf "" > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
2012-01-19 10:32:09 +00:00
cat >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt <<END
${@buildhistory_get_imagevars(d)}
END
imagesize=`du -ks ${IMAGE_ROOTFS} | awk '{ print $1 }'`
echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
2011-11-30 16:48:47 +00:00
# Add some configuration information
2014-07-25 13:15:38 +00:00
echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id.txt
2011-12-23 16:23:57 +00:00
2014-07-25 13:15:38 +00:00
cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id.txt <<END
${@buildhistory_get_build_id(d)}
2011-12-23 16:23:57 +00:00
END
2011-11-30 16:48:47 +00:00
}
2013-03-22 19:53:52 +00:00
buildhistory_get_sdkinfo() {
2014-04-24 18:59:20 +00:00
if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
2013-03-22 19:53:52 +00:00
return
fi
buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt
# Record some machine-readable meta-information about the SDK
printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END
${@buildhistory_get_sdkvars(d)}
END
sdksize=`du -ks ${SDK_OUTPUT} | awk '{ print $1 }'`
echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
}
2015-01-26 14:40:40 +00:00
# By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of
# unneeded packages but before the removal of packaging files
ROOTFS_POSTUNINSTALL_COMMAND += " buildhistory_list_installed_image ;\
2014-02-19 10:13:09 +00:00
buildhistory_get_image_installed ; "
2011-11-30 16:48:47 +00:00
IMAGE_POSTPROCESS_COMMAND += " buildhistory_get_imageinfo ; "
2013-03-27 01:58:16 +00:00
# We want these to be the last run so that we get called after complementary package installation
2014-03-18 11:19:20 +00:00
POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target ;\
2014-02-19 10:13:09 +00:00
buildhistory_get_sdk_installed_target ; "
2014-03-18 11:19:20 +00:00
POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_list_installed_sdk_host ;\
2014-02-19 10:13:09 +00:00
buildhistory_get_sdk_installed_host ; "
2013-03-22 19:53:52 +00:00
2015-12-20 20:04:59 +00:00
SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; "
2013-03-22 19:53:52 +00:00
2014-07-25 13:15:38 +00:00
def buildhistory_get_build_id(d):
2014-08-24 16:22:46 +00:00
if d.getVar('BB_WORKERCONTEXT', True) != '1':
return ""
2014-07-25 13:15:38 +00:00
localdata = bb.data.createCopy(d)
bb.data.update_data(localdata)
statuslines = []
for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata):
g = globals()
if func not in g:
bb.warn("Build configuration function '%s' does not exist" % func)
else:
flines = g[func](localdata)
if flines:
statuslines.extend(flines)
statusheader = d.getVar('BUILDCFG_HEADER', True)
return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines)))
2011-11-30 16:48:47 +00:00
2012-08-02 09:23:09 +00:00
def buildhistory_get_metadata_revs(d):
# We want an easily machine-readable format here, so get_layers_branch_rev isn't quite what we want
layers = (d.getVar("BBLAYERS", True) or "").split()
medadata_revs = ["%-17s = %s:%s" % (os.path.basename(i), \
base_get_metadata_git_branch(i, None).strip(), \
base_get_metadata_git_revision(i, None)) \
for i in layers]
return '\n'.join(medadata_revs)
2013-03-22 19:53:52 +00:00
def outputvars(vars, listvars, d):
vars = vars.split()
2012-07-19 11:32:20 +00:00
listvars = listvars.split()
ret = ""
2013-03-22 19:53:52 +00:00
for var in vars:
2012-07-19 11:32:20 +00:00
value = d.getVar(var, True) or ""
if var in listvars:
# Squash out spaces
2014-11-05 19:21:58 +00:00
value = oe.utils.squashspaces(value)
2012-07-19 11:32:20 +00:00
ret += "%s = %s\n" % (var, value)
return ret.rstrip('\n')
2012-01-19 10:32:09 +00:00
2013-03-22 19:53:52 +00:00
def buildhistory_get_imagevars(d):
2013-12-02 18:50:50 +00:00
if d.getVar('BB_WORKERCONTEXT', True) != '1':
return ""
2013-12-02 18:50:48 +00:00
imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND"
listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE"
2013-03-22 19:53:52 +00:00
return outputvars(imagevars, listvars, d)
def buildhistory_get_sdkvars(d):
2013-12-02 18:50:50 +00:00
if d.getVar('BB_WORKERCONTEXT', True) != '1':
return ""
2013-12-02 18:50:48 +00:00
sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE"
listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE"
2013-03-22 19:53:52 +00:00
return outputvars(sdkvars, listvars, d)
2012-01-19 10:32:09 +00:00
2013-08-15 17:04:36 +00:00
def buildhistory_get_cmdline(d):
if sys.argv[0].endswith('bin/bitbake'):
bincmd = 'bitbake'
else:
bincmd = sys.argv[0]
return '%s %s' % (bincmd, ' '.join(sys.argv[1:]))
2015-08-26 09:54:17 +00:00
buildhistory_single_commit() {
if [ "$3" = "" ] ; then
commitopts="${BUILDHISTORY_DIR}/ --allow-empty"
item="No changes"
else
commitopts="$3 metadata-revs"
item="$3"
fi
2015-08-26 10:05:18 +00:00
if [ "${BUILDHISTORY_BUILD_FAILURES}" = "0" ] ; then
result="succeeded"
else
result="failed"
fi
case ${BUILDHISTORY_BUILD_INTERRUPTED} in
1)
result="$result (interrupted)"
;;
2)
result="$result (force interrupted)"
;;
esac
2015-08-26 09:54:17 +00:00
commitmsgfile=`mktemp`
cat > $commitmsgfile << END
$item: Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $2
cmd: $1
2015-08-26 10:05:18 +00:00
result: $result
2015-08-26 09:54:17 +00:00
metadata revisions:
END
cat ${BUILDHISTORY_DIR}/metadata-revs >> $commitmsgfile
git commit $commitopts -F $commitmsgfile --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null
rm $commitmsgfile
}
2011-11-30 16:48:47 +00:00
buildhistory_commit() {
2011-12-30 13:08:01 +00:00
if [ ! -d ${BUILDHISTORY_DIR} ] ; then
# Code above that creates this dir never executed, so there can't be anything to commit
2012-03-16 11:37:22 +00:00
return
2011-12-30 13:08:01 +00:00
fi
2012-08-02 09:23:09 +00:00
# Create a machine-readable list of metadata revisions for each layer
cat > ${BUILDHISTORY_DIR}/metadata-revs <<END
${@buildhistory_get_metadata_revs(d)}
END
2011-11-30 16:48:47 +00:00
( cd ${BUILDHISTORY_DIR}/
2011-12-30 13:08:01 +00:00
# Initialise the repo if necessary
if [ ! -d .git ] ; then
git init -q
2013-08-15 17:04:37 +00:00
else
git tag -f build-minus-3 build-minus-2 > /dev/null 2>&1 || true
git tag -f build-minus-2 build-minus-1 > /dev/null 2>&1 || true
git tag -f build-minus-1 > /dev/null 2>&1 || true
2011-12-30 13:08:01 +00:00
fi
2015-05-21 12:16:34 +00:00
# If the user hasn't set up their name/email, set some defaults
# just for this repo (otherwise the commit will fail with older
# versions of git)
if ! git config user.email > /dev/null ; then
git config --local user.email "buildhistory@${DISTRO}"
fi
if ! git config user.name > /dev/null ; then
git config --local user.name "buildhistory"
fi
2012-08-02 09:23:09 +00:00
# Check if there are new/changed files to commit (other than metadata-revs)
repostatus=`git status --porcelain | grep -v " metadata-revs$"`
2012-10-11 21:46:41 +00:00
HOSTNAME=`hostname 2>/dev/null || echo unknown`
2013-08-15 17:04:36 +00:00
CMDLINE="${@buildhistory_get_cmdline(d)}"
2011-12-30 13:08:01 +00:00
if [ "$repostatus" != "" ] ; then
2013-05-31 11:09:14 +00:00
git add -A .
2012-03-31 22:06:44 +00:00
# porcelain output looks like "?? packages/foo/bar"
2012-08-02 09:23:09 +00:00
# Ensure we commit metadata-revs with the first commit
2012-05-04 13:36:01 +00:00
for entry in `echo "$repostatus" | awk '{print $2}' | awk -F/ '{print $1}' | sort | uniq` ; do
2015-08-26 09:54:17 +00:00
buildhistory_single_commit "$CMDLINE" "$HOSTNAME" "$entry"
2012-03-31 22:06:44 +00:00
done
2013-12-18 16:20:03 +00:00
git gc --auto --quiet
2012-03-31 22:06:43 +00:00
else
2015-08-26 09:54:17 +00:00
buildhistory_single_commit "$CMDLINE" "$HOSTNAME"
2015-08-26 14:01:45 +00:00
fi
if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then
git push -q ${BUILDHISTORY_PUSH_REPO}
2011-11-30 16:48:47 +00:00
fi) || true
}
python buildhistory_eventhandler() {
2013-06-12 16:31:57 +00:00
if e.data.getVar('BUILDHISTORY_FEATURES', True).strip():
if e.data.getVar("BUILDHISTORY_COMMIT", True) == "1":
2013-08-02 12:15:07 +00:00
bb.note("Writing buildhistory")
2015-08-26 10:05:18 +00:00
localdata = bb.data.createCopy(e.data)
localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
interrupted = getattr(e, '_interrupted', 0)
localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
bb.build.exec_func("buildhistory_commit", localdata)
2011-11-30 16:48:47 +00:00
}
addhandler buildhistory_eventhandler
2013-06-12 16:31:57 +00:00
buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted"
2013-03-27 18:05:50 +00:00
# FIXME this ought to be moved into the fetcher
def _get_srcrev_values(d):
"""
Return the version strings for the current recipe
"""
scms = []
fetcher = bb.fetch.Fetch(d.getVar('SRC_URI', True).split(), d)
urldata = fetcher.ud
for u in urldata:
if urldata[u].method.supports_srcrev():
scms.append(u)
autoinc_templ = 'AUTOINC+'
2013-04-03 08:49:36 +00:00
dict_srcrevs = {}
dict_tag_srcrevs = {}
2013-03-27 18:05:50 +00:00
for scm in scms:
ud = urldata[scm]
for name in ud.names:
2013-11-20 14:44:11 +00:00
try:
rev = ud.method.sortable_revision(ud, d, name)
except TypeError:
# support old bitbake versions
rev = ud.method.sortable_revision(scm, ud, d, name)
2013-05-23 10:40:40 +00:00
# Clean this up when we next bump bitbake version
if type(rev) != str:
autoinc, rev = rev
elif rev.startswith(autoinc_templ):
2013-03-27 18:05:50 +00:00
rev = rev[len(autoinc_templ):]
2013-04-03 08:49:36 +00:00
dict_srcrevs[name] = rev
if 'tag' in ud.parm:
tag = ud.parm['tag'];
key = name+'_'+tag
dict_tag_srcrevs[key] = rev
return (dict_srcrevs, dict_tag_srcrevs)
2013-03-27 18:05:50 +00:00
2013-04-09 08:39:24 +00:00
do_fetch[postfuncs] += "write_srcrev"
2014-02-11 16:10:11 +00:00
do_fetch[vardepsexclude] += "write_srcrev"
2013-04-09 08:39:24 +00:00
python write_srcrev() {
2013-03-27 18:05:50 +00:00
pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True)
srcrevfile = os.path.join(pkghistdir, 'latest_srcrev')
2013-04-03 08:49:36 +00:00
srcrevs, tag_srcrevs = _get_srcrev_values(d)
2013-03-27 18:05:50 +00:00
if srcrevs:
if not os.path.exists(pkghistdir):
2013-08-06 22:09:49 +00:00
bb.utils.mkdirhier(pkghistdir)
2013-04-03 08:49:36 +00:00
old_tag_srcrevs = {}
if os.path.exists(srcrevfile):
with open(srcrevfile) as f:
for line in f:
if line.startswith('# tag_'):
key, value = line.split("=", 1)
key = key.replace('# tag_', '').strip()
value = value.replace('"', '').strip()
old_tag_srcrevs[key] = value
2013-03-27 18:05:50 +00:00
with open(srcrevfile, 'w') as f:
orig_srcrev = d.getVar('SRCREV', False) or 'INVALID'
if orig_srcrev != 'INVALID':
f.write('# SRCREV = "%s"\n' % orig_srcrev)
if len(srcrevs) > 1:
for name, srcrev in srcrevs.items():
orig_srcrev = d.getVar('SRCREV_%s' % name, False)
if orig_srcrev:
f.write('# SRCREV_%s = "%s"\n' % (name, orig_srcrev))
f.write('SRCREV_%s = "%s"\n' % (name, srcrev))
else:
f.write('SRCREV = "%s"\n' % srcrevs.itervalues().next())
2013-04-03 08:49:36 +00:00
if len(tag_srcrevs) > 0:
for name, srcrev in tag_srcrevs.items():
f.write('# tag_%s = "%s"\n' % (name, srcrev))
if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev:
pkg = d.getVar('PN', True)
bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev))
2013-03-27 18:05:50 +00:00
else:
if os.path.exists(srcrevfile):
os.remove(srcrevfile)
}