scripts/oe-selftest: script to run builds as unittest against bitbake or various scripts
The purpose of oe-selftest is to run unittest modules added from meta/lib/oeqa/selftest, which are tests against bitbake tools. Right now the script it's useful for simple tests like: - "bitbake --someoption, change some metadata, bitbake X, check something" type scenarios (PR service, error output, etc) - or "bitbake-layers <...>" type scripts and yocto-bsp tools. This commit also adds some helper modules that the tests will use and a base class. Also, most of the tests will have a dependency on a meta-selftest layer which contains specially modified recipes/bbappends/include files for the purpose of the tests. The tests themselves will usually write to ".inc" files from the layer or in conf/selftest.inc (which is added as an include in local.conf at the start and removed at the end) It's a simple matter or sourcing the enviroment, adding the meta-selftest layer to bblayers.conf and running: oe-selftest to get some results. It would finish faster if at least a core-image-minimal was built before. [ YOCTO #4740 ] (From OE-Core rev: 41a4f8fb005328d3a631a9036ceb6dcf75754410) Signed-off-by: Stefan Stanacar <stefanx.stanacar@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Stefan Stanacar
Richard Purdie
parent
1fa51bf949
commit
645dd61cd2
|
|
@ -0,0 +1,2 @@
|
|||
from pkgutil import extend_path
|
||||
__path__ = extend_path(__path__, __name__)
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
# Copyright (c) 2013 Intel Corporation
|
||||
#
|
||||
# Released under the MIT license (see COPYING.MIT)
|
||||
|
||||
|
||||
# DESCRIPTION
|
||||
# Base class inherited by test classes in meta/lib/selftest
|
||||
|
||||
import unittest
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import errno
|
||||
|
||||
import oeqa.utils.ftools as ftools
|
||||
|
||||
|
||||
class oeSelfTest(unittest.TestCase):
|
||||
|
||||
log = logging.getLogger("selftest.base")
|
||||
longMessage = True
|
||||
|
||||
def __init__(self, methodName="runTest"):
|
||||
self.builddir = os.environ.get("BUILDDIR")
|
||||
self.localconf_path = os.path.join(self.builddir, "conf/local.conf")
|
||||
self.testinc_path = os.path.join(self.builddir, "conf/selftest.inc")
|
||||
self.testlayer_path = oeSelfTest.testlayer_path
|
||||
super(oeSelfTest, self).__init__(methodName)
|
||||
|
||||
def setUp(self):
|
||||
os.chdir(self.builddir)
|
||||
# we don't know what the previous test left around in config or inc files
|
||||
# if it failed so we need a fresh start
|
||||
try:
|
||||
os.remove(self.testinc_path)
|
||||
except OSError as e:
|
||||
if e.errno != errno.ENOENT:
|
||||
raise
|
||||
for root, _, files in os.walk(self.testlayer_path):
|
||||
for f in files:
|
||||
if f == 'test_recipe.inc':
|
||||
os.remove(os.path.join(root, f))
|
||||
# tests might need their own setup
|
||||
# but if they overwrite this one they have to call
|
||||
# super each time, so let's give them an alternative
|
||||
self.setUpLocal()
|
||||
|
||||
def setUpLocal(self):
|
||||
pass
|
||||
|
||||
def tearDown(self):
|
||||
self.tearDownLocal()
|
||||
|
||||
def tearDownLocal(self):
|
||||
pass
|
||||
|
||||
# write to <builddir>/conf/selftest.inc
|
||||
def write_config(self, data):
|
||||
self.log.debug("Writing to: %s\n%s\n" % (self.testinc_path, data))
|
||||
ftools.write_file(self.testinc_path, data)
|
||||
|
||||
# append to <builddir>/conf/selftest.inc
|
||||
def append_config(self, data):
|
||||
self.log.debug("Appending to: %s\n%s\n" % (self.testinc_path, data))
|
||||
ftools.append_file(self.testinc_path, data)
|
||||
|
||||
# remove data from <builddir>/conf/selftest.inc
|
||||
def remove_config(self, data):
|
||||
self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_path, data))
|
||||
ftools.remove_from_file(self.testinc_path, data)
|
||||
|
||||
# write to meta-sefltest/recipes-test/<recipe>/test_recipe.inc
|
||||
def write_recipeinc(self, recipe, data):
|
||||
inc_file = os.path.join(self.testlayer_path, 'recipes-test', recipe, 'test_recipe.inc')
|
||||
self.log.debug("Writing to: %s\n%s\n" % (inc_file, data))
|
||||
ftools.write_file(inc_file, data)
|
||||
|
||||
# append data to meta-sefltest/recipes-test/<recipe>/test_recipe.inc
|
||||
def append_recipeinc(self, recipe, data):
|
||||
inc_file = os.path.join(self.testlayer_path, 'recipes-test', recipe, 'test_recipe.inc')
|
||||
self.log.debug("Appending to: %s\n%s\n" % (inc_file, data))
|
||||
ftools.append_file(inc_file, data)
|
||||
|
||||
# remove data from meta-sefltest/recipes-test/<recipe>/test_recipe.inc
|
||||
def remove_recipeinc(self, recipe, data):
|
||||
inc_file = os.path.join(self.testlayer_path, 'recipes-test', recipe, 'test_recipe.inc')
|
||||
self.log.debug("Removing from: %s\n%s\n" % (inc_file, data))
|
||||
ftools.remove_from_file(inc_file, data)
|
||||
|
||||
# delete meta-sefltest/recipes-test/<recipe>/test_recipe.inc file
|
||||
def delete_recipeinc(self, recipe):
|
||||
inc_file = os.path.join(self.testlayer_path, 'recipes-test', recipe, 'test_recipe.inc')
|
||||
self.log.debug("Deleting file: %s" % inc_file)
|
||||
try:
|
||||
os.remove(self.testinc_path)
|
||||
except OSError as e:
|
||||
if e.errno != errno.ENOENT:
|
||||
raise
|
||||
|
|
@ -0,0 +1,137 @@
|
|||
# Copyright (c) 2013 Intel Corporation
|
||||
#
|
||||
# Released under the MIT license (see COPYING.MIT)
|
||||
|
||||
# DESCRIPTION
|
||||
# This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest
|
||||
# It provides a class and methods for running commands on the host in a convienent way for tests.
|
||||
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
import logging
|
||||
|
||||
class Command(object):
|
||||
def __init__(self, command, bg=False, timeout=None, data=None, **options):
|
||||
|
||||
self.defaultopts = {
|
||||
"stdout": subprocess.PIPE,
|
||||
"stderr": subprocess.STDOUT,
|
||||
"stdin": None,
|
||||
"shell": False,
|
||||
"bufsize": -1,
|
||||
}
|
||||
|
||||
self.cmd = command
|
||||
self.bg = bg
|
||||
self.timeout = timeout
|
||||
self.data = data
|
||||
|
||||
self.options = dict(self.defaultopts)
|
||||
if isinstance(self.cmd, basestring):
|
||||
self.options["shell"] = True
|
||||
if self.data:
|
||||
self.options['stdin'] = subprocess.PIPE
|
||||
self.options.update(options)
|
||||
|
||||
self.status = None
|
||||
self.output = None
|
||||
self.error = None
|
||||
self.thread = None
|
||||
|
||||
self.log = logging.getLogger("utils.commands")
|
||||
|
||||
def run(self):
|
||||
self.process = subprocess.Popen(self.cmd, **self.options)
|
||||
|
||||
def commThread():
|
||||
self.output, self.error = self.process.communicate(self.data)
|
||||
|
||||
self.thread = threading.Thread(target=commThread)
|
||||
self.thread.start()
|
||||
|
||||
self.log.debug("Running command '%s'" % self.cmd)
|
||||
|
||||
if not self.bg:
|
||||
self.thread.join(self.timeout)
|
||||
self.stop()
|
||||
|
||||
def stop(self):
|
||||
if self.thread.isAlive():
|
||||
self.process.terminate()
|
||||
# let's give it more time to terminate gracefully before killing it
|
||||
self.thread.join(5)
|
||||
if self.thread.isAlive():
|
||||
self.process.kill()
|
||||
self.thread.join()
|
||||
|
||||
self.output = self.output.rstrip()
|
||||
self.status = self.process.poll()
|
||||
|
||||
self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status))
|
||||
# logging the complete output is insane
|
||||
# bitbake -e output is really big
|
||||
# and makes the log file useless
|
||||
if self.status:
|
||||
lout = "\n".join(self.output.splitlines()[-20:])
|
||||
self.log.debug("Last 20 lines:\n%s" % lout)
|
||||
|
||||
|
||||
class Result(object):
|
||||
pass
|
||||
|
||||
def runCmd(command, ignore_status=False, timeout=None, **options):
|
||||
|
||||
result = Result()
|
||||
|
||||
cmd = Command(command, timeout=timeout, **options)
|
||||
cmd.run()
|
||||
|
||||
result.command = command
|
||||
result.status = cmd.status
|
||||
result.output = cmd.output
|
||||
result.pid = cmd.process.pid
|
||||
|
||||
if result.status and not ignore_status:
|
||||
raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, result.output))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def bitbake(command, ignore_status=False, timeout=None, **options):
|
||||
if isinstance(command, basestring):
|
||||
cmd = "bitbake " + command
|
||||
else:
|
||||
cmd = [ "bitbake" ] + command
|
||||
|
||||
return runCmd(cmd, ignore_status, timeout, **options)
|
||||
|
||||
|
||||
def get_bb_env(target=None):
|
||||
if target:
|
||||
return runCmd("bitbake -e %s" % target).output
|
||||
else:
|
||||
return runCmd("bitbake -e").output
|
||||
|
||||
def get_bb_var(var, target=None):
|
||||
val = None
|
||||
bbenv = get_bb_env(target)
|
||||
for line in bbenv.splitlines():
|
||||
if line.startswith(var + "="):
|
||||
val = line.split('=')[1]
|
||||
val = val.replace('\"','')
|
||||
break
|
||||
return val
|
||||
|
||||
def get_test_layer():
|
||||
layers = get_bb_var("BBLAYERS").split()
|
||||
testlayer = None
|
||||
for l in layers:
|
||||
if "/meta-selftest" in l and os.path.isdir(l):
|
||||
testlayer = l
|
||||
break
|
||||
return testlayer
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
import os
|
||||
import re
|
||||
|
||||
def write_file(path, data):
|
||||
wdata = data.rstrip() + "\n"
|
||||
with open(path, "w") as f:
|
||||
f.write(wdata)
|
||||
|
||||
def append_file(path, data):
|
||||
wdata = data.rstrip() + "\n"
|
||||
with open(path, "a") as f:
|
||||
f.write(wdata)
|
||||
|
||||
def read_file(path):
|
||||
data = None
|
||||
with open(path) as f:
|
||||
data = f.read()
|
||||
return data
|
||||
|
||||
def remove_from_file(path, data):
|
||||
lines = read_file(path).splitlines()
|
||||
rmdata = data.strip().splitlines()
|
||||
for l in rmdata:
|
||||
for c in range(0, lines.count(l)):
|
||||
i = lines.index(l)
|
||||
del(lines[i])
|
||||
write_file(path, "\n".join(lines))
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# Copyright (c) 2013 Intel Corporation
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
# DESCRIPTION
|
||||
# This script runs tests defined in meta/lib/selftest/
|
||||
# It's purpose is to automate the testing of different bitbake tools.
|
||||
# To use it you just need to source your build environment setup script and
|
||||
# add the meta-selftest layer to your BBLAYERS.
|
||||
# Call the script as: "oe-selftest" to run all the tests in in meta/lib/selftest/
|
||||
# Call the script as: "oe-selftest <module>.<Class>.<method>" to run just a single test
|
||||
# E.g: "oe-selftest bboutput.BitbakeLayers" will run just the BitbakeLayers class from meta/lib/selftest/bboutput.py
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
import logging
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'meta/lib')))
|
||||
|
||||
import oeqa.selftest
|
||||
import oeqa.utils.ftools as ftools
|
||||
from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
|
||||
from oeqa.selftest.base import oeSelfTest
|
||||
|
||||
def logger_create():
|
||||
log = logging.getLogger("selftest")
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
fh = logging.FileHandler(filename='oe-selftest.log', mode='w')
|
||||
fh.setLevel(logging.DEBUG)
|
||||
|
||||
ch = logging.StreamHandler(sys.stdout)
|
||||
ch.setLevel(logging.INFO)
|
||||
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
fh.setFormatter(formatter)
|
||||
ch.setFormatter(formatter)
|
||||
|
||||
log.addHandler(fh)
|
||||
log.addHandler(ch)
|
||||
|
||||
return log
|
||||
|
||||
log = logger_create()
|
||||
|
||||
def preflight_check():
|
||||
|
||||
log.info("Checking that everything is in order before running the tests")
|
||||
|
||||
if not os.environ.get("BUILDDIR"):
|
||||
log.error("BUILDDIR isn't set. Did you forget to source your build environment setup script?")
|
||||
return False
|
||||
|
||||
builddir = os.environ.get("BUILDDIR")
|
||||
if os.getcwd() != builddir:
|
||||
log.info("Changing cwd to %s" % builddir)
|
||||
os.chdir(builddir)
|
||||
|
||||
if not "meta-selftest" in get_bb_var("BBLAYERS"):
|
||||
log.error("You don't seem to have the meta-selftest layer in BBLAYERS")
|
||||
return False
|
||||
|
||||
log.info("Running bitbake -p")
|
||||
runCmd("bitbake -p")
|
||||
|
||||
return True
|
||||
|
||||
def add_include():
|
||||
builddir = os.environ.get("BUILDDIR")
|
||||
if "#include added by oe-selftest.py" \
|
||||
not in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
|
||||
log.info("Adding: \"include selftest.inc\" in local.conf")
|
||||
ftools.append_file(os.path.join(builddir, "conf/local.conf"), \
|
||||
"\n#include added by oe-selftest.py\ninclude selftest.inc")
|
||||
|
||||
|
||||
def remove_include():
|
||||
builddir = os.environ.get("BUILDDIR")
|
||||
if "#include added by oe-selftest.py" \
|
||||
in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
|
||||
log.info("Removing the include from local.conf")
|
||||
ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \
|
||||
"#include added by oe-selftest.py\ninclude selftest.inc")
|
||||
|
||||
def get_tests():
|
||||
testslist = []
|
||||
for x in sys.argv[1:]:
|
||||
testslist.append('oeqa.selftest.' + x)
|
||||
if not testslist:
|
||||
testpath = os.path.abspath(os.path.dirname(oeqa.selftest.__file__))
|
||||
files = sorted([f for f in os.listdir(testpath) if f.endswith('.py') and not f.startswith('_') and f != 'base.py'])
|
||||
for f in files:
|
||||
module = 'oeqa.selftest.' + f[:-3]
|
||||
testslist.append(module)
|
||||
|
||||
return testslist
|
||||
|
||||
def main():
|
||||
if not preflight_check():
|
||||
return 1
|
||||
|
||||
testslist = get_tests()
|
||||
suite = unittest.TestSuite()
|
||||
loader = unittest.TestLoader()
|
||||
loader.sortTestMethodsUsing = None
|
||||
runner = unittest.TextTestRunner(verbosity=2)
|
||||
# we need to do this here, otherwise just loading the tests
|
||||
# will take 2 minutes (bitbake -e calls)
|
||||
oeSelfTest.testlayer_path = get_test_layer()
|
||||
for test in testslist:
|
||||
log.info("Loading tests from: %s" % test)
|
||||
try:
|
||||
suite.addTests(loader.loadTestsFromName(test))
|
||||
except AttributeError as e:
|
||||
log.error("Failed to import %s" % test)
|
||||
log.error(e)
|
||||
return 1
|
||||
add_include()
|
||||
result = runner.run(suite)
|
||||
log.info("Finished")
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
ret = main()
|
||||
except Exception:
|
||||
ret = 1
|
||||
import traceback
|
||||
traceback.print_exc(5)
|
||||
finally:
|
||||
remove_include()
|
||||
sys.exit(ret)
|
||||
Loading…
Reference in New Issue