2016-05-20 10:57:44 +00:00
#!/usr/bin/env python3
2013-11-27 17:08:50 +00:00
# Copyright (c) 2013 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
# DESCRIPTION
2016-04-15 17:30:06 +00:00
# This script runs tests defined in meta/lib/oeqa/selftest/
2013-11-27 17:08:50 +00:00
# It's purpose is to automate the testing of different bitbake tools.
# To use it you just need to source your build environment setup script and
# add the meta-selftest layer to your BBLAYERS.
2016-04-15 17:30:06 +00:00
# Call the script as: "oe-selftest -a" to run all the tests in meta/lib/oeqa/selftest/
# Call the script as: "oe-selftest -r <module>.<Class>.<method>" to run just a single test
# E.g: "oe-selftest -r bblayers.BitbakeLayers" will run just the BitbakeLayers class from meta/lib/oeqa/selftest/bblayers.py
2013-11-27 17:08:50 +00:00
import os
import sys
import unittest
import logging
2014-07-17 15:10:43 +00:00
import argparse
2015-11-18 15:04:17 +00:00
import subprocess
2015-12-22 08:36:29 +00:00
import time as t
oe-selftest: Fixed --list-tests-by tag option
Commit 35be67951305950ba797dc2efddbc7d88fc0556a broke the
--list-tests-by tag option.
This patch fixes that.
Having a module in lib/oeqa/selftest named testmodule:
class TestClass(oeSelfTest):
@tag(feature='tag1')
def test_func1(self):
pass
@tag(feature=('tag1', 'tag2'))
def test_func2(self):
pass
@tag(feature=('tag2', 'tag3'))
def test_func3(self):
pass
@tag(feature=('tag1', 'tag2', 'tag3'))
def test_func4(self):
pass
$ oe-selftest --list-tests-by tag tag1
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1
Total found: 3
$ oe-selftest --list-tests-by tag tag1 tag2
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1, tag2
Total found: 4
$ oe-selftest --list-tests-by tag tag*
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag*
Total found: 4
(From OE-Core rev: 28c1cffacf341ad64a5b68d8a0176f92b49135c0)
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-03-18 13:18:09 +00:00
import re
import fnmatch
2016-05-20 10:57:44 +00:00
import collections
import imp
2013-11-27 17:08:50 +00:00
2015-07-27 13:03:56 +00:00
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
import scriptpath
scriptpath.add_bitbake_lib_path()
scriptpath.add_oe_lib_path()
2015-12-22 04:02:54 +00:00
import argparse_oe
2013-11-27 17:08:50 +00:00
import oeqa.selftest
import oeqa.utils.ftools as ftools
from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
2016-11-17 12:18:52 +00:00
from oeqa.utils.metadata import metadata_from_bb, write_metadata_file
2016-01-05 14:53:34 +00:00
from oeqa.selftest.base import oeSelfTest, get_available_machines
2013-11-27 17:08:50 +00:00
2016-07-21 12:02:05 +00:00
try:
import xmlrunner
from xmlrunner.result import _XMLTestResult as TestResult
from xmlrunner import XMLTestRunner as _TestRunner
except ImportError:
# use the base runner instead
from unittest import TextTestResult as TestResult
from unittest import TextTestRunner as _TestRunner
2016-07-21 12:02:06 +00:00
log_prefix = "oe-selftest-" + t.strftime("%Y%m%d-%H%M%S")
2016-07-21 12:02:05 +00:00
2013-11-27 17:08:50 +00:00
def logger_create():
2016-07-21 12:02:05 +00:00
log_file = log_prefix + ".log"
2016-12-22 03:13:57 +00:00
if os.path.lexists("oe-selftest.log"):
os.remove("oe-selftest.log")
2015-12-22 08:36:29 +00:00
os.symlink(log_file, "oe-selftest.log")
2013-11-27 17:08:50 +00:00
log = logging.getLogger("selftest")
log.setLevel(logging.DEBUG)
2015-12-22 08:36:29 +00:00
fh = logging.FileHandler(filename=log_file, mode='w')
2013-11-27 17:08:50 +00:00
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
log.addHandler(fh)
log.addHandler(ch)
return log
log = logger_create()
2014-07-17 15:10:43 +00:00
def get_args_parser():
2017-01-12 01:51:42 +00:00
description = "Script that runs unit tests against bitbake and other Yocto related tools. The goal is to validate tools functionality and metadata integrity. Refer to https://wiki.yoctoproject.org/wiki/Oe-selftest for more information."
2015-12-22 04:02:54 +00:00
parser = argparse_oe.ArgumentParser(description=description)
2014-07-17 15:10:43 +00:00
group = parser.add_mutually_exclusive_group(required=True)
2016-03-11 16:54:03 +00:00
group.add_argument('-r', '--run-tests', required=False, action='store', nargs='*', dest="run_tests", default=None, help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
group.add_argument('-a', '--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False, help='Run all (unhidden) tests')
group.add_argument('-m', '--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.')
2015-02-04 14:13:57 +00:00
group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.')
2015-11-18 15:04:17 +00:00
parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing")
2016-02-19 21:29:31 +00:00
parser.add_argument('--coverage-source', dest="coverage_source", nargs="+", help="Specifiy the directories to take coverage from")
parser.add_argument('--coverage-include', dest="coverage_include", nargs="+", help="Specify extra patterns to include into the coverage measurement")
parser.add_argument('--coverage-omit', dest="coverage_omit", nargs="+", help="Specify with extra patterns to exclude from the coverage measurement")
2015-12-02 14:40:53 +00:00
group.add_argument('--run-tests-by', required=False, dest='run_tests_by', default=False, nargs='*',
help='run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
group.add_argument('--list-tests-by', required=False, dest='list_tests_by', default=False, nargs='*',
help='list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
2016-03-11 16:54:03 +00:00
group.add_argument('-l', '--list-tests', required=False, action="store_true", dest="list_tests", default=False,
2016-01-04 15:47:59 +00:00
help='List all available tests.')
2015-12-02 14:40:53 +00:00
group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true",
help='List all tags that have been set to test cases.')
2016-01-05 14:53:34 +00:00
parser.add_argument('--machine', required=False, dest='machine', choices=['random', 'all'], default=None,
help='Run tests on different machines (random/all).')
2016-11-17 12:18:52 +00:00
parser.add_argument('--repository', required=False, dest='repository', default='', action='store',
help='Submit test results to a repository')
2014-07-17 15:10:43 +00:00
return parser
2017-01-04 23:34:30 +00:00
builddir = None
2014-07-17 15:10:43 +00:00
2013-11-27 17:08:50 +00:00
def preflight_check():
2017-01-04 23:34:30 +00:00
global builddir
2013-11-27 17:08:50 +00:00
log.info("Checking that everything is in order before running the tests")
if not os.environ.get("BUILDDIR"):
log.error("BUILDDIR isn't set. Did you forget to source your build environment setup script?")
return False
builddir = os.environ.get("BUILDDIR")
if os.getcwd() != builddir:
log.info("Changing cwd to %s" % builddir)
os.chdir(builddir)
if not "meta-selftest" in get_bb_var("BBLAYERS"):
2017-01-19 15:55:52 +00:00
log.warn("meta-selftest layer not found in BBLAYERS, adding it")
meta_selftestdir = os.path.join(
get_bb_var("BBLAYERS_FETCH_DIR"),
'meta-selftest')
if os.path.isdir(meta_selftestdir):
runCmd("bitbake-layers add-layer %s" %meta_selftestdir)
else:
log.error("could not locate meta-selftest in:\n%s"
%meta_selftestdir)
return False
2013-11-27 17:08:50 +00:00
2017-01-12 13:30:53 +00:00
if "buildhistory.bbclass" in get_bb_var("BBINCLUDED"):
log.error("You have buildhistory enabled already and this isn't recommended for selftest, please disable it first.")
return False
2013-11-27 17:08:50 +00:00
log.info("Running bitbake -p")
runCmd("bitbake -p")
return True
def add_include():
2017-01-04 23:34:30 +00:00
global builddir
2013-11-27 17:08:50 +00:00
if "#include added by oe-selftest.py" \
not in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
log.info("Adding: \"include selftest.inc\" in local.conf")
ftools.append_file(os.path.join(builddir, "conf/local.conf"), \
2016-01-05 14:53:34 +00:00
"\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
2013-11-27 17:08:50 +00:00
2015-06-29 15:31:55 +00:00
if "#include added by oe-selftest.py" \
not in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
log.info("Adding: \"include bblayers.inc\" in bblayers.conf")
ftools.append_file(os.path.join(builddir, "conf/bblayers.conf"), \
"\n#include added by oe-selftest.py\ninclude bblayers.inc")
2013-11-27 17:08:50 +00:00
def remove_include():
2017-01-04 23:34:30 +00:00
global builddir
2015-02-04 14:13:56 +00:00
if builddir is None:
return
2013-11-27 17:08:50 +00:00
if "#include added by oe-selftest.py" \
in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
log.info("Removing the include from local.conf")
ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \
2016-01-05 14:53:34 +00:00
"\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
2013-11-27 17:08:50 +00:00
2015-06-29 15:31:55 +00:00
if "#include added by oe-selftest.py" \
in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
log.info("Removing the include from bblayers.conf")
ftools.remove_from_file(os.path.join(builddir, "conf/bblayers.conf"), \
2016-01-05 14:53:34 +00:00
"\n#include added by oe-selftest.py\ninclude bblayers.inc")
2013-12-04 11:42:58 +00:00
def remove_inc_files():
2017-01-04 23:34:30 +00:00
global builddir
if builddir is None:
return
2013-12-04 11:42:58 +00:00
try:
2017-01-04 23:34:30 +00:00
os.remove(os.path.join(builddir, "conf/selftest.inc"))
2013-12-04 11:42:58 +00:00
for root, _, files in os.walk(get_test_layer()):
for f in files:
if f == 'test_recipe.inc':
os.remove(os.path.join(root, f))
2017-01-04 23:34:30 +00:00
except OSError as e:
2013-12-04 11:42:58 +00:00
pass
2016-01-05 14:53:34 +00:00
for incl_file in ['conf/bblayers.inc', 'conf/machine.inc']:
try:
2017-01-04 23:34:30 +00:00
os.remove(os.path.join(builddir, incl_file))
2016-01-05 14:53:34 +00:00
except:
pass
2015-06-29 15:31:55 +00:00
2016-02-22 12:32:55 +00:00
def get_tests_modules(include_hidden=False):
modules_list = list()
for modules_path in oeqa.selftest.__path__:
for (p, d, f) in os.walk(modules_path):
files = sorted([f for f in os.listdir(p) if f.endswith('.py') and not (f.startswith('_') and not include_hidden) and not f.startswith('__') and f != 'base.py'])
for f in files:
submodules = p.split("selftest")[-1]
module = ""
if submodules:
module = 'oeqa.selftest' + submodules.replace("/",".") + "." + f.split('.py')[0]
else:
module = 'oeqa.selftest.' + f.split('.py')[0]
if module not in modules_list:
modules_list.append(module)
return modules_list
2014-07-17 15:10:43 +00:00
def get_tests(exclusive_modules=[], include_hidden=False):
2016-02-22 12:32:55 +00:00
test_modules = list()
2014-07-17 15:10:43 +00:00
for x in exclusive_modules:
2016-02-22 12:32:55 +00:00
test_modules.append('oeqa.selftest.' + x)
if not test_modules:
inc_hidden = include_hidden
test_modules = get_tests_modules(inc_hidden)
2013-11-27 17:08:50 +00:00
2016-02-22 12:32:55 +00:00
return test_modules
2013-11-27 17:08:50 +00:00
2015-12-02 14:40:53 +00:00
class Tc:
def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None):
self.tcname = tcname
self.tcclass = tcclass
self.tcmodule = tcmodule
self.tcid = tcid
oe-selftest: Fixed --list-tests-by tag option
Commit 35be67951305950ba797dc2efddbc7d88fc0556a broke the
--list-tests-by tag option.
This patch fixes that.
Having a module in lib/oeqa/selftest named testmodule:
class TestClass(oeSelfTest):
@tag(feature='tag1')
def test_func1(self):
pass
@tag(feature=('tag1', 'tag2'))
def test_func2(self):
pass
@tag(feature=('tag2', 'tag3'))
def test_func3(self):
pass
@tag(feature=('tag1', 'tag2', 'tag3'))
def test_func4(self):
pass
$ oe-selftest --list-tests-by tag tag1
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1
Total found: 3
$ oe-selftest --list-tests-by tag tag1 tag2
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1, tag2
Total found: 4
$ oe-selftest --list-tests-by tag tag*
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag*
Total found: 4
(From OE-Core rev: 28c1cffacf341ad64a5b68d8a0176f92b49135c0)
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-03-18 13:18:09 +00:00
# A test case can have multiple tags (as tuples) otherwise str will suffice
2015-12-02 14:40:53 +00:00
self.tctag = tctag
self.fullpath = '.'.join(['oeqa', 'selftest', tcmodule, tcclass, tcname])
def get_tests_from_module(tmod):
tlist = []
prefix = 'oeqa.selftest.'
try:
import importlib
modlib = importlib.import_module(tmod)
2016-05-20 10:53:11 +00:00
for mod in list(vars(modlib).values()):
2015-12-02 14:40:53 +00:00
if isinstance(mod, type(oeSelfTest)) and issubclass(mod, oeSelfTest) and mod is not oeSelfTest:
for test in dir(mod):
if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'):
# Get test case id and feature tag
# NOTE: if testcase decorator or feature tag not set will throw error
try:
tid = vars(mod)[test].test_case
except:
2016-05-21 11:25:02 +00:00
print('DEBUG: tc id missing for ' + str(test))
2015-12-02 14:40:53 +00:00
tid = None
try:
ttag = vars(mod)[test].tag__feature
except:
2016-05-21 11:25:02 +00:00
# print('DEBUG: feature tag missing for ' + str(test))
2015-12-02 14:40:53 +00:00
ttag = None
# NOTE: for some reason lstrip() doesn't work for mod.__module__
tlist.append(Tc(test, mod.__name__, mod.__module__.replace(prefix, ''), tid, ttag))
except:
pass
return tlist
def get_all_tests():
# Get all the test modules (except the hidden ones)
2016-02-22 12:32:55 +00:00
testlist = []
tests_modules = get_tests_modules()
2015-12-02 14:40:53 +00:00
# Get all the tests from modules
2016-02-22 12:32:55 +00:00
for tmod in sorted(tests_modules):
2015-12-02 14:40:53 +00:00
testlist += get_tests_from_module(tmod)
return testlist
oe-selftest: Fixed --list-tests-by tag option
Commit 35be67951305950ba797dc2efddbc7d88fc0556a broke the
--list-tests-by tag option.
This patch fixes that.
Having a module in lib/oeqa/selftest named testmodule:
class TestClass(oeSelfTest):
@tag(feature='tag1')
def test_func1(self):
pass
@tag(feature=('tag1', 'tag2'))
def test_func2(self):
pass
@tag(feature=('tag2', 'tag3'))
def test_func3(self):
pass
@tag(feature=('tag1', 'tag2', 'tag3'))
def test_func4(self):
pass
$ oe-selftest --list-tests-by tag tag1
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1
Total found: 3
$ oe-selftest --list-tests-by tag tag1 tag2
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1, tag2
Total found: 4
$ oe-selftest --list-tests-by tag tag*
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag*
Total found: 4
(From OE-Core rev: 28c1cffacf341ad64a5b68d8a0176f92b49135c0)
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-03-18 13:18:09 +00:00
2015-12-02 14:40:53 +00:00
def get_testsuite_by(criteria, keyword):
# Get a testsuite based on 'keyword'
# criteria: name, class, module, id, tag
# keyword: a list of tests, classes, modules, ids, tags
2016-03-11 16:28:24 +00:00
ts = []
2015-12-02 14:40:53 +00:00
all_tests = get_all_tests()
2016-03-11 16:28:24 +00:00
def get_matches(values):
oe-selftest: Fixed --list-tests-by tag option
Commit 35be67951305950ba797dc2efddbc7d88fc0556a broke the
--list-tests-by tag option.
This patch fixes that.
Having a module in lib/oeqa/selftest named testmodule:
class TestClass(oeSelfTest):
@tag(feature='tag1')
def test_func1(self):
pass
@tag(feature=('tag1', 'tag2'))
def test_func2(self):
pass
@tag(feature=('tag2', 'tag3'))
def test_func3(self):
pass
@tag(feature=('tag1', 'tag2', 'tag3'))
def test_func4(self):
pass
$ oe-selftest --list-tests-by tag tag1
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1
Total found: 3
$ oe-selftest --list-tests-by tag tag1 tag2
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1, tag2
Total found: 4
$ oe-selftest --list-tests-by tag tag*
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag*
Total found: 4
(From OE-Core rev: 28c1cffacf341ad64a5b68d8a0176f92b49135c0)
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-03-18 13:18:09 +00:00
# Get an item and return the ones that match with keyword(s)
2016-03-11 16:28:24 +00:00
# values: the list of items (names, modules, classes...)
result = []
remaining = values[:]
for key in keyword:
2016-07-12 12:04:01 +00:00
found = False
2016-03-11 16:28:24 +00:00
if key in remaining:
# Regular matching of exact item
result.append(key)
remaining.remove(key)
2016-07-12 12:04:01 +00:00
found = True
2016-03-11 16:28:24 +00:00
else:
# Wildcard matching
pattern = re.compile(fnmatch.translate(r"%s" % key))
oe-selftest: Fixed --list-tests-by tag option
Commit 35be67951305950ba797dc2efddbc7d88fc0556a broke the
--list-tests-by tag option.
This patch fixes that.
Having a module in lib/oeqa/selftest named testmodule:
class TestClass(oeSelfTest):
@tag(feature='tag1')
def test_func1(self):
pass
@tag(feature=('tag1', 'tag2'))
def test_func2(self):
pass
@tag(feature=('tag2', 'tag3'))
def test_func3(self):
pass
@tag(feature=('tag1', 'tag2', 'tag3'))
def test_func4(self):
pass
$ oe-selftest --list-tests-by tag tag1
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1
Total found: 3
$ oe-selftest --list-tests-by tag tag1 tag2
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1, tag2
Total found: 4
$ oe-selftest --list-tests-by tag tag*
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag*
Total found: 4
(From OE-Core rev: 28c1cffacf341ad64a5b68d8a0176f92b49135c0)
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-03-18 13:18:09 +00:00
added = [x for x in remaining if pattern.match(x)]
2016-07-12 12:04:01 +00:00
if added:
result.extend(added)
remaining = [x for x in remaining if x not in added]
found = True
if not found:
log.error("Failed to find test: %s" % key)
2016-03-11 16:28:24 +00:00
return result
2015-12-02 14:40:53 +00:00
if criteria == 'name':
2016-03-11 16:28:24 +00:00
names = get_matches([ tc.tcname for tc in all_tests ])
ts = [ tc for tc in all_tests if tc.tcname in names ]
2015-12-02 14:40:53 +00:00
elif criteria == 'class':
2016-03-11 16:28:24 +00:00
classes = get_matches([ tc.tcclass for tc in all_tests ])
ts = [ tc for tc in all_tests if tc.tcclass in classes ]
2015-12-02 14:40:53 +00:00
elif criteria == 'module':
2016-03-11 16:28:24 +00:00
modules = get_matches([ tc.tcmodule for tc in all_tests ])
ts = [ tc for tc in all_tests if tc.tcmodule in modules ]
2015-12-02 14:40:53 +00:00
elif criteria == 'id':
2016-03-11 16:28:24 +00:00
ids = get_matches([ str(tc.tcid) for tc in all_tests ])
ts = [ tc for tc in all_tests if str(tc.tcid) in ids ]
2015-12-02 14:40:53 +00:00
elif criteria == 'tag':
2016-03-11 16:28:24 +00:00
values = set()
2015-12-02 14:40:53 +00:00
for tc in all_tests:
oe-selftest: Fixed --list-tests-by tag option
Commit 35be67951305950ba797dc2efddbc7d88fc0556a broke the
--list-tests-by tag option.
This patch fixes that.
Having a module in lib/oeqa/selftest named testmodule:
class TestClass(oeSelfTest):
@tag(feature='tag1')
def test_func1(self):
pass
@tag(feature=('tag1', 'tag2'))
def test_func2(self):
pass
@tag(feature=('tag2', 'tag3'))
def test_func3(self):
pass
@tag(feature=('tag1', 'tag2', 'tag3'))
def test_func4(self):
pass
$ oe-selftest --list-tests-by tag tag1
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1
Total found: 3
$ oe-selftest --list-tests-by tag tag1 tag2
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1, tag2
Total found: 4
$ oe-selftest --list-tests-by tag tag*
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag*
Total found: 4
(From OE-Core rev: 28c1cffacf341ad64a5b68d8a0176f92b49135c0)
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-03-18 13:18:09 +00:00
# tc can have multiple tags (as tuple) otherwise str will suffice
if isinstance(tc.tctag, tuple):
2016-03-11 16:28:24 +00:00
values |= { str(tag) for tag in tc.tctag }
else:
values.add(str(tc.tctag))
tags = get_matches(list(values))
oe-selftest: Fixed --list-tests-by tag option
Commit 35be67951305950ba797dc2efddbc7d88fc0556a broke the
--list-tests-by tag option.
This patch fixes that.
Having a module in lib/oeqa/selftest named testmodule:
class TestClass(oeSelfTest):
@tag(feature='tag1')
def test_func1(self):
pass
@tag(feature=('tag1', 'tag2'))
def test_func2(self):
pass
@tag(feature=('tag2', 'tag3'))
def test_func3(self):
pass
@tag(feature=('tag1', 'tag2', 'tag3'))
def test_func4(self):
pass
$ oe-selftest --list-tests-by tag tag1
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1
Total found: 3
$ oe-selftest --list-tests-by tag tag1 tag2
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag1, tag2
Total found: 4
$ oe-selftest --list-tests-by tag tag*
ID TAG(s) NAME CLASS MODULE
---- ---------------- ---------- --------- --------
tag1 test_func1 TestClass testmodule
tag1, tag2 test_func2 TestClass testmodule
tag1, tag2, tag3 test_func4 TestClass testmodule
tag2, tag3 test_func3 TestClass testmodule
______________________________
Filtering by: tag
Looking for: tag*
Total found: 4
(From OE-Core rev: 28c1cffacf341ad64a5b68d8a0176f92b49135c0)
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-03-18 13:18:09 +00:00
for tc in all_tests:
for tag in tags:
if isinstance(tc.tctag, tuple) and tag in tc.tctag:
ts.append(tc)
elif tag == tc.tctag:
ts.append(tc)
# Remove duplicates from the list
ts = list(set(ts))
2015-12-02 14:40:53 +00:00
2016-03-11 16:28:24 +00:00
return ts
2015-12-02 14:40:53 +00:00
def list_testsuite_by(criteria, keyword):
# Get a testsuite based on 'keyword'
# criteria: name, class, module, id, tag
# keyword: a list of tests, classes, modules, ids, tags
oe-selftest: fix handling of test cases without ID in --list-tests-by
Running `oe-selftest --list-tests-by module wic` will produce the
following backtrace:
Traceback (most recent call last):
File "<snip>/poky/scripts/oe-selftest", line 668, in <module>
ret = main()
File "<snip>/poky/scripts/oe-selftest", line 486, in main
list_testsuite_by(criteria, keyword)
File "<snip>/poky/scripts/oe-selftest", line 340, in list_testsuite_by
ts = sorted([ (tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule) for tc in get_testsuite_by(criteria, keyword) ])
TypeError: unorderable types: int() < NoneType()
The root cause is that a test case does not necessarily have an ID
assigned, hence its value is None. Since Python 3 does not allow
comparison of heterogeneous types, TypeError is raised.
(From OE-Core rev: 71c6790689e2cbd3c4e882335b3b03e635ad46ed)
Signed-off-by: Maciej Borzecki <maciej.borzecki@rndity.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-11-10 12:18:37 +00:00
def tc_key(t):
if t[0] is None:
return (0,) + t[1:]
return t
# tcid may be None if no ID was assigned, in which case sorted() will throw
# a TypeError as Python 3 does not allow comparison (<,<=,>=,>) of
# heterogeneous types, handle this by using a custom key generator
ts = sorted([ (tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule) \
for tc in get_testsuite_by(criteria, keyword) ], key=tc_key)
2016-05-21 11:25:02 +00:00
print('_' * 150)
2015-12-02 14:40:53 +00:00
for t in ts:
if isinstance(t[1], (tuple, list)):
2016-05-21 11:25:02 +00:00
print('%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4]))
2015-12-02 14:40:53 +00:00
else:
2016-05-21 11:25:02 +00:00
print('%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t)
print('_' * 150)
print('Filtering by:\t %s' % criteria)
print('Looking for:\t %s' % ', '.join(str(x) for x in keyword))
print('Total found:\t %s' % len(ts))
2015-12-02 14:40:53 +00:00
2016-01-04 15:47:59 +00:00
def list_tests():
# List all available oe-selftest tests
ts = get_all_tests()
2016-05-21 11:25:02 +00:00
print('%-4s\t%-10s\t%-50s' % ('id', 'tag', 'test'))
print('_' * 80)
2016-01-04 15:47:59 +00:00
for t in ts:
if isinstance(t.tctag, (tuple, list)):
2016-05-21 11:25:02 +00:00
print('%-4s\t%-10s\t%-50s' % (t.tcid, ', '.join(t.tctag), '.'.join([t.tcmodule, t.tcclass, t.tcname])))
2016-01-04 15:47:59 +00:00
else:
2016-05-21 11:25:02 +00:00
print('%-4s\t%-10s\t%-50s' % (t.tcid, t.tctag, '.'.join([t.tcmodule, t.tcclass, t.tcname])))
print('_' * 80)
print('Total found:\t %s' % len(ts))
2016-01-04 15:47:59 +00:00
2015-12-02 14:40:53 +00:00
def list_tags():
# Get all tags set to test cases
# This is useful when setting tags to test cases
# The list of tags should be kept as minimal as possible
tags = set()
all_tests = get_all_tests()
for tc in all_tests:
if isinstance(tc.tctag, (tuple, list)):
tags.update(set(tc.tctag))
else:
tags.add(tc.tctag)
2016-05-21 11:25:02 +00:00
print('Tags:\t%s' % ', '.join(str(x) for x in tags))
2015-12-02 14:40:53 +00:00
2016-03-24 00:32:02 +00:00
def coverage_setup(coverage_source, coverage_include, coverage_omit):
2015-12-23 23:36:08 +00:00
""" Set up the coverage measurement for the testcases to be run """
2016-03-24 00:32:02 +00:00
import datetime
import subprocess
2017-01-04 23:34:30 +00:00
global builddir
2016-03-24 00:32:02 +00:00
pokydir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
2016-06-02 10:13:00 +00:00
curcommit= subprocess.check_output(["git", "--git-dir", os.path.join(pokydir, ".git"), "rev-parse", "HEAD"]).decode('utf-8')
2015-12-23 23:36:08 +00:00
coveragerc = "%s/.coveragerc" % builddir
data_file = "%s/.coverage." % builddir
2016-03-24 00:32:02 +00:00
data_file += datetime.datetime.now().strftime('%Y%m%dT%H%M%S')
2015-12-23 23:36:08 +00:00
if os.path.isfile(data_file):
os.remove(data_file)
with open(coveragerc, 'w') as cps:
2016-03-24 00:32:02 +00:00
cps.write("# Generated with command '%s'\n" % " ".join(sys.argv))
cps.write("# HEAD commit %s\n" % curcommit.strip())
2015-12-23 23:36:08 +00:00
cps.write("[run]\n")
cps.write("data_file = %s\n" % data_file)
cps.write("branch = True\n")
# Measure just BBLAYERS, scripts and bitbake folders
cps.write("source = \n")
2016-02-19 21:29:31 +00:00
if coverage_source:
for directory in coverage_source:
if not os.path.isdir(directory):
log.warn("Directory %s is not valid.", directory)
cps.write(" %s\n" % directory)
else:
for layer in get_bb_var('BBLAYERS').split():
cps.write(" %s\n" % layer)
cps.write(" %s\n" % os.path.dirname(os.path.realpath(__file__)))
cps.write(" %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake'))
if coverage_include:
cps.write("include = \n")
for pattern in coverage_include:
cps.write(" %s\n" % pattern)
if coverage_omit:
cps.write("omit = \n")
for pattern in coverage_omit:
cps.write(" %s\n" % pattern)
2015-12-23 23:36:08 +00:00
return coveragerc
def coverage_report():
""" Loads the coverage data gathered and reports it back """
try:
# Coverage4 uses coverage.Coverage
from coverage import Coverage
except:
# Coverage under version 4 uses coverage.coverage
from coverage import coverage as Coverage
2016-05-20 10:57:44 +00:00
import io as StringIO
2015-12-23 23:36:08 +00:00
from coverage.misc import CoverageException
cov_output = StringIO.StringIO()
# Creating the coverage data with the setting from the configuration file
cov = Coverage(config_file = os.environ.get('COVERAGE_PROCESS_START'))
try:
# Load data from the data file specified in the configuration
cov.load()
# Store report data in a StringIO variable
cov.report(file = cov_output, show_missing=False)
log.info("\n%s" % cov_output.getvalue())
except CoverageException as e:
# Show problems with the reporting. Since Coverage4 not finding any data to report raises an exception
log.warn("%s" % str(e))
finally:
cov_output.close()
2015-12-02 14:40:53 +00:00
2013-11-27 17:08:50 +00:00
def main():
2014-07-17 15:10:43 +00:00
parser = get_args_parser()
args = parser.parse_args()
2015-07-13 19:04:21 +00:00
# Add <layer>/lib to sys.path, so layers can add selftests
log.info("Running bitbake -e to get BBPATH")
bbpath = get_bb_var('BBPATH').split(':')
layer_libdirs = [p for p in (os.path.join(l, 'lib') for l in bbpath) if os.path.exists(p)]
sys.path.extend(layer_libdirs)
2016-05-20 10:57:44 +00:00
imp.reload(oeqa.selftest)
2015-07-13 19:04:21 +00:00
2016-11-15 09:52:11 +00:00
# act like bitbake and enforce en_US.UTF-8 locale
os.environ["LC_ALL"] = "en_US.UTF-8"
2015-12-02 14:40:53 +00:00
if args.run_tests_by and len(args.run_tests_by) >= 2:
valid_options = ['name', 'class', 'module', 'id', 'tag']
if args.run_tests_by[0] not in valid_options:
2016-05-21 11:25:02 +00:00
print('--run-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.run_tests_by[0])
2015-12-02 14:40:53 +00:00
return 1
else:
criteria = args.run_tests_by[0]
keyword = args.run_tests_by[1:]
2016-03-11 16:28:24 +00:00
ts = sorted([ tc.fullpath for tc in get_testsuite_by(criteria, keyword) ])
2016-07-12 12:04:01 +00:00
if not ts:
return 1
2015-12-02 14:40:53 +00:00
if args.list_tests_by and len(args.list_tests_by) >= 2:
valid_options = ['name', 'class', 'module', 'id', 'tag']
if args.list_tests_by[0] not in valid_options:
2016-05-21 11:25:02 +00:00
print('--list-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.list_tests_by[0])
2015-12-02 14:40:53 +00:00
return 1
else:
criteria = args.list_tests_by[0]
keyword = args.list_tests_by[1:]
list_testsuite_by(criteria, keyword)
2016-01-04 15:47:59 +00:00
if args.list_tests:
list_tests()
2015-12-02 14:40:53 +00:00
if args.list_tags:
list_tags()
2015-02-04 14:13:57 +00:00
if args.list_allclasses:
args.list_modules = True
2014-07-17 15:10:43 +00:00
if args.list_modules:
log.info('Listing all available test modules:')
testslist = get_tests(include_hidden=True)
for test in testslist:
2016-02-22 12:32:55 +00:00
module = test.split('oeqa.selftest.')[-1]
2014-07-17 15:10:43 +00:00
info = ''
if module.startswith('_'):
info = ' (hidden)'
2016-05-21 11:25:02 +00:00
print(module + info)
2015-02-04 14:13:57 +00:00
if args.list_allclasses:
try:
import importlib
modlib = importlib.import_module(test)
for v in vars(modlib):
t = vars(modlib)[v]
if isinstance(t, type(oeSelfTest)) and issubclass(t, oeSelfTest) and t!=oeSelfTest:
2016-05-21 11:25:02 +00:00
print(" --", v)
2015-02-04 14:13:57 +00:00
for method in dir(t):
2016-05-20 10:57:44 +00:00
if method.startswith("test_") and isinstance(vars(t)[method], collections.Callable):
2016-05-21 11:25:02 +00:00
print(" -- --", method)
2015-02-04 14:13:57 +00:00
except (AttributeError, ImportError) as e:
2016-05-21 11:25:02 +00:00
print(e)
2015-02-04 14:13:57 +00:00
pass
2015-12-02 14:40:53 +00:00
if args.run_tests or args.run_all_tests or args.run_tests_by:
2014-07-17 15:10:43 +00:00
if not preflight_check():
return 1
2015-12-02 14:40:53 +00:00
if args.run_tests_by:
testslist = ts
else:
testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
2014-07-17 15:10:43 +00:00
suite = unittest.TestSuite()
loader = unittest.TestLoader()
loader.sortTestMethodsUsing = None
2016-07-21 12:02:05 +00:00
runner = TestRunner(verbosity=2,
resultclass=buildResultClass(args))
2014-07-17 15:10:43 +00:00
# we need to do this here, otherwise just loading the tests
# will take 2 minutes (bitbake -e calls)
oeSelfTest.testlayer_path = get_test_layer()
for test in testslist:
log.info("Loading tests from: %s" % test)
try:
suite.addTests(loader.loadTestsFromName(test))
except AttributeError as e:
log.error("Failed to import %s" % test)
log.error(e)
return 1
add_include()
2016-01-05 14:53:34 +00:00
if args.machine:
# Custom machine sets only weak default values (??=) for MACHINE in machine.inc
# This let test cases that require a specific MACHINE to be able to override it, using (?= or =)
log.info('Custom machine mode enabled. MACHINE set to %s' % args.machine)
if args.machine == 'random':
os.environ['CUSTOMMACHINE'] = 'random'
result = runner.run(suite)
else: # all
machines = get_available_machines()
for m in machines:
log.info('Run tests with custom MACHINE set to: %s' % m)
os.environ['CUSTOMMACHINE'] = m
result = runner.run(suite)
else:
result = runner.run(suite)
2014-07-17 15:10:43 +00:00
log.info("Finished")
2015-11-18 15:04:17 +00:00
2016-11-17 12:18:52 +00:00
if args.repository:
2016-12-19 13:51:43 +00:00
import git
2016-11-17 12:18:52 +00:00
# Commit tests results to repository
metadata = metadata_from_bb()
git_dir = os.path.join(os.getcwd(), 'selftest')
if not os.path.isdir(git_dir):
os.mkdir(git_dir)
log.debug('Checking for git repository in %s' % git_dir)
try:
repo = git.Repo(git_dir)
except git.exc.InvalidGitRepositoryError:
log.debug("Couldn't find git repository %s; "
"cloning from %s" % (git_dir, args.repository))
repo = git.Repo.clone_from(args.repository, git_dir)
r_branches = repo.git.branch(r=True)
r_branches = set(r_branches.replace('origin/', '').split())
l_branches = {str(branch) for branch in repo.branches}
branch = '%s/%s/%s' % (metadata['hostname'],
2017-01-13 13:12:39 +00:00
metadata['layers']['meta'].get('branch', '(nogit)'),
2017-01-13 13:12:45 +00:00
metadata['config']['MACHINE'])
2016-11-17 12:18:52 +00:00
if branch in l_branches:
log.debug('Found branch in local repository, checking out')
repo.git.checkout(branch)
elif branch in r_branches:
log.debug('Found branch in remote repository, checking'
' out and pulling')
repo.git.checkout(branch)
repo.git.pull()
else:
log.debug('New branch %s' % branch)
repo.git.checkout('master')
repo.git.checkout(b=branch)
cleanResultsDir(repo)
xml_dir = os.path.join(os.getcwd(), log_prefix)
copyResultFiles(xml_dir, git_dir, repo)
metadata_file = os.path.join(git_dir, 'metadata.xml')
write_metadata_file(metadata_file, metadata)
repo.index.add([metadata_file])
repo.index.write()
# Get information for commit message
layer_info = ''
for layer, values in metadata['layers'].items():
layer_info = '%s%-17s = %s:%s\n' % (layer_info, layer,
2017-01-13 13:12:41 +00:00
values.get('branch', '(nogit)'), values.get('commit', '0'*40))
2017-01-13 13:12:38 +00:00
msg = 'Selftest for build %s of %s for machine %s on %s\n\n%s' % (
log_prefix[12:], metadata['distro']['pretty_name'],
2017-01-13 13:12:45 +00:00
metadata['config']['MACHINE'], metadata['hostname'], layer_info)
2016-11-17 12:18:52 +00:00
log.debug('Commiting results to local repository')
repo.index.commit(msg)
if not repo.is_dirty():
try:
if branch in r_branches:
log.debug('Pushing changes to remote repository')
repo.git.push()
else:
log.debug('Pushing changes to remote repository '
'creating new branch')
repo.git.push('-u', 'origin', branch)
except GitCommandError:
log.error('Falied to push to remote repository')
return 1
else:
log.error('Local repository is dirty, not pushing commits')
2014-07-17 15:10:43 +00:00
if result.wasSuccessful():
return 0
else:
2013-11-27 17:08:50 +00:00
return 1
2015-12-23 23:36:08 +00:00
def buildResultClass(args):
"""Build a Result Class to use in the testcase execution"""
2016-02-19 21:29:32 +00:00
import site
2015-12-23 23:36:08 +00:00
2016-07-21 12:02:05 +00:00
class StampedResult(TestResult):
2015-12-23 23:36:08 +00:00
"""
Custom TestResult that prints the time when a test starts. As oe-selftest
can take a long time (ie a few hours) to run, timestamps help us understand
what tests are taking a long time to execute.
If coverage is required, this class executes the coverage setup and reporting.
"""
def startTest(self, test):
import time
self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
super(StampedResult, self).startTest(test)
def startTestRun(self):
""" Setup coverage before running any testcase """
2016-02-19 21:29:32 +00:00
# variable holding the coverage configuration file allowing subprocess to be measured
self.coveragepth = None
# indicates the system if coverage is currently installed
self.coverage_installed = True
2016-02-19 21:29:31 +00:00
if args.coverage or args.coverage_source or args.coverage_include or args.coverage_omit:
2015-12-23 23:36:08 +00:00
try:
# check if user can do coverage
import coverage
except:
2016-04-06 03:15:52 +00:00
log.warn("python coverage is not installed. More info on https://pypi.python.org/pypi/coverage")
2015-12-23 23:36:08 +00:00
self.coverage_installed = False
2016-02-19 21:29:32 +00:00
if self.coverage_installed:
log.info("Coverage is enabled")
2016-09-12 20:30:50 +00:00
major_version = int(coverage.version.__version__[0])
if major_version < 4:
log.error("python coverage %s installed. Require version 4 or greater." % coverage.version.__version__)
self.stop()
2016-04-06 03:15:54 +00:00
# In case the user has not set the variable COVERAGE_PROCESS_START,
# create a default one and export it. The COVERAGE_PROCESS_START
# value indicates where the coverage configuration file resides
# More info on https://pypi.python.org/pypi/coverage
if not os.environ.get('COVERAGE_PROCESS_START'):
os.environ['COVERAGE_PROCESS_START'] = coverage_setup(args.coverage_source, args.coverage_include, args.coverage_omit)
# Use default site.USER_SITE and write corresponding config file
site.ENABLE_USER_SITE = True
if not os.path.exists(site.USER_SITE):
os.makedirs(site.USER_SITE)
self.coveragepth = os.path.join(site.USER_SITE, "coverage.pth")
with open(self.coveragepth, 'w') as cps:
cps.write('import sys,site; sys.path.extend(site.getsitepackages()); import coverage; coverage.process_startup();')
2016-02-19 21:29:32 +00:00
2015-12-23 23:36:08 +00:00
def stopTestRun(self):
""" Report coverage data after the testcases are run """
2016-04-06 03:15:53 +00:00
if args.coverage or args.coverage_source or args.coverage_include or args.coverage_omit:
if self.coverage_installed:
with open(os.environ['COVERAGE_PROCESS_START']) as ccf:
log.info("Coverage configuration file (%s)" % os.environ.get('COVERAGE_PROCESS_START'))
log.info("===========================")
log.info("\n%s" % "".join(ccf.readlines()))
log.info("Coverage Report")
log.info("===============")
2016-02-19 21:29:32 +00:00
try:
2016-04-06 03:15:53 +00:00
coverage_report()
finally:
# remove the pth file
try:
os.remove(self.coveragepth)
except OSError:
log.warn("Expected temporal file from coverage is missing, ignoring removal.")
2015-12-23 23:36:08 +00:00
return StampedResult
2016-11-17 12:18:52 +00:00
def cleanResultsDir(repo):
""" Remove result files from directory """
xml_files = []
directory = repo.working_tree_dir
for f in os.listdir(directory):
path = os.path.join(directory, f)
if os.path.isfile(path) and path.endswith('.xml'):
xml_files.append(f)
repo.index.remove(xml_files, working_tree=True)
def copyResultFiles(src, dst, repo):
""" Copy result files from src to dst removing the time stamp. """
import shutil
re_time = re.compile("-[0-9]+")
file_list = []
for root, subdirs, files in os.walk(src):
tmp_dir = root.replace(src, '').lstrip('/')
for s in subdirs:
os.mkdir(os.path.join(dst, tmp_dir, s))
for f in files:
file_name = os.path.join(dst, tmp_dir, re_time.sub("", f))
shutil.copy2(os.path.join(root, f), file_name)
file_list.append(file_name)
repo.index.add(file_list)
2016-07-21 12:02:05 +00:00
class TestRunner(_TestRunner):
"""Test runner class aware of exporting tests."""
def __init__(self, *args, **kwargs):
try:
exportdir = os.path.join(os.getcwd(), log_prefix)
kwargsx = dict(**kwargs)
# argument specific to XMLTestRunner, if adding a new runner then
# also add logic to use other runner's args.
kwargsx['output'] = exportdir
kwargsx['descriptions'] = False
# done for the case where telling the runner where to export
super(TestRunner, self).__init__(*args, **kwargsx)
except TypeError:
log.info("test runner init'ed like unittest")
super(TestRunner, self).__init__(*args, **kwargs)
2015-06-22 16:20:42 +00:00
2013-11-27 17:08:50 +00:00
if __name__ == "__main__":
try:
ret = main()
except Exception:
ret = 1
import traceback
2016-03-31 11:46:38 +00:00
traceback.print_exc()
2013-11-27 17:08:50 +00:00
finally:
remove_include()
2013-12-04 11:42:58 +00:00
remove_inc_files()
2013-11-27 17:08:50 +00:00
sys.exit(ret)