scripts: oe-selftest Added new features.
[YOCTO #8750] Allow oe-selftest to run custom test suites based on different criteria 1. Can run custom lists of tests based on different criteria: --run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags> eg: --run-tests-by module imagefeatures signing recipetool --run-tests-by id 1377 1273 935 --run-tests-by tag wic sstate bitbake 2. Can list tests based on different criteria: --list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags> eg: --list-tests-by module imagefeatures signing recipetool --list-tests-by id 1377 1273 935 --list-tests-by tag wic sstate bitbake 3. Can list all tags that have been set to test cases: --list-tags The list of tags should be kept as minimal as possible. This helps preview the tags used so far. To take advantage of the 'tag' feature: - add @tag(feature=<>) to testcases eg: @tag(feature='signing') for a single tag @tag(feature=(('signing', 'sstate')) or @tag(feature=['signing', 'sstate']) for multiple tags (From OE-Core rev: 2d3a6d22e155911e39e4b7e323317f4a7cb1cb95) Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Daniel Istrate
Richard Purdie
parent
98d2485fee
commit
220a78b47e
|
|
@ -72,6 +72,12 @@ def get_args_parser():
|
|||
group.add_argument('--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.')
|
||||
group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.')
|
||||
parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing")
|
||||
group.add_argument('--run-tests-by', required=False, dest='run_tests_by', default=False, nargs='*',
|
||||
help='run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
|
||||
group.add_argument('--list-tests-by', required=False, dest='list_tests_by', default=False, nargs='*',
|
||||
help='list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
|
||||
group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true",
|
||||
help='List all tags that have been set to test cases.')
|
||||
return parser
|
||||
|
||||
|
||||
|
|
@ -156,6 +162,187 @@ def get_tests(exclusive_modules=[], include_hidden=False):
|
|||
|
||||
return testslist
|
||||
|
||||
|
||||
class Tc:
|
||||
def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None):
|
||||
self.tcname = tcname
|
||||
self.tcclass = tcclass
|
||||
self.tcmodule = tcmodule
|
||||
self.tcid = tcid
|
||||
# A test case can have multiple tags (as list or as tuples) otherwise str suffice
|
||||
self.tctag = tctag
|
||||
self.fullpath = '.'.join(['oeqa', 'selftest', tcmodule, tcclass, tcname])
|
||||
|
||||
|
||||
def get_tests_from_module(tmod):
|
||||
tlist = []
|
||||
prefix = 'oeqa.selftest.'
|
||||
|
||||
try:
|
||||
import importlib
|
||||
modlib = importlib.import_module(tmod)
|
||||
for mod in vars(modlib).values():
|
||||
if isinstance(mod, type(oeSelfTest)) and issubclass(mod, oeSelfTest) and mod is not oeSelfTest:
|
||||
for test in dir(mod):
|
||||
if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'):
|
||||
# Get test case id and feature tag
|
||||
# NOTE: if testcase decorator or feature tag not set will throw error
|
||||
try:
|
||||
tid = vars(mod)[test].test_case
|
||||
except:
|
||||
print 'DEBUG: tc id missing for ' + str(test)
|
||||
tid = None
|
||||
try:
|
||||
ttag = vars(mod)[test].tag__feature
|
||||
except:
|
||||
# print 'DEBUG: feature tag missing for ' + str(test)
|
||||
ttag = None
|
||||
|
||||
# NOTE: for some reason lstrip() doesn't work for mod.__module__
|
||||
tlist.append(Tc(test, mod.__name__, mod.__module__.replace(prefix, ''), tid, ttag))
|
||||
except:
|
||||
pass
|
||||
|
||||
return tlist
|
||||
|
||||
|
||||
def get_all_tests():
|
||||
tmodules = set()
|
||||
testlist = []
|
||||
prefix = 'oeqa.selftest.'
|
||||
|
||||
# Get all the test modules (except the hidden ones)
|
||||
for tpath in oeqa.selftest.__path__:
|
||||
files = sorted([f for f in os.listdir(tpath) if f.endswith('.py') and not
|
||||
f.startswith(('_', '__')) and f != 'base.py'])
|
||||
for f in files:
|
||||
tmodules.add(prefix + f.rstrip('.py'))
|
||||
|
||||
# Get all the tests from modules
|
||||
tmodules = sorted(list(tmodules))
|
||||
|
||||
for tmod in tmodules:
|
||||
testlist += get_tests_from_module(tmod)
|
||||
|
||||
return testlist
|
||||
|
||||
|
||||
def create_testsuite_by(criteria, keyword):
|
||||
# Create a testsuite based on 'keyword'
|
||||
# criteria: name, class, module, id, tag
|
||||
# keyword: a list of tests, classes, modules, ids, tags
|
||||
# NOTE: globing would be nice?
|
||||
|
||||
ts = set()
|
||||
all_tests = get_all_tests()
|
||||
|
||||
if criteria == 'name':
|
||||
for tc in all_tests:
|
||||
if tc.tcname in keyword:
|
||||
ts.add(tc.fullpath)
|
||||
|
||||
elif criteria == 'class':
|
||||
for tc in all_tests:
|
||||
if tc.tcclass in keyword:
|
||||
ts.add(tc.fullpath)
|
||||
|
||||
elif criteria == 'module':
|
||||
for tc in all_tests:
|
||||
if tc.tcmodule in keyword:
|
||||
ts.add(tc.fullpath)
|
||||
elif criteria == 'id':
|
||||
for tc in all_tests:
|
||||
if str(tc.tcid) in keyword:
|
||||
ts.add(tc.fullpath)
|
||||
elif criteria == 'tag':
|
||||
for tc in all_tests:
|
||||
# tc can have multiple tags (as list or tuple) otherwise as str
|
||||
if isinstance(tc.tctag, (list, tuple)):
|
||||
for tag in tc.tctag:
|
||||
if str(tag) in keyword:
|
||||
ts.add(tc.fullpath)
|
||||
elif tc.tctag in keyword:
|
||||
ts.add(tc.fullpath)
|
||||
|
||||
return sorted(list(ts))
|
||||
|
||||
|
||||
def get_testsuite_by(criteria, keyword):
|
||||
# Get a testsuite based on 'keyword'
|
||||
# criteria: name, class, module, id, tag
|
||||
# keyword: a list of tests, classes, modules, ids, tags
|
||||
# NOTE: globing would be nice?
|
||||
ts = set()
|
||||
all_tests = get_all_tests()
|
||||
|
||||
if criteria == 'name':
|
||||
for tc in all_tests:
|
||||
if tc.tcname in keyword:
|
||||
ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
|
||||
|
||||
elif criteria == 'class':
|
||||
for tc in all_tests:
|
||||
if tc.tcclass in keyword:
|
||||
ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
|
||||
|
||||
elif criteria == 'module':
|
||||
for tc in all_tests:
|
||||
if tc.tcmodule in keyword:
|
||||
ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
|
||||
elif criteria == 'id':
|
||||
for tc in all_tests:
|
||||
if str(tc.tcid) in keyword:
|
||||
ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
|
||||
elif criteria == 'tag':
|
||||
for tc in all_tests:
|
||||
# tc can have multiple tags (as list or tuple) otherwise as str
|
||||
if isinstance(tc.tctag, (list, tuple)):
|
||||
for tag in tc.tctag:
|
||||
if str(tag) in keyword:
|
||||
ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
|
||||
elif str(tc.tctag) in keyword:
|
||||
ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
|
||||
|
||||
return sorted(list(ts))
|
||||
|
||||
|
||||
def list_testsuite_by(criteria, keyword):
|
||||
# Get a testsuite based on 'keyword'
|
||||
# criteria: name, class, module, id, tag
|
||||
# keyword: a list of tests, classes, modules, ids, tags
|
||||
# NOTE: globing would be nice?
|
||||
|
||||
ts = get_testsuite_by(criteria, keyword)
|
||||
|
||||
print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module')
|
||||
print '_' * 150
|
||||
for t in ts:
|
||||
if isinstance(t[1], (tuple, list)):
|
||||
print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4])
|
||||
else:
|
||||
print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t
|
||||
print '_' * 150
|
||||
print 'Filtering by:\t %s' % criteria
|
||||
print 'Looking for:\t %s' % ', '.join(str(x) for x in keyword)
|
||||
print 'Total found:\t %s' % len(ts)
|
||||
|
||||
|
||||
def list_tags():
|
||||
# Get all tags set to test cases
|
||||
# This is useful when setting tags to test cases
|
||||
# The list of tags should be kept as minimal as possible
|
||||
tags = set()
|
||||
all_tests = get_all_tests()
|
||||
|
||||
for tc in all_tests:
|
||||
if isinstance(tc.tctag, (tuple, list)):
|
||||
tags.update(set(tc.tctag))
|
||||
else:
|
||||
tags.add(tc.tctag)
|
||||
|
||||
print 'Tags:\t%s' % ', '.join(str(x) for x in tags)
|
||||
|
||||
|
||||
def main():
|
||||
parser = get_args_parser()
|
||||
args = parser.parse_args()
|
||||
|
|
@ -167,6 +354,29 @@ def main():
|
|||
sys.path.extend(layer_libdirs)
|
||||
reload(oeqa.selftest)
|
||||
|
||||
if args.run_tests_by and len(args.run_tests_by) >= 2:
|
||||
valid_options = ['name', 'class', 'module', 'id', 'tag']
|
||||
if args.run_tests_by[0] not in valid_options:
|
||||
print '--run-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.run_tests_by[0]
|
||||
return 1
|
||||
else:
|
||||
criteria = args.run_tests_by[0]
|
||||
keyword = args.run_tests_by[1:]
|
||||
ts = create_testsuite_by(criteria, keyword)
|
||||
|
||||
if args.list_tests_by and len(args.list_tests_by) >= 2:
|
||||
valid_options = ['name', 'class', 'module', 'id', 'tag']
|
||||
if args.list_tests_by[0] not in valid_options:
|
||||
print '--list-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.list_tests_by[0]
|
||||
return 1
|
||||
else:
|
||||
criteria = args.list_tests_by[0]
|
||||
keyword = args.list_tests_by[1:]
|
||||
list_testsuite_by(criteria, keyword)
|
||||
|
||||
if args.list_tags:
|
||||
list_tags()
|
||||
|
||||
if args.list_allclasses:
|
||||
args.list_modules = True
|
||||
|
||||
|
|
@ -195,7 +405,7 @@ def main():
|
|||
print e
|
||||
pass
|
||||
|
||||
if args.run_tests or args.run_all_tests:
|
||||
if args.run_tests or args.run_all_tests or args.run_tests_by:
|
||||
if not preflight_check():
|
||||
return 1
|
||||
|
||||
|
|
@ -235,7 +445,11 @@ def main():
|
|||
|
||||
coverage_process_start = os.environ["COVERAGE_PROCESS_START"] = coveragerc
|
||||
|
||||
testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
|
||||
if args.run_tests_by:
|
||||
testslist = ts
|
||||
else:
|
||||
testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
|
||||
|
||||
suite = unittest.TestSuite()
|
||||
loader = unittest.TestLoader()
|
||||
loader.sortTestMethodsUsing = None
|
||||
|
|
|
|||
Loading…
Reference in New Issue