bitbake: toaster: Update UI test runner
Add new runner options:
--run-all-tests: finds all tests, ignores config
--run-suite <suite> (from cfg)
Without arguments, run tests from current os section (config), e.g.:
1. ./run_toastertests
2. ./run_toastertests --run-all-tests
3. ./run_toastertests --run-suite darwin
Update toaster logging to meet QA CI requirements.
(Bitbake rev: 5685feb51fbb6d54fde6027cc765b9edd8eda65a)
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Daniel Istrate
Richard Purdie
parent
c192bd60e8
commit
72f98ba577
|
|
@ -28,60 +28,128 @@
|
|||
# put chromedriver in PATH, (e.g. /usr/bin/, bear in mind to chmod)
|
||||
# For windows host, you may put chromedriver.exe in the same directory as chrome.exe
|
||||
|
||||
|
||||
import unittest, time, re, sys, getopt, os, logging, platform
|
||||
import unittest, sys, os, platform
|
||||
import ConfigParser
|
||||
import subprocess
|
||||
import argparse
|
||||
from toaster_automation_test import toaster_cases
|
||||
|
||||
|
||||
class toaster_run_all():
|
||||
def __init__(self):
|
||||
# in case this script is called from other directory
|
||||
os.chdir(os.path.abspath(sys.path[0]))
|
||||
self.starttime = time.strptime(time.ctime())
|
||||
self.parser = ConfigParser.SafeConfigParser()
|
||||
found = self.parser.read('toaster_test.cfg')
|
||||
self.host_os = platform.system().lower()
|
||||
self.run_all_cases()
|
||||
self.collect_log()
|
||||
def get_args_parser():
|
||||
description = "Script that runs toaster auto tests."
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument('--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False,
|
||||
help='Run all tests.')
|
||||
parser.add_argument('--run-suite', required=False, dest='run_suite', default=False,
|
||||
help='run suite (defined in cfg file)')
|
||||
|
||||
def get_test_cases(self):
|
||||
# we have config groups for different os type in toaster_test.cfg
|
||||
cases_to_run = eval(self.parser.get('toaster_test_' + self.host_os, 'test_cases'))
|
||||
return cases_to_run
|
||||
return parser
|
||||
|
||||
|
||||
def run_all_cases(self):
|
||||
cases_temp = self.get_test_cases()
|
||||
for case in cases_temp:
|
||||
single_case_cmd = "python -m unittest toaster_automation_test.toaster_cases.test_" + str(case)
|
||||
print single_case_cmd
|
||||
subprocess.call(single_case_cmd, shell=True)
|
||||
def get_tests():
|
||||
testslist = []
|
||||
|
||||
def collect_log(self):
|
||||
"""
|
||||
the log files are temporarily stored in ./log/tmp/..
|
||||
After all cases are done, they should be transfered to ./log/$TIMESTAMP/
|
||||
"""
|
||||
def comple(number):
|
||||
if number < 10:
|
||||
return str(0) + str(number)
|
||||
else:
|
||||
return str(number)
|
||||
now = self.starttime
|
||||
now_str = comple(now.tm_year) + comple(now.tm_mon) + comple(now.tm_mday) + \
|
||||
comple(now.tm_hour) + comple(now.tm_min) + comple(now.tm_sec)
|
||||
log_dir = os.path.abspath(sys.path[0]) + os.sep + 'log' + os.sep + now_str
|
||||
log_tmp_dir = os.path.abspath(sys.path[0]) + os.sep + 'log' + os.sep + 'tmp'
|
||||
prefix = 'toaster_automation_test.toaster_cases'
|
||||
|
||||
for t in dir(toaster_cases):
|
||||
if t.startswith('test_'):
|
||||
testslist.append('.'.join((prefix, t)))
|
||||
|
||||
return testslist
|
||||
|
||||
|
||||
def get_tests_from_cfg(suite=None):
|
||||
|
||||
testslist = []
|
||||
config = ConfigParser.SafeConfigParser()
|
||||
config.read('toaster_test.cfg')
|
||||
|
||||
if suite is not None:
|
||||
target_suite = suite.lower()
|
||||
|
||||
# TODO: if suite is valid suite
|
||||
|
||||
else:
|
||||
target_suite = platform.system().lower()
|
||||
|
||||
try:
|
||||
tests_from_cfg = eval(config.get('toaster_test_' + target_suite, 'test_cases'))
|
||||
except:
|
||||
print 'Failed to get test cases from cfg file. Make sure the format is correct.'
|
||||
return None
|
||||
|
||||
prefix = 'toaster_automation_test.toaster_cases.test_'
|
||||
for t in tests_from_cfg:
|
||||
testslist.append(prefix + str(t))
|
||||
|
||||
return testslist
|
||||
|
||||
def main():
|
||||
|
||||
# In case this script is called from other directory
|
||||
os.chdir(os.path.abspath(sys.path[0]))
|
||||
|
||||
parser = get_args_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.run_all_tests:
|
||||
testslist = get_tests()
|
||||
elif args.run_suite:
|
||||
testslist = get_tests_from_cfg(args.run_suite)
|
||||
os.environ['TOASTER_SUITE'] = args.run_suite
|
||||
else:
|
||||
testslist = get_tests_from_cfg()
|
||||
|
||||
if not testslist:
|
||||
print 'Failed to get test cases.'
|
||||
exit(1)
|
||||
|
||||
suite = unittest.TestSuite()
|
||||
loader = unittest.TestLoader()
|
||||
loader.sortTestMethodsUsing = None
|
||||
runner = unittest.TextTestRunner(verbosity=2, resultclass=buildResultClass(args))
|
||||
|
||||
for test in testslist:
|
||||
try:
|
||||
os.renames(log_tmp_dir, log_dir)
|
||||
except OSError :
|
||||
logging.error(" Cannot create log dir(timestamp) under log, please check your privilege")
|
||||
suite.addTests(loader.loadTestsFromName(test))
|
||||
except:
|
||||
return 1
|
||||
|
||||
result = runner.run(suite)
|
||||
|
||||
if result.wasSuccessful():
|
||||
return 0
|
||||
else:
|
||||
return 1
|
||||
|
||||
|
||||
def buildResultClass(args):
|
||||
"""Build a Result Class to use in the testcase execution"""
|
||||
|
||||
class StampedResult(unittest.TextTestResult):
|
||||
"""
|
||||
Custom TestResult that prints the time when a test starts. As toaster-auto
|
||||
can take a long time (ie a few hours) to run, timestamps help us understand
|
||||
what tests are taking a long time to execute.
|
||||
"""
|
||||
def startTest(self, test):
|
||||
import time
|
||||
self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
|
||||
super(StampedResult, self).startTest(test)
|
||||
|
||||
return StampedResult
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
toaster_run_all()
|
||||
|
||||
|
||||
|
||||
try:
|
||||
ret = main()
|
||||
except:
|
||||
ret = 1
|
||||
import traceback
|
||||
traceback.print_exc(5)
|
||||
finally:
|
||||
if os.getenv('TOASTER_SUITE'):
|
||||
del os.environ['TOASTER_SUITE']
|
||||
sys.exit(ret)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -230,78 +230,98 @@ class NoParsingFilter(logging.Filter):
|
|||
def LogResults(original_class):
|
||||
orig_method = original_class.run
|
||||
|
||||
from time import strftime, gmtime
|
||||
caller = 'toaster'
|
||||
timestamp = strftime('%Y%m%d%H%M%S',gmtime())
|
||||
logfile = os.path.join(os.getcwd(),'results-'+caller+'.'+timestamp+'.log')
|
||||
linkfile = os.path.join(os.getcwd(),'results-'+caller+'.log')
|
||||
|
||||
#rewrite the run method of unittest.TestCase to add testcase logging
|
||||
def run(self, result, *args, **kws):
|
||||
orig_method(self, result, *args, **kws)
|
||||
passed = True
|
||||
testMethod = getattr(self, self._testMethodName)
|
||||
|
||||
#if test case is decorated then use it's number, else use it's name
|
||||
try:
|
||||
test_case = testMethod.test_case
|
||||
except AttributeError:
|
||||
test_case = self._testMethodName
|
||||
|
||||
class_name = str(testMethod.im_class).split("'")[1]
|
||||
|
||||
#create custom logging level for filtering.
|
||||
custom_log_level = 100
|
||||
logging.addLevelName(custom_log_level, 'RESULTS')
|
||||
caller = os.path.basename(sys.argv[0])
|
||||
|
||||
def results(self, message, *args, **kws):
|
||||
if self.isEnabledFor(custom_log_level):
|
||||
self.log(custom_log_level, message, *args, **kws)
|
||||
logging.Logger.results = results
|
||||
|
||||
logging.basicConfig(filename=os.path.join(os.getcwd(),'results-'+caller+'.log'),
|
||||
logging.basicConfig(filename=logfile,
|
||||
filemode='w',
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
datefmt='%H:%M:%S',
|
||||
level=custom_log_level)
|
||||
for handler in logging.root.handlers:
|
||||
handler.addFilter(NoParsingFilter())
|
||||
# local_log = logging.getLogger(caller)
|
||||
local_log = logging.getLogger()
|
||||
local_log = logging.getLogger(caller)
|
||||
|
||||
#check status of tests and record it
|
||||
|
||||
for (name, msg) in result.errors:
|
||||
if self._testMethodName == str(name).split(' ')[0]:
|
||||
if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]):
|
||||
local_log.results("Testcase "+str(test_case)+": ERROR")
|
||||
local_log.results("Testcase "+str(test_case)+":\n"+msg+"\n\n\n")
|
||||
local_log.results("Testcase "+str(test_case)+":\n"+msg)
|
||||
passed = False
|
||||
for (name, msg) in result.failures:
|
||||
if self._testMethodName == str(name).split(' ')[0]:
|
||||
if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]):
|
||||
local_log.results("Testcase "+str(test_case)+": FAILED")
|
||||
local_log.results("Testcase "+str(test_case)+":\n"+msg+"\n\n\n")
|
||||
local_log.results("Testcase "+str(test_case)+":\n"+msg)
|
||||
passed = False
|
||||
for (name, msg) in result.skipped:
|
||||
if self._testMethodName == str(name).split(' ')[0]:
|
||||
local_log.results("Testcase "+str(test_case)+": SKIPPED"+"\n\n\n")
|
||||
if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]):
|
||||
local_log.results("Testcase "+str(test_case)+": SKIPPED")
|
||||
passed = False
|
||||
if passed:
|
||||
local_log.results("Testcase "+str(test_case)+": PASSED"+"\n\n\n")
|
||||
local_log.results("Testcase "+str(test_case)+": PASSED")
|
||||
|
||||
# Create symlink to the current log
|
||||
if os.path.exists(linkfile):
|
||||
os.remove(linkfile)
|
||||
os.symlink(logfile, linkfile)
|
||||
|
||||
original_class.run = run
|
||||
|
||||
return original_class
|
||||
|
||||
|
||||
|
||||
|
||||
###########################################
|
||||
# #
|
||||
# PART II: base class #
|
||||
# #
|
||||
###########################################
|
||||
|
||||
@LogResults
|
||||
class toaster_cases_base(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.log = cls.logger_create()
|
||||
|
||||
def setUp(self):
|
||||
self.screenshot_sequence = 1
|
||||
self.verificationErrors = []
|
||||
self.accept_next_alert = True
|
||||
self.host_os = platform.system().lower()
|
||||
if os.getenv('TOASTER_SUITE'):
|
||||
self.target_suite = os.getenv('TOASTER_SUITE')
|
||||
else:
|
||||
self.target_suite = self.host_os
|
||||
|
||||
self.parser = ConfigParser.SafeConfigParser()
|
||||
configs = self.parser.read('toaster_test.cfg')
|
||||
self.base_url = eval(self.parser.get('toaster_test_' + self.host_os, 'toaster_url'))
|
||||
self.parser.read('toaster_test.cfg')
|
||||
self.base_url = eval(self.parser.get('toaster_test_' + self.target_suite, 'toaster_url'))
|
||||
|
||||
# create log dir . Currently , we put log files in log/tmp. After all
|
||||
# test cases are done, move them to log/$datetime dir
|
||||
|
|
@ -310,37 +330,37 @@ class toaster_cases_base(unittest.TestCase):
|
|||
mkdir_p(self.log_tmp_dir)
|
||||
except OSError :
|
||||
logging.error("%(asctime)s Cannot create tmp dir under log, please check your privilege")
|
||||
self.log = self.logger_create()
|
||||
# self.log = self.logger_create()
|
||||
# driver setup
|
||||
self.setup_browser()
|
||||
|
||||
def logger_create(self):
|
||||
"""
|
||||
we use root logger for every testcase.
|
||||
The reason why we don't use TOASTERXXX_logger is to avoid setting respective level for
|
||||
root logger and TOASTERXXX_logger
|
||||
To Be Discussed
|
||||
"""
|
||||
log_level_dict = {'CRITICAL':logging.CRITICAL, 'ERROR':logging.ERROR, 'WARNING':logging.WARNING, \
|
||||
'INFO':logging.INFO, 'DEBUG':logging.DEBUG, 'NOTSET':logging.NOTSET}
|
||||
log = logging.getLogger()
|
||||
# log = logging.getLogger('TOASTER_' + str(self.case_no))
|
||||
self.logging_level = eval(self.parser.get('toaster_test_' + self.host_os, 'logging_level'))
|
||||
key = self.logging_level.upper()
|
||||
log.setLevel(log_level_dict[key])
|
||||
fh = logging.FileHandler(filename=self.log_tmp_dir + os.sep + 'case_all' + '.log', mode='a')
|
||||
@staticmethod
|
||||
def logger_create():
|
||||
log_file = "toaster-auto-" + time.strftime("%Y%m%d%H%M%S") + ".log"
|
||||
if os.path.exists("toaster-auto.log"): os.remove("toaster-auto.log")
|
||||
os.symlink(log_file, "toaster-auto.log")
|
||||
|
||||
log = logging.getLogger("toaster")
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
fh = logging.FileHandler(filename=log_file, mode='w')
|
||||
fh.setLevel(logging.DEBUG)
|
||||
|
||||
ch = logging.StreamHandler(sys.stdout)
|
||||
formatter = logging.Formatter('%(pathname)s - %(lineno)d - %(asctime)s \n \
|
||||
%(name)s - %(levelname)s - %(message)s')
|
||||
ch.setLevel(logging.INFO)
|
||||
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
fh.setFormatter(formatter)
|
||||
ch.setFormatter(formatter)
|
||||
|
||||
log.addHandler(fh)
|
||||
log.addHandler(ch)
|
||||
|
||||
return log
|
||||
|
||||
|
||||
def setup_browser(self, *browser_path):
|
||||
self.browser = eval(self.parser.get('toaster_test_' + self.host_os, 'test_browser'))
|
||||
self.browser = eval(self.parser.get('toaster_test_' + self.target_suite, 'test_browser'))
|
||||
print self.browser
|
||||
if self.browser == "firefox":
|
||||
driver = webdriver.Firefox()
|
||||
|
|
@ -660,7 +680,7 @@ class toaster_cases_base(unittest.TestCase):
|
|||
# Note: to comply with the unittest framework, we call these test_xxx functions
|
||||
# from run_toastercases.py to avoid calling setUp() and tearDown() multiple times
|
||||
|
||||
@LogResults
|
||||
|
||||
class toaster_cases(toaster_cases_base):
|
||||
##############
|
||||
# CASE 901 #
|
||||
|
|
|
|||
Loading…
Reference in New Issue