test: add automation framework and sanitytest
Automation test is disabled by default. User need set TESTCLASS to qemu in conf/local.conf and run bitbake command "bitbake poky-image-xxx" or "bitbake poky-image-xxx -c qemuimagetest" to trigger it. Currently only the sanity test with two testcases are added. To run the test, user need prepare a testing environment: 1) "expect" should be installed on system 2) NOPASSWD should be set for user to run bitbake Signed-off-by Jiajun Xu <jiajun.xu@intel.com>
This commit is contained in:
parent
bd19169a60
commit
fd1d661b39
|
@ -153,3 +153,11 @@ ENABLE_BINARY_LOCALE_GENERATION = "1"
|
||||||
# Poky can try and fetch packaged-staging packages from a http, https or ftp
|
# Poky can try and fetch packaged-staging packages from a http, https or ftp
|
||||||
# mirror. Set this variable to the root of a pstage directory on a server.
|
# mirror. Set this variable to the root of a pstage directory on a server.
|
||||||
#PSTAGE_MIRROR ?= "http://someserver.tld/share/pstage"
|
#PSTAGE_MIRROR ?= "http://someserver.tld/share/pstage"
|
||||||
|
|
||||||
|
# Set IMAGETEST to qemu if you want to build testcases and start
|
||||||
|
# testing in qemu after do_rootfs.
|
||||||
|
#IMAGETEST="qemu"
|
||||||
|
|
||||||
|
# By default testing will run the sanitytest suite. If you want to run other tests
|
||||||
|
# (e.g. bat), list them here
|
||||||
|
#TEST_SCEN="sanitytest bat"
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
inherit rootfs_${IMAGE_PKGTYPE}
|
inherit rootfs_${IMAGE_PKGTYPE}
|
||||||
|
|
||||||
|
IMAGETEST ?= "dummy"
|
||||||
|
inherit imagetest-${IMAGETEST}
|
||||||
|
|
||||||
LICENSE = "MIT"
|
LICENSE = "MIT"
|
||||||
PACKAGES = ""
|
PACKAGES = ""
|
||||||
RDEPENDS += "${IMAGE_INSTALL}"
|
RDEPENDS += "${IMAGE_INSTALL}"
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
# dummy testclass file
|
|
@ -0,0 +1,137 @@
|
||||||
|
addtask qemuimagetest before do_build
|
||||||
|
# after do_rootfs
|
||||||
|
do_qemuimagetest[nostamp] = "1"
|
||||||
|
do_qemuimagetest[depends] += "qemu-native:do_populate_sysroot"
|
||||||
|
|
||||||
|
# Test related variables
|
||||||
|
# By default, TEST_DIR is created under WORKDIR
|
||||||
|
TEST_DIR ?= "${WORKDIR}/qemuimagetest"
|
||||||
|
TEST_LOG ?= "${LOG_DIR}/qemuimagetests"
|
||||||
|
TEST_RESULT ?= "${TEST_DIR}/result"
|
||||||
|
TEST_LIST ?= "${TEST_DIR}/list"
|
||||||
|
TEST_TMP ?= "${TEST_DIR}/tmp"
|
||||||
|
TEST_SCEN ?= "sanity"
|
||||||
|
|
||||||
|
python do_qemuimagetest() {
|
||||||
|
import sys
|
||||||
|
import re
|
||||||
|
import os
|
||||||
|
|
||||||
|
"""
|
||||||
|
Test Controller for Poky Testing.
|
||||||
|
"""
|
||||||
|
|
||||||
|
casestr = re.compile(r'(?P<scen>\w+\b)\s*(?P<case>\w+$)')
|
||||||
|
resultstr = re.compile(r'\s*(?P<case>\w+)\s*(?P<pass>\d+)\s*(?P<fail>\d+)\s*(?P<noresult>\d+)')
|
||||||
|
|
||||||
|
"""funtion to run each case under scenario"""
|
||||||
|
def runtest(scen, case, fulltestpath):
|
||||||
|
resultpath = bb.data.getVar('TEST_RESULT', d, 1)
|
||||||
|
machine = bb.data.getVar('MACHINE', d, 1)
|
||||||
|
pname = bb.data.getVar('PN', d, 1)
|
||||||
|
|
||||||
|
testpath = bb.data.getVar('TEST_DIR', d, 1)
|
||||||
|
|
||||||
|
"""initialize log file for testcase"""
|
||||||
|
logpath = bb.data.getVar('TEST_LOG', d, 1)
|
||||||
|
bb.utils.mkdirhier("%s/%s" % (logpath, scen))
|
||||||
|
caselog = os.path.join(logpath, "%s/log_%s.%s" % (scen, case, bb.data.getVar('DATETIME', d, 1)))
|
||||||
|
os.system("touch %s" % caselog)
|
||||||
|
|
||||||
|
|
||||||
|
"""export TEST_TMP, TEST_RESULT, DEPLOY_DIR and QEMUARCH"""
|
||||||
|
os.environ["PATH"] = bb.data.getVar("PATH", d, True)
|
||||||
|
os.environ["TEST_TMP"] = testpath
|
||||||
|
os.environ["TEST_RESULT"] = resultpath
|
||||||
|
os.environ["DEPLOY_DIR"] = bb.data.getVar("DEPLOY_DIR", d, True)
|
||||||
|
os.environ["QEMUARCH"] = machine
|
||||||
|
os.environ["QEMUTARGET"] = pname
|
||||||
|
os.environ["DISPLAY"] = bb.data.getVar("DISPLAY", d, True)
|
||||||
|
|
||||||
|
"""run Test Case"""
|
||||||
|
bb.note("Run %s test in scenario %s" % (case, scen))
|
||||||
|
os.system("%s | tee -a %s" % (fulltestpath, caselog))
|
||||||
|
|
||||||
|
"""Generate testcase list in runtime"""
|
||||||
|
def generate_list(testfile, testlist):
|
||||||
|
list = []
|
||||||
|
if len(testlist) == 0:
|
||||||
|
raise bb.build.FuncFailed("No testcase defined in TEST_SCEN")
|
||||||
|
|
||||||
|
"""remove old testcase list file"""
|
||||||
|
if os.path.exists(testfile):
|
||||||
|
os.remove(testfile)
|
||||||
|
os.system("touch %s" % testfile)
|
||||||
|
|
||||||
|
"""check testcase folder and add case to TEST_LIST"""
|
||||||
|
for item in testlist.split(" "):
|
||||||
|
found = False
|
||||||
|
for dir in bb.data.getVar("QEMUIMAGETESTS", d, True).split():
|
||||||
|
casepath = os.path.join(dir, item)
|
||||||
|
if not os.path.isdir(casepath):
|
||||||
|
continue
|
||||||
|
files = os.listdir(casepath)
|
||||||
|
for casefile in files:
|
||||||
|
fulltestcase = "%s/%s" % (casepath, casefile)
|
||||||
|
if os.path.isfile(fulltestcase):
|
||||||
|
list.append((item, casefile, fulltestcase))
|
||||||
|
found = True
|
||||||
|
if not found:
|
||||||
|
raise bb.build.FuncFailed("Testcase folder not found for test %s" % item)
|
||||||
|
return list
|
||||||
|
|
||||||
|
"""check testcase folder and create test log folder"""
|
||||||
|
testpath = bb.data.getVar('TEST_DIR', d, 1)
|
||||||
|
bb.utils.mkdirhier(testpath)
|
||||||
|
|
||||||
|
logpath = bb.data.getVar('TEST_LOG', d, 1)
|
||||||
|
bb.utils.mkdirhier(logpath)
|
||||||
|
|
||||||
|
tmppath = bb.data.getVar('TEST_TMP', d, 1)
|
||||||
|
bb.utils.mkdirhier(tmppath)
|
||||||
|
|
||||||
|
"""initialize result file"""
|
||||||
|
resultpath = bb.data.getVar('TEST_RESULT', d, 1)
|
||||||
|
bb.utils.mkdirhier(resultpath)
|
||||||
|
resultfile = os.path.join(resultpath, "testresult.%s" % bb.data.getVar('DATETIME', d, 1))
|
||||||
|
sresultfile = os.path.join(resultpath, "testresult.log")
|
||||||
|
|
||||||
|
machine = bb.data.getVar('MACHINE', d, 1)
|
||||||
|
|
||||||
|
if os.path.exists(sresultfile):
|
||||||
|
os.remove(sresultfile)
|
||||||
|
os.system("touch %s" % resultfile)
|
||||||
|
os.symlink(resultfile, sresultfile)
|
||||||
|
f = open(sresultfile, "a")
|
||||||
|
f.write("\tTest Result for %s\n" % machine)
|
||||||
|
f.write("\tTestcase\tPASS\tFAIL\tNORESULT\n")
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
"""generate pre-defined testcase list"""
|
||||||
|
testfile = bb.data.getVar('TEST_LIST', d, 1)
|
||||||
|
testlist = bb.data.getVar('TEST_SCEN', d, 1)
|
||||||
|
fulllist = generate_list(testfile, testlist)
|
||||||
|
|
||||||
|
"""Begin testing"""
|
||||||
|
for test in fulllist:
|
||||||
|
(scen, case, fullpath) = test
|
||||||
|
runtest(scen, case, fullpath)
|
||||||
|
|
||||||
|
"""Print Test Result"""
|
||||||
|
ret = 0
|
||||||
|
f = open(sresultfile, "r")
|
||||||
|
for line in f:
|
||||||
|
m = resultstr.match(line)
|
||||||
|
if m:
|
||||||
|
if m.group('fail') == "1":
|
||||||
|
ret = 1
|
||||||
|
elif m.group('noresult') == "1":
|
||||||
|
ret = 2
|
||||||
|
print line,
|
||||||
|
else:
|
||||||
|
print line,
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
if ret != 0:
|
||||||
|
raise bb.build.FuncFailed("Some testcases fail, pls. check test result and test log!!!")
|
||||||
|
}
|
|
@ -8,5 +8,12 @@ BBFILE_COLLECTIONS += "normal"
|
||||||
BBFILE_PATTERN_normal := "^${LAYERDIR}/"
|
BBFILE_PATTERN_normal := "^${LAYERDIR}/"
|
||||||
BBFILE_PRIORITY_normal = "5"
|
BBFILE_PRIORITY_normal = "5"
|
||||||
|
|
||||||
|
# Add scripts to PATH
|
||||||
|
PATH := "${PATH}:${LAYERDIR}/scripts"
|
||||||
|
|
||||||
|
# Set path to qemu image tests included in this layer
|
||||||
|
|
||||||
|
QEMUIMAGETESTS := "${OEROOT}/scripts/qemuimage-tests"
|
||||||
|
|
||||||
require conf/distro/include/poky-default-revisions.inc
|
require conf/distro/include/poky-default-revisions.inc
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,202 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Common function for test
|
||||||
|
# Expect should be installed for SSH Testing
|
||||||
|
# To execute `runqemu`, NOPASSWD needs to be set in /etc/sudoers for user
|
||||||
|
# For example, for user "builder", /etc/sudoers can be like following:
|
||||||
|
# #########
|
||||||
|
# #Members of the admin group may gain root privileges
|
||||||
|
# %builder ALL=(ALL) NOPASSWD: NOPASSWD: ALL
|
||||||
|
# #########
|
||||||
|
#
|
||||||
|
# Author: Jiajun Xu <jiajun.xu@intel.com>
|
||||||
|
#
|
||||||
|
# This file is licensed under the GNU General Public License,
|
||||||
|
# Version 2.
|
||||||
|
#
|
||||||
|
|
||||||
|
TYPE="ext3"
|
||||||
|
|
||||||
|
# common function for information print
|
||||||
|
Test_Error()
|
||||||
|
{
|
||||||
|
echo -e "\tTest_Error: $*"
|
||||||
|
}
|
||||||
|
|
||||||
|
Test_Info()
|
||||||
|
{
|
||||||
|
echo -e "\tTest_Info: $*"
|
||||||
|
}
|
||||||
|
|
||||||
|
# function to run command in $ip_addr via ssh
|
||||||
|
Test_SSH()
|
||||||
|
{
|
||||||
|
local ip_addr=$1
|
||||||
|
shift
|
||||||
|
local command=$@
|
||||||
|
local tmpfile=`mktemp`
|
||||||
|
local timeout=60
|
||||||
|
local ret=0
|
||||||
|
local exp_cmd=`cat << EOF
|
||||||
|
eval spawn ssh -o UserKnownHostsFile=$tmpfile root@$ip_addr "$command"
|
||||||
|
set timeout $time_out
|
||||||
|
expect {
|
||||||
|
"*assword:" { send "\r"; exp_continue}
|
||||||
|
"*(yes/no)?" { send "yes\r"; exp_continue }
|
||||||
|
eof { exit [ lindex [wait] 3 ] }
|
||||||
|
}
|
||||||
|
EOF`
|
||||||
|
expect -c "$exp_cmd"
|
||||||
|
ret=$?
|
||||||
|
rm -rf $tmpfile
|
||||||
|
return $ret
|
||||||
|
}
|
||||||
|
|
||||||
|
# function to check if ssh is up in $ip_addr
|
||||||
|
Test_SSH_UP()
|
||||||
|
{
|
||||||
|
local ip_addr=$1
|
||||||
|
local timeout=$2
|
||||||
|
local interval=0
|
||||||
|
|
||||||
|
while [ ${interval} -lt ${timeout} ]
|
||||||
|
do
|
||||||
|
Test_SSH ${ip_addr} "hostname"
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
interval=`expr $interval + 10`
|
||||||
|
sleep 10
|
||||||
|
else
|
||||||
|
Test_Info "We can ssh on ${ip_addr} now"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
done
|
||||||
|
|
||||||
|
Test_Info "We can not ssh on ${ip_addr} in ${timeout}"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# function to record test result in $TEST_RESULT/testresult.log
|
||||||
|
Test_Print_Result()
|
||||||
|
{
|
||||||
|
local PASS=0
|
||||||
|
local FAIL=0
|
||||||
|
local NORESULT=0
|
||||||
|
if [ $2 -eq 0 ]; then
|
||||||
|
PASS=1
|
||||||
|
elif [ $2 -eq 1 ]; then
|
||||||
|
FAIL=1
|
||||||
|
else
|
||||||
|
NORESULT=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo -e "\t$1\t\t$PASS\t$FAIL\t$NORESULT" >> $TEST_RESULT/testresult.log
|
||||||
|
}
|
||||||
|
|
||||||
|
# function to kill qemu
|
||||||
|
Test_Kill_Qemu()
|
||||||
|
{
|
||||||
|
sudo pkill -9 qemu
|
||||||
|
return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
# function to check if there is any qemu process
|
||||||
|
Test_Check_Qemu_UP()
|
||||||
|
{
|
||||||
|
local count=`ps -ef | grep -c qemu`
|
||||||
|
if [ ${count} -lt 2 ]; then
|
||||||
|
Test_Info "There is no Qemu process"
|
||||||
|
return 1
|
||||||
|
else
|
||||||
|
Test_Info "There is at least Qemu process running"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# function to check if network is up
|
||||||
|
Test_Check_IP_UP()
|
||||||
|
{
|
||||||
|
ping -c1 $1
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
Test_Info "IP $1 is not up"
|
||||||
|
return 1
|
||||||
|
else
|
||||||
|
Test_Info "IP $1 is up"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# function to check if qemu and its network
|
||||||
|
Test_Create_Qemu()
|
||||||
|
{
|
||||||
|
local ret=1
|
||||||
|
local up_time=0
|
||||||
|
local ip_addr=$1
|
||||||
|
local timeout=$2
|
||||||
|
local kernel=""
|
||||||
|
local rootfs=""
|
||||||
|
|
||||||
|
which poky-qemu
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
RUNQEMU=`which poky-qemu`
|
||||||
|
else
|
||||||
|
Test_Error "Can not find poky-qemu in \$PATH, return fail"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$QEMUARCH" = "qemux86" ]; then
|
||||||
|
KERNEL=${DEPLOY_DIR}/images/bzImage-${QEMUARCH}.bin
|
||||||
|
elif [ "$QEMUARCH" = "qemuarm" -o "$QEMUARCH" = "spitz" -o "$QEMUARCH" = "borzoi" -o "$QEMUARCH" = "akita" -o "$QEMUARCH" = "nokia800" ]; then
|
||||||
|
KERNEL=${DEPLOY_DIR}/images/zImage-${QEMUARCH}.bin
|
||||||
|
fi
|
||||||
|
|
||||||
|
ROOTFS_IMAGE=`readlink -f ${DEPLOY_DIR}/images/${QEMUTARGET}-${QEMUARCH}.ext3`
|
||||||
|
TEST_ROOTFS_IMAGE="${TEST_TMP}/${QEMUTARGET}-${QEMUARCH}-test.ext3"
|
||||||
|
|
||||||
|
CP=`which cp`
|
||||||
|
if [ -e "$TEST_ROOTFS_IMAGE" ]; then
|
||||||
|
rm -rf $TEST_ROOTFS_IMAGE
|
||||||
|
fi
|
||||||
|
$CP $ROOTFS_IMAGE $TEST_ROOTFS_IMAGE
|
||||||
|
|
||||||
|
export MACHINE=$QEMUARCH
|
||||||
|
# Create Qemu in localhost VNC Port 1
|
||||||
|
|
||||||
|
xterm -display ${DISPLAY} -e "${RUNQEMU} ${KERNEL} ${TEST_ROOTFS_IMAGE}" &
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
while [ ${up_time} -lt ${timeout} ]
|
||||||
|
do
|
||||||
|
Test_Check_Qemu_UP
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
Test_Info "Wait for qemu up..."
|
||||||
|
up_time=`expr $up_time + 5`
|
||||||
|
sleep 5
|
||||||
|
else
|
||||||
|
Test_Info "Begin to check if qemu network is up"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
while [ ${up_time} -lt ${timeout} ]
|
||||||
|
do
|
||||||
|
Test_Check_IP_UP ${ip_addr}
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
Test_Info "Qemu Network is up, ping with ${ip_addr} is OK"
|
||||||
|
ret=0
|
||||||
|
break
|
||||||
|
else
|
||||||
|
Test_Info "Wait for Qemu Network up"
|
||||||
|
up_time=`expr $up_time + 5`
|
||||||
|
sleep 5
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ $ret -eq 0 ]; then
|
||||||
|
Test_Info "Qemu and its network is up"
|
||||||
|
return $ret
|
||||||
|
else
|
||||||
|
Test_Info "Qemu or its network is not up in ${timeout}"
|
||||||
|
return $ret
|
||||||
|
fi
|
||||||
|
}
|
|
@ -0,0 +1,31 @@
|
||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Boot Test Case for Sanity Test
|
||||||
|
# The case boot up the Qemu target with `runqemu qemux86`.
|
||||||
|
# Then check if qemu and qemu network is up.
|
||||||
|
#
|
||||||
|
# Author: Jiajun Xu <jiajun.xu@intel.com>
|
||||||
|
#
|
||||||
|
# This file is licensed under the GNU General Public License,
|
||||||
|
# Version 2.
|
||||||
|
#
|
||||||
|
|
||||||
|
. $OEROOT/scripts/qemuimage-testlib
|
||||||
|
|
||||||
|
TIMEOUT=120
|
||||||
|
QEMU_IPADDR="192.168.7.2"
|
||||||
|
|
||||||
|
# Start qemu and check its network
|
||||||
|
Test_Create_Qemu ${QEMU_IPADDR} ${TIMEOUT}
|
||||||
|
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
Test_Info "Boot Test PASS"
|
||||||
|
Test_Kill_Qemu
|
||||||
|
Test_Print_Result "Boot" 0
|
||||||
|
exit 0
|
||||||
|
else
|
||||||
|
Test_Info "Boot Test FAIL"
|
||||||
|
Test_Kill_Qemu
|
||||||
|
Test_Print_Result "Boot" 1
|
||||||
|
exit 1
|
||||||
|
fi
|
|
@ -0,0 +1,40 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# SSH Test Case for Sanity Test
|
||||||
|
# The case boot up the Qemu target with `runqemu qemux86`.
|
||||||
|
# Then check if ssh service in qemu is up.
|
||||||
|
#
|
||||||
|
# Author: Jiajun Xu <jiajun.xu@intel.com>
|
||||||
|
#
|
||||||
|
# This file is licensed under the GNU General Public License,
|
||||||
|
# Version 2.
|
||||||
|
#
|
||||||
|
|
||||||
|
. $OEROOT/scripts/qemuimage-testlib
|
||||||
|
|
||||||
|
TIMEOUT=360
|
||||||
|
QEMU_IPADDR="192.168.7.2"
|
||||||
|
RET=1
|
||||||
|
|
||||||
|
# Start qemu and check its network
|
||||||
|
Test_Create_Qemu ${QEMU_IPADDR} ${TIMEOUT}
|
||||||
|
|
||||||
|
# If qemu network is up, check ssh service in qemu
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
Test_Info "Begin to Test SSH Service in Qemu"
|
||||||
|
Test_SSH_UP ${QEMU_IPADDR} ${TIMEOUT}
|
||||||
|
RET=$?
|
||||||
|
else
|
||||||
|
RET=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${RET} -eq 0 ]; then
|
||||||
|
Test_Info "SSH Test PASS"
|
||||||
|
Test_Kill_Qemu
|
||||||
|
Test_Print_Result "SSH" 0
|
||||||
|
exit 0
|
||||||
|
else
|
||||||
|
Test_Info "SSH Test FAIL"
|
||||||
|
Test_Kill_Qemu
|
||||||
|
Test_Print_Result "SSH" 1
|
||||||
|
exit 1
|
||||||
|
fi
|
Loading…
Reference in New Issue