mirror of
https://git.yoctoproject.org/poky
synced 2026-04-28 15:32:27 +02:00
c6743c05a3
Since commit d6065f136f6d ("oeqa/logparser: Various misc cleanups"),
7b17274c30 in poky, the ptest OEQA is unable to detect failures
in any of the test results.
The reason is that the test result string changed from 'fail' to
'FAILED', because the original mapping has been removed as part of
that commit, but the code in here is still trying to match against
the old string, resulting in no matches, i.e. everything is treated
as successful, even if it shouldn't be.
Update the OEQA ptest test to actually work again and report
failure if there was a failure.
Note that the ptest test is marked as @expectedfail, so even though
this test now again starts to fail, the overall OEQA test result is
not affected - but at least the overall OEQA test summary reflects
the correct status again.
In other words:
RESULTS:
RESULTS - ping.PingTest.test_ping: PASSED (0.26s)
RESULTS - ptest.PtestRunnerTest.test_ptestrunner: PASSED (4.05s)
RESULTS - ssh.SSHTest.test_ssh: PASSED (0.60s)
SUMMARY:
image-debug () - Ran 3 tests in 4.937s
correctly changes to:
AssertionError: Failed ptests:
{'dummytest': ['check_True_is_True', 'test_basic']}
RESULTS:
RESULTS - ping.PingTest.test_ping: PASSED (0.24s)
RESULTS - ssh.SSHTest.test_ssh: PASSED (0.56s)
RESULTS - ptest.PtestRunnerTest.test_ptestrunner: EXPECTEDFAIL (4.13s)
SUMMARY:
image-debug () - Ran 3 tests in 4.937s
instead and we see a summary of the ptest subtests that failed.
(From OE-Core rev: 08ff4b9195a9de91b8090173c5bd03a5ff263616)
Signed-off-by: André Draszik <git@andred.net>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
85 lines
3.3 KiB
Python
85 lines
3.3 KiB
Python
#
|
|
# SPDX-License-Identifier: MIT
|
|
#
|
|
|
|
import unittest
|
|
import pprint
|
|
import datetime
|
|
|
|
from oeqa.runtime.case import OERuntimeTestCase
|
|
from oeqa.core.decorator.depends import OETestDepends
|
|
from oeqa.core.decorator.data import skipIfNotFeature
|
|
from oeqa.runtime.decorator.package import OEHasPackage
|
|
from oeqa.utils.logparser import PtestParser
|
|
|
|
class PtestRunnerTest(OERuntimeTestCase):
|
|
|
|
@skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES')
|
|
@OETestDepends(['ssh.SSHTest.test_ssh'])
|
|
@OEHasPackage(['ptest-runner'])
|
|
@unittest.expectedFailure
|
|
def test_ptestrunner(self):
|
|
status, output = self.target.run('which ptest-runner', 0)
|
|
if status != 0:
|
|
self.skipTest("No -ptest packages are installed in the image")
|
|
|
|
test_log_dir = self.td.get('TEST_LOG_DIR', '')
|
|
# The TEST_LOG_DIR maybe NULL when testimage is added after
|
|
# testdata.json is generated.
|
|
if not test_log_dir:
|
|
test_log_dir = os.path.join(self.td.get('WORKDIR', ''), 'testimage')
|
|
# Don't use self.td.get('DATETIME'), it's from testdata.json, not
|
|
# up-to-date, and may cause "File exists" when re-reun.
|
|
timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
|
|
ptest_log_dir_link = os.path.join(test_log_dir, 'ptest_log')
|
|
ptest_log_dir = '%s.%s' % (ptest_log_dir_link, timestamp)
|
|
ptest_runner_log = os.path.join(ptest_log_dir, 'ptest-runner.log')
|
|
|
|
status, output = self.target.run('ptest-runner', 0)
|
|
os.makedirs(ptest_log_dir)
|
|
with open(ptest_runner_log, 'w') as f:
|
|
f.write(output)
|
|
|
|
# status != 0 is OK since some ptest tests may fail
|
|
self.assertTrue(status != 127, msg="Cannot execute ptest-runner!")
|
|
|
|
if not hasattr(self.tc, "extraresults"):
|
|
self.tc.extraresults = {}
|
|
extras = self.tc.extraresults
|
|
extras['ptestresult.rawlogs'] = {'log': output}
|
|
|
|
# Parse and save results
|
|
parser = PtestParser()
|
|
results, sections = parser.parse(ptest_runner_log)
|
|
parser.results_as_files(ptest_log_dir)
|
|
if os.path.exists(ptest_log_dir_link):
|
|
# Remove the old link to create a new one
|
|
os.remove(ptest_log_dir_link)
|
|
os.symlink(os.path.basename(ptest_log_dir), ptest_log_dir_link)
|
|
|
|
extras['ptestresult.sections'] = sections
|
|
|
|
trans = str.maketrans("()", "__")
|
|
for section in results:
|
|
for test in results[section]:
|
|
result = results[section][test]
|
|
testname = "ptestresult." + (section or "No-section") + "." + "_".join(test.translate(trans).split())
|
|
extras[testname] = {'status': result}
|
|
|
|
failed_tests = {}
|
|
for section in results:
|
|
failed_testcases = [ "_".join(test.translate(trans).split()) for test in results[section] if results[section][test] == 'FAILED' ]
|
|
if failed_testcases:
|
|
failed_tests[section] = failed_testcases
|
|
|
|
failmsg = ""
|
|
status, output = self.target.run('dmesg | grep "Killed process"', 0)
|
|
if output:
|
|
failmsg = "ERROR: Processes were killed by the OOM Killer:\n%s\n" % output
|
|
|
|
if failed_tests:
|
|
failmsg = failmsg + "Failed ptests:\n%s" % pprint.pformat(failed_tests)
|
|
|
|
if failmsg:
|
|
self.fail(failmsg)
|