mirror of
https://git.yoctoproject.org/poky
synced 2026-02-14 12:43:03 +01:00
1fd5ebdb06
OEQA outputs test results into json files and these files were
archived by Autobuilder during QA releases. Example: each oe-selftest
run by Autobuilder for different host distro generate a
testresults.json file.
These scripts were developed as a test result tools to manage
these testresults.json file.
Using the "store" operation, user can store multiple testresults.json
files as well as the pre-configured directories used to hold those files.
Using the "merge" operation, user can merge multiple testresults.json
files to a target file.
Using the "report" operation, user can view the test result summary
for all available testresults.json files inside a ordinary directory
or a git repository.
Using the "regression-file" operation, user can perform regression
analysis on testresults.json files specified. Using the "regression-dir"
and "regression-git" operations, user can perform regression analysis
on directory and git accordingly.
These resulttool operations expect the testresults.json file to use
the json format below.
{
"<testresult_1>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
...
"<testresult_n>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
}
To use these scripts, first source oe environment, then run the
entry point script to look for help.
$ resulttool
To store test result from oeqa automated tests, execute the below
$ resulttool store <source_dir> <git_branch>
To merge multiple testresults.json files, execute the below
$ resulttool merge <base_result_file> <target_result_file>
To report test report, execute the below
$ resulttool report <source_dir>
To perform regression file analysis, execute the below
$ resulttool regression-file <base_result_file> <target_result_file>
To perform regression dir analysis, execute the below
$ resulttool regression-dir <base_result_dir> <target_result_dir>
To perform regression git analysis, execute the below
$ resulttool regression-git <source_dir> <base_branch> <target_branch>
[YOCTO# 13012]
[YOCTO# 12654]
(From OE-Core rev: 78a322d7be402a5b9b5abf26ad35670a8535408a)
Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
72 lines
3.3 KiB
Python
72 lines
3.3 KiB
Python
# test result tool - merge multiple testresults.json files
|
|
#
|
|
# Copyright (c) 2019, Intel Corporation.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify it
|
|
# under the terms and conditions of the GNU General Public License,
|
|
# version 2, as published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope it will be useful, but WITHOUT
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
# more details.
|
|
#
|
|
from resulttool.resultsutils import load_json_file, get_dict_value, dump_json_data
|
|
import os
|
|
import json
|
|
|
|
class ResultsMerge(object):
|
|
|
|
def get_test_results(self, logger, file, result_id):
|
|
results = load_json_file(file)
|
|
if result_id:
|
|
result = get_dict_value(logger, results, result_id)
|
|
if result:
|
|
return {result_id: result}
|
|
return result
|
|
return results
|
|
|
|
def merge_results(self, base_results, target_results):
|
|
for k in target_results:
|
|
base_results[k] = target_results[k]
|
|
return base_results
|
|
|
|
def _get_write_dir(self):
|
|
basepath = os.environ['BUILDDIR']
|
|
return basepath + '/tmp/'
|
|
|
|
def dump_merged_results(self, results, output_dir):
|
|
file_output_dir = output_dir if output_dir else self._get_write_dir()
|
|
dump_json_data(file_output_dir, 'testresults.json', results)
|
|
print('Successfully merged results to: %s' % os.path.join(file_output_dir, 'testresults.json'))
|
|
|
|
def run(self, logger, base_result_file, target_result_file, target_result_id, output_dir):
|
|
base_results = self.get_test_results(logger, base_result_file, '')
|
|
target_results = self.get_test_results(logger, target_result_file, target_result_id)
|
|
if base_results and target_results:
|
|
merged_results = self.merge_results(base_results, target_results)
|
|
self.dump_merged_results(merged_results, output_dir)
|
|
|
|
def merge(args, logger):
|
|
merge = ResultsMerge()
|
|
merge.run(logger, args.base_result_file, args.target_result_file, args.target_result_id, args.output_dir)
|
|
return 0
|
|
|
|
def register_commands(subparsers):
|
|
"""Register subcommands from this plugin"""
|
|
parser_build = subparsers.add_parser('merge', help='merge test results',
|
|
description='merge results from multiple files',
|
|
group='setup')
|
|
parser_build.set_defaults(func=merge)
|
|
parser_build.add_argument('base_result_file',
|
|
help='base result file provide the base result set')
|
|
parser_build.add_argument('target_result_file',
|
|
help='target result file provide the target result set for merging into the '
|
|
'base result set')
|
|
parser_build.add_argument('-t', '--target-result-id', default='',
|
|
help='(optional) default merge all result sets available from target to base '
|
|
'unless specific target result id was provided')
|
|
parser_build.add_argument('-o', '--output-dir', default='',
|
|
help='(optional) default write merged results to <poky>/build/tmp/ unless specific '
|
|
'output directory was provided')
|