mirror of
https://git.yoctoproject.org/poky
synced 2026-02-08 18:02:12 +01:00
1fd5ebdb06
OEQA outputs test results into json files and these files were
archived by Autobuilder during QA releases. Example: each oe-selftest
run by Autobuilder for different host distro generate a
testresults.json file.
These scripts were developed as a test result tools to manage
these testresults.json file.
Using the "store" operation, user can store multiple testresults.json
files as well as the pre-configured directories used to hold those files.
Using the "merge" operation, user can merge multiple testresults.json
files to a target file.
Using the "report" operation, user can view the test result summary
for all available testresults.json files inside a ordinary directory
or a git repository.
Using the "regression-file" operation, user can perform regression
analysis on testresults.json files specified. Using the "regression-dir"
and "regression-git" operations, user can perform regression analysis
on directory and git accordingly.
These resulttool operations expect the testresults.json file to use
the json format below.
{
"<testresult_1>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
...
"<testresult_n>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
}
To use these scripts, first source oe environment, then run the
entry point script to look for help.
$ resulttool
To store test result from oeqa automated tests, execute the below
$ resulttool store <source_dir> <git_branch>
To merge multiple testresults.json files, execute the below
$ resulttool merge <base_result_file> <target_result_file>
To report test report, execute the below
$ resulttool report <source_dir>
To perform regression file analysis, execute the below
$ resulttool regression-file <base_result_file> <target_result_file>
To perform regression dir analysis, execute the below
$ resulttool regression-dir <base_result_dir> <target_result_dir>
To perform regression git analysis, execute the below
$ resulttool regression-git <source_dir> <base_branch> <target_branch>
[YOCTO# 13012]
[YOCTO# 12654]
(From OE-Core rev: 78a322d7be402a5b9b5abf26ad35670a8535408a)
Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
111 lines
5.1 KiB
Python
111 lines
5.1 KiB
Python
# test result tool - store test results
|
|
#
|
|
# Copyright (c) 2019, Intel Corporation.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify it
|
|
# under the terms and conditions of the GNU General Public License,
|
|
# version 2, as published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope it will be useful, but WITHOUT
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
# more details.
|
|
#
|
|
import datetime
|
|
import tempfile
|
|
import os
|
|
import subprocess
|
|
import scriptpath
|
|
scriptpath.add_bitbake_lib_path()
|
|
scriptpath.add_oe_lib_path()
|
|
from resulttool.resultsutils import checkout_git_dir
|
|
try:
|
|
import bb
|
|
except ImportError:
|
|
pass
|
|
|
|
class ResultsGitStore(object):
|
|
|
|
def _get_output_dir(self):
|
|
basepath = os.environ['BUILDDIR']
|
|
return basepath + '/testresults_%s/' % datetime.datetime.now().strftime("%Y%m%d%H%M%S")
|
|
|
|
def _create_temporary_workspace_dir(self):
|
|
return tempfile.mkdtemp(prefix='testresults.')
|
|
|
|
def _remove_temporary_workspace_dir(self, workspace_dir):
|
|
return subprocess.run(["rm", "-rf", workspace_dir])
|
|
|
|
def _oe_copy_files(self, source_dir, destination_dir):
|
|
from oe.path import copytree
|
|
copytree(source_dir, destination_dir)
|
|
|
|
def _copy_files(self, source_dir, destination_dir, copy_ignore=None):
|
|
from shutil import copytree
|
|
copytree(source_dir, destination_dir, ignore=copy_ignore)
|
|
|
|
def _store_files_to_git(self, logger, file_dir, git_dir, git_branch, commit_msg_subject, commit_msg_body):
|
|
logger.debug('Storing test result into git repository (%s) and branch (%s)'
|
|
% (git_dir, git_branch))
|
|
return subprocess.run(["oe-git-archive",
|
|
file_dir,
|
|
"-g", git_dir,
|
|
"-b", git_branch,
|
|
"--commit-msg-subject", commit_msg_subject,
|
|
"--commit-msg-body", commit_msg_body])
|
|
|
|
def store_to_existing(self, logger, source_dir, git_dir, git_branch):
|
|
logger.debug('Storing files to existing git repository and branch')
|
|
from shutil import ignore_patterns
|
|
dest_dir = self._create_temporary_workspace_dir()
|
|
dest_top_dir = os.path.join(dest_dir, 'top_dir')
|
|
self._copy_files(git_dir, dest_top_dir, copy_ignore=ignore_patterns('.git'))
|
|
self._oe_copy_files(source_dir, dest_top_dir)
|
|
self._store_files_to_git(logger, dest_top_dir, git_dir, git_branch,
|
|
'Store as existing git and branch', 'Store as existing git repository and branch')
|
|
self._remove_temporary_workspace_dir(dest_dir)
|
|
return git_dir
|
|
|
|
def store_to_existing_with_new_branch(self, logger, source_dir, git_dir, git_branch):
|
|
logger.debug('Storing files to existing git repository with new branch')
|
|
self._store_files_to_git(logger, source_dir, git_dir, git_branch,
|
|
'Store as existing git with new branch',
|
|
'Store as existing git repository with new branch')
|
|
return git_dir
|
|
|
|
def store_to_new(self, logger, source_dir, git_branch):
|
|
logger.debug('Storing files to new git repository')
|
|
output_dir = self._get_output_dir()
|
|
self._store_files_to_git(logger, source_dir, output_dir, git_branch,
|
|
'Store as new', 'Store as new git repository')
|
|
return output_dir
|
|
|
|
def store(self, logger, source_dir, git_dir, git_branch):
|
|
if git_dir:
|
|
if checkout_git_dir(git_dir, git_branch):
|
|
self.store_to_existing(logger, source_dir, git_dir, git_branch)
|
|
else:
|
|
self.store_to_existing_with_new_branch(logger, source_dir, git_dir, git_branch)
|
|
else:
|
|
self.store_to_new(logger, source_dir, git_branch)
|
|
|
|
def store(args, logger):
|
|
gitstore = ResultsGitStore()
|
|
gitstore.store(logger, args.source_dir, args.git_dir, args.git_branch)
|
|
return 0
|
|
|
|
def register_commands(subparsers):
|
|
"""Register subcommands from this plugin"""
|
|
parser_build = subparsers.add_parser('store', help='store test result files and directories into git repository',
|
|
description='store the testresults.json files and related directories '
|
|
'from the source directory into the destination git repository '
|
|
'with the given git branch',
|
|
group='setup')
|
|
parser_build.set_defaults(func=store)
|
|
parser_build.add_argument('source_dir',
|
|
help='source directory that contain the test result files and directories to be stored')
|
|
parser_build.add_argument('git_branch', help='git branch used for store')
|
|
parser_build.add_argument('-d', '--git-dir', default='',
|
|
help='(optional) default store to new <top_dir>/<build>/<testresults_datetime> '
|
|
'directory unless provided with existing git repository as destination')
|