alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-09 20:41:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

add component_ut in assign-test and target-test stage.

Browse Source 
Add one template_test python file to get test case
This commit is contained in:
Fu Hanxi
2020-08-25 16:42:36 +08:00
parent e35328afd9
commit edc7cc9c00
19 changed files with 638 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
tools/ci/python_packages/ttfw_idf/IDFAssignTest.py
View File

@@ -19,7 +19,7 @@ from tiny_test_fw.Utility import CIAssignTest
from idf_py_actions.constants import SUPPORTED_TARGETS
IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
IDF_PATH_FROM_ENV = os.getenv('IDF_PATH')
class IDFCaseGroup(CIAssignTest.Group):
@@ -30,9 +30,9 @@ class IDFCaseGroup(CIAssignTest.Group):
def get_artifact_index_file(cls):
assert cls.LOCAL_BUILD_DIR
if IDF_PATH_FROM_ENV:
artifact_index_file = os.path.join(IDF_PATH_FROM_ENV, cls.LOCAL_BUILD_DIR, "artifact_index.json")
artifact_index_file = os.path.join(IDF_PATH_FROM_ENV, cls.LOCAL_BUILD_DIR, 'artifact_index.json')
else:
artifact_index_file = "artifact_index.json"
artifact_index_file = 'artifact_index.json'
return artifact_index_file
@@ -41,25 +41,25 @@ class IDFAssignTest(CIAssignTest.AssignTest):
super(IDFAssignTest, self).__init__(test_case_path, ci_config_file, case_group)
def format_build_log_path(self, parallel_num):
return "{}/list_job_{}.json".format(self.case_group.LOCAL_BUILD_DIR, parallel_num)
return '{}/list_job_{}.json'.format(self.case_group.LOCAL_BUILD_DIR, parallel_num)
def create_artifact_index_file(self, project_id=None, pipeline_id=None):
if project_id is None:
project_id = os.getenv("CI_PROJECT_ID")
project_id = os.getenv('CI_PROJECT_ID')
if pipeline_id is None:
pipeline_id = os.getenv("CI_PIPELINE_ID")
pipeline_id = os.getenv('CI_PIPELINE_ID')
gitlab_inst = gitlab_api.Gitlab(project_id)
artifact_index_list = []
for build_job_name in self.case_group.BUILD_JOB_NAMES:
job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
for job_info in job_info_list:
parallel_num = job_info["parallel_num"] or 1 # Could be None if "parallel_num" not defined for the job
raw_data = gitlab_inst.download_artifact(job_info["id"],
parallel_num = job_info['parallel_num'] or 1 # Could be None if "parallel_num" not defined for the job
raw_data = gitlab_inst.download_artifact(job_info['id'],
[self.format_build_log_path(parallel_num)])[0]
build_info_list = [json.loads(line) for line in raw_data.decode().splitlines()]
for build_info in build_info_list:
build_info["ci_job_id"] = job_info["id"]
build_info['ci_job_id'] = job_info['id']
artifact_index_list.append(build_info)
artifact_index_file = self.case_group.get_artifact_index_file()
try:
@@ -68,42 +68,47 @@ class IDFAssignTest(CIAssignTest.AssignTest):
if e.errno != errno.EEXIST:
raise e
with open(artifact_index_file, "w") as f:
with open(artifact_index_file, 'w') as f:
json.dump(artifact_index_list, f)
class ExampleGroup(IDFCaseGroup):
SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "target"]
SORT_KEYS = CI_JOB_MATCH_KEYS = ['env_tag', 'target']
LOCAL_BUILD_DIR = "build_examples"
BUILD_JOB_NAMES = ["build_examples_cmake_{}".format(target) for target in SUPPORTED_TARGETS]
LOCAL_BUILD_DIR = 'build_examples'
BUILD_JOB_NAMES = ['build_examples_cmake_{}'.format(target) for target in SUPPORTED_TARGETS]
class TestAppsGroup(ExampleGroup):
LOCAL_BUILD_DIR = "build_test_apps"
BUILD_JOB_NAMES = ["build_test_apps_{}".format(target) for target in SUPPORTED_TARGETS]
LOCAL_BUILD_DIR = 'build_test_apps'
BUILD_JOB_NAMES = ['build_test_apps_{}'.format(target) for target in SUPPORTED_TARGETS]
class ComponentUTGroup(TestAppsGroup):
LOCAL_BUILD_DIR = 'build_component_ut'
BUILD_JOB_NAMES = ['build_component_ut_{}'.format(target) for target in SUPPORTED_TARGETS]
class UnitTestGroup(IDFCaseGroup):
SORT_KEYS = ["test environment", "tags", "chip_target"]
CI_JOB_MATCH_KEYS = ["test environment"]
SORT_KEYS = ['test environment', 'tags', 'chip_target']
CI_JOB_MATCH_KEYS = ['test environment']
LOCAL_BUILD_DIR = "tools/unit-test-app/builds"
BUILD_JOB_NAMES = ["build_esp_idf_tests_cmake_{}".format(target) for target in SUPPORTED_TARGETS]
LOCAL_BUILD_DIR = 'tools/unit-test-app/builds'
BUILD_JOB_NAMES = ['build_esp_idf_tests_cmake_{}'.format(target) for target in SUPPORTED_TARGETS]
MAX_CASE = 50
ATTR_CONVERT_TABLE = {
"execution_time": "execution time"
'execution_time': 'execution time'
}
DUT_CLS_NAME = {
"esp32": "ESP32DUT",
"esp32s2": "ESP32S2DUT",
"esp8266": "ESP8266DUT",
'esp32': 'ESP32DUT',
'esp32s2': 'ESP32S2DUT',
'esp8266': 'ESP8266DUT',
}
def __init__(self, case):
super(UnitTestGroup, self).__init__(case)
for tag in self._get_case_attr(case, "tags"):
for tag in self._get_case_attr(case, 'tags'):
self.ci_job_match_keys.add(tag)
@staticmethod
@@ -118,7 +123,7 @@ class UnitTestGroup(IDFCaseGroup):
if self.accept_new_case():
for key in self.filters:
if self._get_case_attr(case, key) != self.filters[key]:
if key == "tags":
if key == 'tags':
if set(self._get_case_attr(case, key)).issubset(set(self.filters[key])):
continue
break
@@ -135,18 +140,18 @@ class UnitTestGroup(IDFCaseGroup):
case_data = []
for case in test_cases:
one_case_data = {
"config": self._get_case_attr(case, "config"),
"name": self._get_case_attr(case, "summary"),
"reset": self._get_case_attr(case, "reset"),
"timeout": self._get_case_attr(case, "timeout"),
'config': self._get_case_attr(case, 'config'),
'name': self._get_case_attr(case, 'summary'),
'reset': self._get_case_attr(case, 'reset'),
'timeout': self._get_case_attr(case, 'timeout'),
}
if test_function in ["run_multiple_devices_cases", "run_multiple_stage_cases"]:
if test_function in ['run_multiple_devices_cases', 'run_multiple_stage_cases']:
try:
one_case_data["child case num"] = self._get_case_attr(case, "child case num")
one_case_data['child case num'] = self._get_case_attr(case, 'child case num')
except KeyError as e:
print("multiple devices/stages cases must contains at least two test functions")
print("case name: {}".format(one_case_data["name"]))
print('multiple devices/stages cases must contains at least two test functions')
print('case name: {}'.format(one_case_data['name']))
raise e
case_data.append(one_case_data)
@@ -159,18 +164,18 @@ class UnitTestGroup(IDFCaseGroup):
:return: dict of list of cases for each test functions
"""
case_by_test_function = {
"run_multiple_devices_cases": [],
"run_multiple_stage_cases": [],
"run_unit_test_cases": [],
'run_multiple_devices_cases': [],
'run_multiple_stage_cases': [],
'run_unit_test_cases': [],
}
for case in self.case_list:
if case["multi_device"] == "Yes":
case_by_test_function["run_multiple_devices_cases"].append(case)
elif case["multi_stage"] == "Yes":
case_by_test_function["run_multiple_stage_cases"].append(case)
if case['multi_device'] == 'Yes':
case_by_test_function['run_multiple_devices_cases'].append(case)
elif case['multi_stage'] == 'Yes':
case_by_test_function['run_multiple_stage_cases'].append(case)
else:
case_by_test_function["run_unit_test_cases"].append(case)
case_by_test_function['run_unit_test_cases'].append(case)
return case_by_test_function
def output(self):
@@ -180,12 +185,12 @@ class UnitTestGroup(IDFCaseGroup):
:return: {"Filter": case filter, "CaseConfig": list of case configs for cases in this group}
"""
target = self._get_case_attr(self.case_list[0], "chip_target")
target = self._get_case_attr(self.case_list[0], 'chip_target')
if target:
overwrite = {
"dut": {
"package": "ttfw_idf",
"class": self.DUT_CLS_NAME[target],
'dut': {
'package': 'ttfw_idf',
'class': self.DUT_CLS_NAME[target],
}
}
else:
@@ -195,11 +200,11 @@ class UnitTestGroup(IDFCaseGroup):
output_data = {
# we don't need filter for test function, as UT uses a few test functions for all cases
"CaseConfig": [
'CaseConfig': [
{
"name": test_function,
"extra_data": self._create_extra_data(test_cases, test_function),
"overwrite": overwrite,
'name': test_function,
'extra_data': self._create_extra_data(test_cases, test_function),
'overwrite': overwrite,
} for test_function, test_cases in case_by_test_function.items() if test_cases
],
}
@@ -258,13 +263,14 @@ class UnitTestAssignTest(IDFAssignTest):
return test_cases
test_cases = []
if os.path.isdir(self.test_case_path):
for yml_file in find_by_suffix('.yml', self.test_case_path):
test_cases.extend(get_test_cases_from_yml(yml_file))
elif os.path.isfile(self.test_case_path):
test_cases.extend(get_test_cases_from_yml(self.test_case_path))
else:
print("Test case path is invalid. Should only happen when use @bot to skip unit test.")
for path in self.test_case_paths:
if os.path.isdir(path):
for yml_file in find_by_suffix('.yml', path):
test_cases.extend(get_test_cases_from_yml(yml_file))
elif os.path.isfile(path) and path.endswith('.yml'):
test_cases.extend(get_test_cases_from_yml(path))
else:
print('Test case path is invalid. Should only happen when use @bot to skip unit test.')
# filter keys are lower case. Do map lower case keys with original keys.
try:
@@ -291,27 +297,30 @@ class UnitTestAssignTest(IDFAssignTest):
# sort cases with configs and test functions
# in later stage cases with similar attributes are more likely to be assigned to the same job
# it will reduce the count of flash DUT operations
test_cases.sort(key=lambda x: x["config"] + x["multi_stage"] + x["multi_device"])
test_cases.sort(key=lambda x: x['config'] + x['multi_stage'] + x['multi_device'])
return test_cases
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("case_group", choices=["example_test", "custom_test", "unit_test"])
parser.add_argument("test_case", help="test case folder or file")
parser.add_argument("ci_config_file", help="gitlab ci config file")
parser.add_argument("output_path", help="output path of config files")
parser.add_argument("--pipeline_id", "-p", type=int, default=None, help="pipeline_id")
parser.add_argument("--test-case-file-pattern", help="file name pattern used to find Python test case files")
parser.add_argument('case_group', choices=['example_test', 'custom_test', 'unit_test', 'component_ut'])
parser.add_argument('test_case_paths', nargs='+', help='test case folder or file')
parser.add_argument('-c', '--config', help='gitlab ci config file')
parser.add_argument('-o', '--output', help='output path of config files')
parser.add_argument('--pipeline_id', '-p', type=int, default=None, help='pipeline_id')
parser.add_argument('--test-case-file-pattern', help='file name pattern used to find Python test case files')
args = parser.parse_args()
args_list = [args.test_case, args.ci_config_file]
test_case_paths = [os.path.join(IDF_PATH_FROM_ENV, path) if not os.path.isabs(path) else path for path in args.test_case_paths]
args_list = [test_case_paths, args.config]
if args.case_group == 'example_test':
assigner = ExampleAssignTest(*args_list)
elif args.case_group == 'custom_test':
assigner = TestAppsAssignTest(*args_list)
elif args.case_group == 'unit_test':
assigner = UnitTestAssignTest(*args_list)
elif args.case_group == 'component_ut':
assigner = ComponentUTAssignTest(*args_list)
else:
raise SystemExit(1) # which is impossible
@@ -319,5 +328,5 @@ if __name__ == '__main__':
assigner.CI_TEST_JOB_PATTERN = re.compile(r'{}'.format(args.test_case_file_pattern))
assigner.assign_cases()
assigner.output_configs(args.output_path)
assigner.output_configs(args.output)
assigner.create_artifact_index_file()