mirror of
https://github.com/espressif/esp-idf.git
synced 2025-08-09 12:35:28 +00:00
ci: dynamic pipeline
build: - upgrade idf-build-apps to 2.x - unify get_pytest_apps and get_cmake_apps to get_all_apps - returns (test_apps, non_test_apps) tuple - add tests for the new get_all_apps assign: - generate build report - generate target test pipeline based on the build report target test: - download artifacts from minio server - users can use `pytest --pipeline-id xxxxx` to download and flash the binaries from the artifacts .post: - generate target test reports
This commit is contained in:
Fu Hanxi
@@ -0,0 +1,193 @@
|
||||
# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""This file is used for generating the child pipeline for build jobs."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import typing as t
|
||||
|
||||
import __init__ # noqa: F401 # inject the system path
|
||||
import yaml
|
||||
from dynamic_pipelines.constants import (DEFAULT_APPS_BUILD_PER_JOB, DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH,
|
||||
DEFAULT_TEST_PATHS, NON_TEST_RELATED_APPS_FILENAME,
|
||||
NON_TEST_RELATED_BUILD_JOB_NAME, TEST_RELATED_APPS_FILENAME,
|
||||
TEST_RELATED_BUILD_JOB_NAME)
|
||||
from dynamic_pipelines.models import BuildJob, EmptyJob
|
||||
from dynamic_pipelines.utils import dump_jobs_to_yaml
|
||||
from idf_build_apps.utils import semicolon_separated_str_to_list
|
||||
from idf_ci.app import dump_apps_to_txt
|
||||
from idf_ci_utils import IDF_PATH
|
||||
from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR, DEFAULT_FULL_BUILD_TEST_FILEPATTERNS, CollectMode
|
||||
from idf_pytest.script import get_all_apps
|
||||
|
||||
|
||||
def main(arguments: argparse.Namespace) -> None:
|
||||
# load from default build test rules config file
|
||||
extra_default_build_targets: t.List[str] = []
|
||||
if arguments.default_build_test_rules:
|
||||
with open(arguments.default_build_test_rules) as fr:
|
||||
configs = yaml.safe_load(fr)
|
||||
|
||||
if configs:
|
||||
extra_default_build_targets = configs.get('extra_default_build_targets') or []
|
||||
|
||||
build_jobs = []
|
||||
###########################################
|
||||
# special case with -k, ignore other args #
|
||||
###########################################
|
||||
if arguments.filter_expr:
|
||||
# build only test related apps
|
||||
test_related_apps, _ = get_all_apps(
|
||||
arguments.paths,
|
||||
target=CollectMode.ALL,
|
||||
config_rules_str=DEFAULT_CONFIG_RULES_STR,
|
||||
filter_expr=arguments.filter_expr,
|
||||
marker_expr='not host_test',
|
||||
extra_default_build_targets=extra_default_build_targets,
|
||||
)
|
||||
dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME)
|
||||
print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps')
|
||||
|
||||
test_apps_build_job = BuildJob(
|
||||
name=TEST_RELATED_BUILD_JOB_NAME,
|
||||
parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1,
|
||||
variables={
|
||||
'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME,
|
||||
},
|
||||
)
|
||||
|
||||
build_jobs.append(test_apps_build_job)
|
||||
else:
|
||||
#############
|
||||
# all cases #
|
||||
#############
|
||||
test_related_apps, non_test_related_apps = get_all_apps(
|
||||
arguments.paths,
|
||||
CollectMode.ALL,
|
||||
marker_expr='not host_test',
|
||||
config_rules_str=DEFAULT_CONFIG_RULES_STR,
|
||||
extra_default_build_targets=extra_default_build_targets,
|
||||
modified_components=arguments.modified_components,
|
||||
modified_files=arguments.modified_files,
|
||||
ignore_app_dependencies_filepatterns=arguments.ignore_app_dependencies_filepatterns,
|
||||
)
|
||||
|
||||
dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME)
|
||||
print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps')
|
||||
dump_apps_to_txt(sorted(non_test_related_apps), NON_TEST_RELATED_APPS_FILENAME)
|
||||
print(
|
||||
f'Generate non-test related apps file {NON_TEST_RELATED_APPS_FILENAME} with {len(non_test_related_apps)} apps'
|
||||
)
|
||||
|
||||
if test_related_apps:
|
||||
test_apps_build_job = BuildJob(
|
||||
name=TEST_RELATED_BUILD_JOB_NAME,
|
||||
parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1,
|
||||
variables={
|
||||
'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME,
|
||||
},
|
||||
)
|
||||
build_jobs.append(test_apps_build_job)
|
||||
|
||||
if non_test_related_apps:
|
||||
non_test_apps_build_job = BuildJob(
|
||||
name=NON_TEST_RELATED_BUILD_JOB_NAME,
|
||||
parallel=len(non_test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1,
|
||||
variables={
|
||||
'APP_LIST_FILE': NON_TEST_RELATED_APPS_FILENAME,
|
||||
},
|
||||
)
|
||||
build_jobs.append(non_test_apps_build_job)
|
||||
|
||||
# check if there's no jobs
|
||||
if not build_jobs:
|
||||
print('No apps need to be built. Create one empty job instead')
|
||||
build_jobs.append(EmptyJob())
|
||||
extra_include_yml = []
|
||||
else:
|
||||
extra_include_yml = ['tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml']
|
||||
|
||||
dump_jobs_to_yaml(build_jobs, arguments.yaml_output, extra_include_yml)
|
||||
print(f'Generate child pipeline yaml file {arguments.yaml_output} with {sum(j.parallel for j in build_jobs)} jobs')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Generate build child pipeline',
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
)
|
||||
parser.add_argument(
|
||||
'-o',
|
||||
'--yaml-output',
|
||||
default=DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH,
|
||||
help='Output YAML path',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-p',
|
||||
'--paths',
|
||||
nargs='+',
|
||||
default=DEFAULT_TEST_PATHS,
|
||||
help='Paths to the apps to build.',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-k',
|
||||
'--filter-expr',
|
||||
help='only build tests matching given filter expression. For example: -k "test_hello_world". Works only'
|
||||
'for pytest',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--default-build-test-rules',
|
||||
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
|
||||
help='default build test rules config file',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--modified-components',
|
||||
type=semicolon_separated_str_to_list,
|
||||
help='semicolon-separated string which specifies the modified components. '
|
||||
'app with `depends_components` set in the corresponding manifest files would only be built '
|
||||
'if depends on any of the specified components. '
|
||||
'If set to "", the value would be considered as None. '
|
||||
'If set to ";", the value would be considered as an empty list',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--modified-files',
|
||||
type=semicolon_separated_str_to_list,
|
||||
help='semicolon-separated string which specifies the modified files. '
|
||||
'app with `depends_filepatterns` set in the corresponding manifest files would only be built '
|
||||
'if any of the specified file pattern matches any of the specified modified files. '
|
||||
'If set to "", the value would be considered as None. '
|
||||
'If set to ";", the value would be considered as an empty list',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-if',
|
||||
'--ignore-app-dependencies-filepatterns',
|
||||
type=semicolon_separated_str_to_list,
|
||||
help='semicolon-separated string which specifies the file patterns used for '
|
||||
'ignoring checking the app dependencies. '
|
||||
'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored '
|
||||
'when any of the specified file patterns matches any of the modified files. '
|
||||
'Must be used together with --modified-files. '
|
||||
'If set to "", the value would be considered as None. '
|
||||
'If set to ";", the value would be considered as an empty list',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1':
|
||||
print('Build and run all test cases, and compile all cmake apps')
|
||||
args.modified_components = None
|
||||
args.modified_files = None
|
||||
args.ignore_app_dependencies_filepatterns = None
|
||||
elif args.filter_expr is not None:
|
||||
print('Build and run only test cases matching "%s"' % args.filter_expr)
|
||||
args.modified_components = None
|
||||
args.modified_files = None
|
||||
args.ignore_app_dependencies_filepatterns = None
|
||||
else:
|
||||
print('Build and run only test cases matching the modified components and files')
|
||||
if args.modified_files and not args.ignore_app_dependencies_filepatterns:
|
||||
# setting default values
|
||||
args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS
|
||||
|
||||
main(args)
|
||||
Reference in New Issue
Block a user