alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2025-10-07 05:35:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: apply idf-ci pytest plugin

Browse Source 
Removed

- target markers. Now must use target as parametrization in esp-idf
- host test markers. Now will be automatically added with linux target and qemu marker
This commit is contained in:
Fu Hanxi
2025-06-13 14:30:23 +02:00
parent f33469dd63
commit a5257dcc39
39 changed files with 441 additions and 2415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

450
tools/ci/check_build_test_rules.py
View File

@@ -1,29 +1,23 @@
#!/usr/bin/env python
# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import difflib
import inspect
import os
import re
import sys
import typing as t
from collections import defaultdict
from pathlib import Path
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
import yaml
from idf_ci_utils import get_all_manifest_files
from idf_ci_utils import IDF_PATH
YES = u'\u2713'
NO = u'\u2717'
from idf_ci_utils import idf_relpath
# | Supported Target | ... |
# | ---------------- | --- |
SUPPORTED_TARGETS_TABLE_REGEX = re.compile(
r'^\|\s*Supported Targets.+$\n^\|(?:\s*|-).+$\n?', re.MULTILINE
)
SUPPORTED_TARGETS_TABLE_REGEX = re.compile(r'^\|\s*Supported Targets.+$\n^\|(?:\s*|-).+$\n?', re.MULTILINE)
USUAL_TO_FORMAL = {
'esp32': 'ESP32',
@@ -41,54 +35,51 @@ USUAL_TO_FORMAL = {
'linux': 'Linux',
}
FORMAL_TO_USUAL = {
'ESP32': 'esp32',
'ESP32-S2': 'esp32s2',
'ESP32-S3': 'esp32s3',
'ESP32-C3': 'esp32c3',
'ESP32-C2': 'esp32c2',
'ESP32-C6': 'esp32c6',
'ESP32-C5': 'esp32c5',
'ESP32-H2': 'esp32h2',
'ESP32-P4': 'esp32p4',
'ESP32-C61': 'esp32c61',
'ESP32-H21': 'esp32h21',
'ESP32-H4': 'esp32h4',
'Linux': 'linux',
}
FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()}
def doublequote(s: str) -> str:
if s.startswith('"') and s.endswith('"'):
return s
def diff_lists(
list1: t.List[str], list2: t.List[str], title1: str, title2: str, excluded: t.Optional[t.List[str]] = None
) -> None:
"""
Compare two lists and print the differences.
"""
diff = difflib.ndiff(list1, list2)
if not diff:
return
return f'"{s}"'
print(f'Difference between {title1} and {title2}:')
for line in diff:
last_part = line.split(' ', 1)[-1]
if excluded and last_part in excluded:
print(line + ' ' + '(excluded)')
else:
print(line)
def check_readme(
paths: List[str],
exclude_dirs: Optional[List[str]] = None,
extra_default_build_targets: Optional[List[str]] = None,
paths: t.List[str],
exclude_dirs: t.Optional[t.List[str]] = None,
extra_default_build_targets: t.Optional[t.List[str]] = None,
) -> None:
from idf_build_apps import App, find_apps
from idf_build_apps import App
from idf_build_apps import find_apps
from idf_build_apps.constants import SUPPORTED_TARGETS
def get_readme_path(_app: App) -> Optional[str]:
_readme_path = os.path.join(_app.app_dir, 'README.md')
def get_readme_path(_app_dir: str) -> t.Optional[str]:
_readme_path = os.path.join(_app_dir, 'README.md')
if not os.path.isfile(_readme_path):
_readme_path = os.path.join(_app.app_dir, '..', 'README.md')
_readme_path = os.path.join(_app_dir, '..', 'README.md')
if not os.path.isfile(_readme_path):
_readme_path = None # type: ignore
return _readme_path
def _generate_new_support_table_str(_app: App) -> str:
def _generate_new_support_table_str(_app_dir: str, _manifest_supported_targets: t.List[str]) -> str:
# extra space here
table_headers = [
f'{USUAL_TO_FORMAL[target]}' for target in _app.supported_targets
]
table_headers = [f'{USUAL_TO_FORMAL[target]}' for target in _manifest_supported_targets]
table_headers = ['Supported Targets'] + table_headers
res = '| ' + ' | '.join(table_headers) + ' |\n'
@@ -96,8 +87,8 @@ def check_readme(
return res
def _parse_existing_support_table_str(_app: App) -> Tuple[Optional[str], List[str]]:
_readme_path = get_readme_path(_app)
def _parse_existing_support_table_str(_app_dir: str) -> t.Tuple[t.Optional[str], t.List[str]]:
_readme_path = get_readme_path(_app_dir)
if not _readme_path:
return None, SUPPORTED_TARGETS
@@ -109,41 +100,31 @@ def check_readme(
return None, SUPPORTED_TARGETS
# old style
parts = [
part.strip()
for part in support_string[0].split('\n', 1)[0].split('|')
if part.strip()
]
parts = [part.strip() for part in support_string[0].split('\n', 1)[0].split('|') if part.strip()]
return support_string[0].strip(), [FORMAL_TO_USUAL[part] for part in parts[1:] if part in FORMAL_TO_USUAL]
def check_enable_build(_app: App, _old_supported_targets: List[str]) -> bool:
if _app.supported_targets == sorted(_old_supported_targets):
def check_enable_build(
_app_dir: str, _manifest_supported_targets: t.List[str], _old_supported_targets: t.List[str]
) -> bool:
if _manifest_supported_targets == sorted(_old_supported_targets):
return True
_readme_path = get_readme_path(_app)
if_clause = f'IDF_TARGET in [{", ".join([doublequote(target) for target in sorted(_old_supported_targets)])}]'
_readme_path = get_readme_path(_app_dir)
diff_lists(
sorted(_manifest_supported_targets),
sorted(_old_supported_targets),
'manifest-enabled targets',
f'supported targets table in {_readme_path}',
)
print(
inspect.cleandoc(
f'''
{_app.app_dir}:
- enable build targets according to the manifest file: {_app.supported_targets}
- enable build targets according to the old Supported Targets table under readme "{_readme_path}": {_old_supported_targets}
f"""
To enable/disable build targets, please modify your manifest file:
{App.MANIFEST.most_suitable_rule(app_dir).by_manifest_file}
If you want to disable some targets, please use the following snippet:
# Please combine this with the original one
#
# Notes:
# - please keep in mind to avoid duplicated folders as yaml keys
# - please use parentheses to group conditions, the "or" and "and" operators could only accept two operands
{_app.app_dir}:
enable:
- if: {if_clause}
temporary: true
reason: <why only enable build jobs for these targets>
'''
Please refer to https://docs.espressif.com/projects/idf-build-apps/en/latest/references/manifest.html#enable-disable-rules
for more details.
"""
)
)
@@ -153,47 +134,52 @@ def check_readme(
find_apps(
paths,
'all',
recursive=True,
exclude_list=exclude_dirs or [],
manifest_files=get_all_manifest_files(),
default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets,
)
)
exit_code = 0
checked_app_dirs = set()
apps_grouped: t.Dict[str, t.List[App]] = defaultdict(list)
for app in apps:
if app.app_dir not in checked_app_dirs:
checked_app_dirs.add(app.app_dir)
else:
continue
apps_grouped[app.app_dir].append(app)
replace_str, old_supported_targets = _parse_existing_support_table_str(app)
success = check_enable_build(app, old_supported_targets)
for app_dir in apps_grouped:
replace_str, old_supported_targets = _parse_existing_support_table_str(app_dir)
# manifest defined ones
manifest_defined_targets = sorted(
{
target
for app in apps_grouped[app_dir]
for target in (
App.MANIFEST.enable_build_targets(app_dir)
+ App.MANIFEST.enable_build_targets(app_dir, config_name=app.config_name)
)
}
)
success = check_enable_build(app_dir, manifest_defined_targets, old_supported_targets)
if not success:
print(f'check_enable_build failed for app: {app}')
print(f'check_enable_build failed for app: {app_dir}')
print('-' * 80)
exit_code = 1
readme_path = get_readme_path(app)
readme_path = get_readme_path(app_dir)
new_readme_str = _generate_new_support_table_str(app_dir, manifest_defined_targets)
# no readme, create a new file
if not readme_path:
with open(os.path.join(app.app_dir, 'README.md'), 'w') as fw:
fw.write(_generate_new_support_table_str(app) + '\n')
print(f'Added new README file: {os.path.join(app.app_dir, "README.md")}')
with open(os.path.join(app_dir, 'README.md'), 'w') as fw:
fw.write(new_readme_str + '\n')
print(f'Added new README file: {os.path.join(app_dir, "README.md")}')
print('-' * 80)
exit_code = 1
# has old table, but different string
elif replace_str and replace_str != _generate_new_support_table_str(app):
elif replace_str and replace_str != new_readme_str:
with open(readme_path) as fr:
readme_str = fr.read()
with open(readme_path, 'w') as fw:
fw.write(
readme_str.replace(
replace_str, _generate_new_support_table_str(app)
)
)
fw.write(readme_str.replace(replace_str, new_readme_str))
print(f'Modified README file: {readme_path}')
print('-' * 80)
exit_code = 1
@@ -203,9 +189,7 @@ def check_readme(
readme_str = fr.read()
with open(readme_path, 'w') as fw:
fw.write(
_generate_new_support_table_str(app) + '\n\n' + readme_str
) # extra new line
fw.write(new_readme_str + '\n\n' + readme_str) # extra new line
print(f'Modified README file: {readme_path}')
print('-' * 80)
@@ -215,90 +199,61 @@ def check_readme(
def check_test_scripts(
paths: List[str],
exclude_dirs: Optional[List[str]] = None,
bypass_check_test_targets: Optional[List[str]] = None,
extra_default_build_targets: Optional[List[str]] = None,
paths: t.List[str],
exclude_dirs: t.Optional[t.List[str]] = None,
bypass_check_test_targets: t.Optional[t.List[str]] = None,
extra_default_build_targets: t.Optional[t.List[str]] = None,
) -> None:
from idf_build_apps import App, find_apps
from idf_build_apps import App
from idf_build_apps import find_apps
from idf_build_apps.constants import SUPPORTED_TARGETS
from idf_pytest.script import get_pytest_cases
from idf_ci import get_pytest_cases
# takes long time, run only in CI
# dict:
# {
# app_dir: {
# 'script_path': 'path/to/script',
# 'targets': ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c2', 'linux'],
# 'script_paths': {'path/to/script1', 'path/to/script2'},
# 'targets': {'esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c2', 'linux'},
# }
# }
def check_enable_test(
_app: App,
_pytest_app_dir_targets_dict: Dict[str, Dict[str, str]],
_app_dir: str,
_manifest_verified_targets: t.List[str],
_pytest_app_dir_targets_dict: t.Dict[str, t.Dict[str, t.Set[str]]],
) -> bool:
if _app.app_dir in _pytest_app_dir_targets_dict:
test_script_path = _pytest_app_dir_targets_dict[_app.app_dir]['script_path']
actual_verified_targets = sorted(
set(_pytest_app_dir_targets_dict[_app.app_dir]['targets'])
)
if _app_dir in _pytest_app_dir_targets_dict:
test_script_paths = _pytest_app_dir_targets_dict[_app_dir]['script_paths']
actual_verified_targets = sorted(set(_pytest_app_dir_targets_dict[_app_dir]['targets']))
else:
return True # no test case
actual_extra_tested_targets = set(actual_verified_targets) - set(
_app.verified_targets
if _manifest_verified_targets == actual_verified_targets:
return True
elif not (set(_manifest_verified_targets) - set(actual_verified_targets + (bypass_check_test_targets or []))):
return True
_title2 = 'pytest enabled targets in test scripts: \n'
for script_path in test_script_paths:
_title2 += f' - {script_path}\n'
diff_lists(
_manifest_verified_targets,
actual_verified_targets,
'manifest-enabled targets',
_title2.rstrip(),
excluded=bypass_check_test_targets or [],
)
if actual_extra_tested_targets - set(bypass_check_test_targets or []):
print(
inspect.cleandoc(
f'''
{_app.app_dir}:
- enable test targets according to the manifest file: {_app.verified_targets}
- enable test targets according to the test scripts: {actual_verified_targets}
test scripts enabled targets should be a subset of the manifest file declared ones.
Please check the test script: {test_script_path}.
'''
)
)
return False
if _app.verified_targets == actual_verified_targets:
return True
elif not (set(_app.verified_targets) - set(actual_verified_targets + (bypass_check_test_targets or []))):
print(f'WARNING: bypass test script check on {_app.app_dir} for targets {bypass_check_test_targets} ')
return True
if_clause = f'IDF_TARGET in [{", ".join([doublequote(target) for target in sorted(set(_app.verified_targets) - set(actual_verified_targets))])}]'
print(
inspect.cleandoc(
f'''
{_app.app_dir}:
- enable test targets according to the manifest file: {_app.verified_targets}
- enable test targets according to the test scripts: {actual_verified_targets}
f"""
To enable/disable test targets, please modify your manifest file:
{App.MANIFEST.most_suitable_rule(app_dir).by_manifest_file}
the test scripts enabled test targets should be the same with the manifest file enabled ones. Please check
the test script manually: {test_script_path}.
To understand how to enable/disable test targets, please refer to:
https://docs.espressif.com/projects/pytest-embedded/en/latest/usages/markers.html#idf-parametrize
If you want to enable test targets in the pytest test scripts, please add `@pytest.mark.MISSING_TARGET`
marker above the test case function.
If you want to disable the test targets in the manifest file, please modify your manifest file with
the following code snippet:
# Please combine this with the original one
#
# Notes:
# - please keep in mind to avoid duplicated folders as yaml keys
# - please use parentheses to group conditions, the "or" and "and" operators could only accept two operands
{_app.app_dir}:
disable_test:
- if: {if_clause}
temporary: true
reason: <why you disable this test>
'''
"""
)
)
return False
@@ -307,42 +262,50 @@ def check_test_scripts(
find_apps(
paths,
'all',
recursive=True,
exclude_list=exclude_dirs or [],
manifest_files=get_all_manifest_files(),
default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets,
)
)
apps_grouped: t.Dict[str, t.List[App]] = defaultdict(list)
for app in apps:
apps_grouped[app.app_dir].append(app)
exit_code = 0
pytest_cases = get_pytest_cases(paths)
pytest_cases = get_pytest_cases(
paths=paths,
marker_expr=None, # don't filter host_test
)
pytest_app_dir_targets_dict = {}
for case in pytest_cases:
for pytest_app in case.apps:
app_dir = os.path.relpath(pytest_app.path, IDF_PATH)
app_dir = idf_relpath(pytest_app.path)
if app_dir not in pytest_app_dir_targets_dict:
pytest_app_dir_targets_dict[app_dir] = {
'script_path': case.path,
'targets': [pytest_app.target],
'script_paths': {case.path},
'targets': {pytest_app.target},
}
else:
pytest_app_dir_targets_dict[app_dir]['targets'].append(
pytest_app.target
pytest_app_dir_targets_dict[app_dir]['script_paths'].add(case.path)
pytest_app_dir_targets_dict[app_dir]['targets'].add(pytest_app.target)
for app_dir in apps_grouped:
# manifest defined ones
manifest_defined_targets = sorted(
{
target
for app in apps_grouped[app_dir]
for target in (
App.MANIFEST.enable_test_targets(app_dir)
+ App.MANIFEST.enable_test_targets(app_dir, config_name=app.config_name)
)
checked_app_dirs = set()
for app in apps:
if app.app_dir not in checked_app_dirs:
checked_app_dirs.add(app.app_dir)
else:
continue
success = check_enable_test(
app, pytest_app_dir_targets_dict
}
)
success = check_enable_test(app_dir, manifest_defined_targets, pytest_app_dir_targets_dict)
if not success:
print(f'check_enable_test failed for app: {app}')
print(f'check_enable_test failed for app: {app_dir}')
print('-' * 80)
exit_code = 1
continue
@@ -350,26 +313,6 @@ def check_test_scripts(
sys.exit(exit_code)
def check_exist() -> None:
exit_code = 0
config_files = get_all_manifest_files()
for file in config_files:
if 'managed_components' in Path(file).parts:
continue
with open(file) as fr:
configs = yaml.safe_load(fr)
for path in configs.keys():
if path.startswith('.'):
continue
if not os.path.isdir(path):
print(f'Path \'{path}\' referred in \'{file}\' does not exist!')
exit_code = 1
sys.exit(exit_code)
if __name__ == '__main__':
if 'CI_JOB_ID' not in os.environ:
os.environ['CI_JOB_ID'] = 'fake' # this is a CI script
@@ -400,69 +343,60 @@ if __name__ == '__main__':
arg = parser.parse_args()
# Since this script is executed from the pre-commit hook environment, make sure IDF_PATH is set
os.environ['IDF_PATH'] = os.path.realpath(
os.path.join(os.path.dirname(__file__), '..', '..')
)
os.environ['IDF_PATH'] = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))
if arg.action == 'check-exist':
check_exist()
else:
check_dirs = set()
check_dirs = set()
# check if *_caps.h files changed
check_all = False
soc_caps_header_files = list(
(Path(IDF_PATH) / 'components' / 'soc').glob('**/*_caps.h')
)
for p in arg.paths:
if Path(p).resolve() in soc_caps_header_files:
check_all = True
break
if os.path.isfile(p):
check_dirs.add(os.path.dirname(p))
else:
check_dirs.add(p)
if 'tools/idf_py_actions/constants.py' in arg.paths or 'tools/ci/check_build_test_rules.py' in arg.paths:
# check if *_caps.h files changed
check_all = False
soc_caps_header_files = list((Path(IDF_PATH) / 'components' / 'soc').glob('**/*_caps.h'))
for p in arg.paths:
if Path(p).resolve() in soc_caps_header_files:
check_all = True
break
if check_all:
check_dirs = {IDF_PATH}
_exclude_dirs = [os.path.join(IDF_PATH, 'tools', 'unit-test-app'),
os.path.join(IDF_PATH, 'tools', 'test_build_system', 'build_test_app'),
os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project')]
if os.path.isfile(p):
check_dirs.add(os.path.dirname(p))
else:
_exclude_dirs = [os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project')]
check_dirs.add(p)
extra_default_build_targets_list: List[str] = []
bypass_check_test_targets_list: List[str] = []
if arg.config:
with open(arg.config) as fr:
configs = yaml.safe_load(fr)
if 'tools/idf_py_actions/constants.py' in arg.paths or 'tools/ci/check_build_test_rules.py' in arg.paths:
check_all = True
if configs:
extra_default_build_targets_list = (
configs.get('extra_default_build_targets') or []
)
bypass_check_test_targets_list = (
configs.get('bypass_check_test_targets') or []
)
if check_all:
check_dirs = {IDF_PATH}
_exclude_dirs = [
os.path.join(IDF_PATH, 'tools', 'unit-test-app'),
os.path.join(IDF_PATH, 'tools', 'test_build_system', 'build_test_app'),
os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project'),
]
else:
_exclude_dirs = [os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project')]
if arg.action == 'check-readmes':
os.environ['INCLUDE_NIGHTLY_RUN'] = '1'
os.environ['NIGHTLY_RUN'] = '1'
check_readme(
list(check_dirs),
exclude_dirs=_exclude_dirs,
extra_default_build_targets=extra_default_build_targets_list,
)
elif arg.action == 'check-test-scripts':
os.environ['INCLUDE_NIGHTLY_RUN'] = '1'
os.environ['NIGHTLY_RUN'] = '1'
check_test_scripts(
list(check_dirs),
exclude_dirs=_exclude_dirs,
bypass_check_test_targets=bypass_check_test_targets_list,
extra_default_build_targets=extra_default_build_targets_list,
)
extra_default_build_targets_list: t.List[str] = []
bypass_check_test_targets_list: t.List[str] = []
if arg.config:
with open(arg.config) as fr:
configs = yaml.safe_load(fr)
if configs:
extra_default_build_targets_list = configs.get('extra_default_build_targets') or []
bypass_check_test_targets_list = configs.get('bypass_check_test_targets') or []
if arg.action == 'check-readmes':
os.environ['INCLUDE_NIGHTLY_RUN'] = '1'
os.environ['NIGHTLY_RUN'] = '1'
check_readme(
list(check_dirs),
exclude_dirs=_exclude_dirs,
extra_default_build_targets=extra_default_build_targets_list,
)
elif arg.action == 'check-test-scripts':
os.environ['INCLUDE_NIGHTLY_RUN'] = '1'
os.environ['NIGHTLY_RUN'] = '1'
check_test_scripts(
list(check_dirs),
exclude_dirs=_exclude_dirs,
bypass_check_test_targets=bypass_check_test_targets_list,
extra_default_build_targets=extra_default_build_targets_list,
)