# SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # pylint: disable=W0621 # redefined-outer-name # This file is a pytest root configuration file and provide the following functionalities: # 1. Defines a few fixtures that could be used under the whole project. # 2. Defines a few hook functions. # # IDF is using [pytest](https://github.com/pytest-dev/pytest) and # [pytest-embedded plugin](https://github.com/espressif/pytest-embedded) as its example test framework. # # This is an experimental feature, and if you found any bug or have any question, please report to # https://github.com/espressif/pytest-embedded/issues import glob import json import logging import os import re import sys import typing as t from copy import deepcopy from datetime import datetime import pytest from _pytest.config import Config from _pytest.fixtures import FixtureRequest from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture from pytest_embedded_idf.dut import IdfDut from pytest_embedded_idf.unity_tester import CaseTester try: from idf_ci_utils import IDF_PATH from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, IdfPytestEmbedded from idf_pytest.utils import format_case_id except ImportError: sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci')) from idf_ci_utils import IDF_PATH from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, IdfPytestEmbedded from idf_pytest.utils import format_case_id try: import common_test_methods # noqa: F401 except ImportError: sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages')) import common_test_methods # noqa: F401 ############ # Fixtures # ############ @pytest.fixture(scope='session') def idf_path() -> str: return os.path.dirname(__file__) @pytest.fixture(scope='session', autouse=True) def session_tempdir() -> str: _tmpdir = os.path.join( os.path.dirname(__file__), 'pytest_embedded_log', datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), ) os.makedirs(_tmpdir, exist_ok=True) return _tmpdir @pytest.fixture def case_tester(unity_tester: CaseTester) -> CaseTester: return unity_tester @pytest.fixture @multi_dut_argument def config(request: FixtureRequest) -> str: return getattr(request, 'param', None) or DEFAULT_SDKCONFIG # type: ignore @pytest.fixture def test_func_name(request: FixtureRequest) -> str: return request.node.function.__name__ # type: ignore @pytest.fixture def test_case_name(request: FixtureRequest, target: str, config: str) -> str: is_qemu = request._pyfuncitem.get_closest_marker('qemu') is not None if hasattr(request._pyfuncitem, 'callspec'): params = deepcopy(request._pyfuncitem.callspec.params) # type: ignore else: params = {} filtered_params = {} for k, v in params.items(): if k not in request.session._fixturemanager._arg2fixturedefs: # type: ignore filtered_params[k] = v # not fixture ones return format_case_id(target, config, request.node.originalname, is_qemu=is_qemu, params=filtered_params) # type: ignore @pytest.fixture @multi_dut_fixture def build_dir(app_path: str, target: t.Optional[str], config: t.Optional[str]) -> str: """ Check local build dir with the following priority: 1. build__ 2. build_ 3. build_ 4. build Returns: valid build directory """ check_dirs = [] if target is not None and config is not None: check_dirs.append(f'build_{target}_{config}') if target is not None: check_dirs.append(f'build_{target}') if config is not None: check_dirs.append(f'build_{config}') check_dirs.append('build') for check_dir in check_dirs: binary_path = os.path.join(app_path, check_dir) if os.path.isdir(binary_path): logging.info(f'found valid binary path: {binary_path}') return check_dir logging.warning('checking binary path: %s... missing... try another place', binary_path) raise ValueError( f'no build dir valid. Please build the binary via "idf.py -B {check_dirs[0]} build" and run pytest again' ) @pytest.fixture(autouse=True) @multi_dut_fixture def junit_properties(test_case_name: str, record_xml_attribute: t.Callable[[str, object], None]) -> None: """ This fixture is autoused and will modify the junit report test case name to .. """ record_xml_attribute('name', test_case_name) @pytest.fixture(autouse=True) def set_test_case_name(request: FixtureRequest, test_case_name: str) -> None: request.node.funcargs['test_case_name'] = test_case_name ###################### # Log Util Functions # ###################### @pytest.fixture def log_performance(record_property: t.Callable[[str, object], None]) -> t.Callable[[str, str], None]: """ log performance item with pre-defined format to the console and record it under the ``properties`` tag in the junit report if available. """ def real_func(item: str, value: str) -> None: """ :param item: performance item name :param value: performance value """ logging.info('[Performance][%s]: %s', item, value) record_property(item, value) return real_func @pytest.fixture def check_performance(idf_path: str) -> t.Callable[[str, float, str], None]: """ check if the given performance item meets the passing standard or not """ def real_func(item: str, value: float, target: str) -> None: """ :param item: performance item name :param value: performance item value :param target: target chip :raise: AssertionError: if check fails """ def _find_perf_item(operator: str, path: str) -> float: with open(path) as f: data = f.read() match = re.search(fr'#define\s+IDF_PERFORMANCE_{operator}_{item.upper()}\s+([\d.]+)', data) return float(match.group(1)) # type: ignore def _check_perf(operator: str, standard_value: float) -> None: if operator == 'MAX': ret = value <= standard_value else: ret = value >= standard_value if not ret: raise AssertionError( f"[Performance] {item} value is {value}, doesn't meet pass standard {standard_value}" ) path_prefix = os.path.join(idf_path, 'components', 'idf_test', 'include') performance_files = ( os.path.join(path_prefix, target, 'idf_performance_target.h'), os.path.join(path_prefix, 'idf_performance.h'), ) found_item = False for op in ['MIN', 'MAX']: for performance_file in performance_files: try: standard = _find_perf_item(op, performance_file) except (OSError, AttributeError): # performance file doesn't exist or match is not found in it continue _check_perf(op, standard) found_item = True break if not found_item: raise AssertionError(f'Failed to get performance standard for {item}') return real_func @pytest.fixture def log_minimum_free_heap_size(dut: IdfDut, config: str) -> t.Callable[..., None]: def real_func() -> None: res = dut.expect(r'Minimum free heap size: (\d+) bytes') logging.info( '\n------ heap size info ------\n' '[app_name] {}\n' '[config_name] {}\n' '[target] {}\n' '[minimum_free_heap_size] {} Bytes\n' '------ heap size end ------'.format( os.path.basename(dut.app.app_path), config, dut.target, res.group(1).decode('utf8'), ) ) return real_func @pytest.fixture def dev_password(request: FixtureRequest) -> str: return request.config.getoption('dev_passwd') or '' @pytest.fixture def dev_user(request: FixtureRequest) -> str: return request.config.getoption('dev_user') or '' ################## # Hook functions # ################## def pytest_addoption(parser: pytest.Parser) -> None: idf_group = parser.getgroup('idf') idf_group.addoption( '--sdkconfig', help='sdkconfig postfix, like sdkconfig.ci.. (Default: None, which would build all found apps)', ) idf_group.addoption( '--dev-user', help='user name associated with some specific device/service used during the test execution', ) idf_group.addoption( '--dev-passwd', help='password associated with some specific device/service used during the test execution', ) idf_group.addoption( '--app-info-basedir', default=IDF_PATH, help='app info base directory. specify this value when you\'re building under a ' 'different IDF_PATH. (Default: $IDF_PATH)', ) idf_group.addoption( '--app-info-filepattern', help='glob pattern to specify the files that include built app info generated by ' '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' 'paths not exist in local file system if not listed recorded in the app info.', ) def pytest_configure(config: Config) -> None: # cli option "--target" target = [_t.strip().lower() for _t in (config.getoption('target', '') or '').split(',') if _t.strip()] # add markers based on idf_pytest/constants.py for name, description in { **TARGET_MARKERS, **ENV_MARKERS, **SPECIAL_MARKERS, }.items(): config.addinivalue_line('markers', f'{name}: {description}') help_commands = ['--help', '--fixtures', '--markers', '--version'] for cmd in help_commands: if cmd in config.invocation_params.args: target = ['unneeded'] break markexpr = config.getoption('markexpr') or '' # check marker expr set via "pytest -m" if not target and markexpr: # we use `-m "esp32 and generic"` in our CI to filter the test cases # this doesn't cover all use cases, but fit what we do in CI. for marker in markexpr.split('and'): marker = marker.strip() if marker in TARGET_MARKERS: target.append(marker) # "--target" must be set if not target: raise SystemExit( """Pass `--target TARGET[,TARGET...]` to specify all targets the test cases are using. - for single DUT, we run with `pytest --target esp32` - for multi DUT, we run with `pytest --target esp32,esp32,esp32s2` to indicate all DUTs """ ) apps_list = None app_info_basedir = config.getoption('app_info_basedir') app_info_filepattern = config.getoption('app_info_filepattern') if app_info_filepattern: apps_list = [] for file in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)): with open(file) as fr: for line in fr.readlines(): if not line.strip(): continue # each line is a valid json app_info = json.loads(line.strip()) if app_info_basedir and app_info['app_dir'].startswith(app_info_basedir): relative_app_dir = os.path.relpath(app_info['app_dir'], app_info_basedir) apps_list.append(os.path.join(IDF_PATH, os.path.join(relative_app_dir, app_info['build_dir']))) print('Detected app: ', apps_list[-1]) else: print( f'WARNING: app_info base dir {app_info_basedir} not recognizable in {app_info["app_dir"]}, skipping...' ) continue config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded( target=target, apps_list=apps_list, ) config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY]) def pytest_unconfigure(config: Config) -> None: _pytest_embedded = config.stash.get(IDF_PYTEST_EMBEDDED_KEY, None) if _pytest_embedded: del config.stash[IDF_PYTEST_EMBEDDED_KEY] config.pluginmanager.unregister(_pytest_embedded)