test(tools): Moved preset parsing into core_ext.py and added tests

This commit is contained in:
Marek Fiala
2025-09-25 16:21:06 +02:00
committed by BOT
parent 78ae7ab085
commit 1e351a8b67
4 changed files with 311 additions and 162 deletions

View File

@@ -62,10 +62,23 @@ To build and run the app with `prod2` configuration, repeat the steps above, rep
To avoid having to specify `--preset` argument every time you run `idf.py`, you can set `IDF_PRESET` environment variable:
<!-- FIXME: Windows instructions -->
For UNIX-like systems (Linux, macOS):
```shell
export IDF_PRESET=prod1
# subsequent commands will work with 'prod1' configuration:
```
For Windows (PowerShell):
```powershell
$ENV:IDF_PRESET='prod1'
```
For Windows (cmd.exe):
```shell
set IDF_PRESET=prod1
```
Then subsequent commands will work with `prod1` configuration:
```shell
idf.py build
idf.py flash monitor
```

View File

@@ -28,15 +28,15 @@ from idf_py_actions.tools import PropertyDict
from idf_py_actions.tools import TargetChoice
from idf_py_actions.tools import ensure_build_directory
from idf_py_actions.tools import generate_hints
from idf_py_actions.tools import get_build_directory_from_preset
from idf_py_actions.tools import get_cmake_preset
from idf_py_actions.tools import get_target
from idf_py_actions.tools import idf_version
from idf_py_actions.tools import load_cmake_presets
from idf_py_actions.tools import merge_action_lists
from idf_py_actions.tools import run_target
from idf_py_actions.tools import yellow_print
# If a CMake preset with this name exists, it will be used by default when no '--preset' argument is given.
DEFAULT_CMAKE_PRESET_NAME = 'default'
def action_extensions(base_actions: dict, project_path: str) -> Any:
def build_target(target_name: str, ctx: Context, args: PropertyDict) -> None:
@@ -276,26 +276,85 @@ def action_extensions(base_actions: dict, project_path: str) -> Any:
args.build_dir = os.path.realpath(args.build_dir)
def initialize_cmake_presets(args: PropertyDict) -> None:
# Load the CMake presets from the project directory and determine the preset name to use.
load_cmake_presets(args.project_dir, args.preset)
# Get the selected CMake configuration preset, if any
preset_info = get_cmake_preset()
if preset_info:
# If the preset specifies a build directory, use it
build_dir_from_preset = get_build_directory_from_preset()
if build_dir_from_preset:
if args.build_dir:
raise FatalError(
'Build directory specified both in CMake preset and on the command line. This is not supported.'
)
if not os.path.isabs(build_dir_from_preset):
build_dir_from_preset = os.path.join(args.project_dir, build_dir_from_preset)
# Store the build directory back into the args object, the rest of idf.py will look for it there
args.build_dir = build_dir_from_preset
else:
yellow_print(
f'Warning: preset {preset_info["name"]} doesn\'t specify the build directory ("binaryDir")'
)
"""
Initialize and validate the use of CMake presets.
Parse preset, extract variables (like build_dir, generator, preset), and update fields in args accordingly
to ensure ESP-IDF actions use correct values from chosen preset.
"""
preset_name = args.preset
# Load CMake presets from the project directory
cmakepresets_file_names = ['CMakePresets.json', 'CMakeUserPresets.json']
config_presets_info = []
for cmakepresets_file_name in cmakepresets_file_names:
cmakepresets_file_path = os.path.join(args.project_dir, cmakepresets_file_name)
if not os.path.exists(cmakepresets_file_path):
continue
try:
with open(cmakepresets_file_path) as f_in:
presets_info = json.load(f_in)
if not presets_info.get('version'):
raise FatalError(
f'Found CMake preset file without a version in {cmakepresets_file_name}, '
'field "version" is required. '
'Current recommended version is 3, as minimal supported cmake version by ESP-IDF is 3.22.1'
)
for config_preset in presets_info['configurePresets']:
if not config_preset.get('name'):
raise FatalError(
f'Found CMake preset without a name in {cmakepresets_file_name}, '
'field "name" is required'
)
config_presets_info.append(config_preset)
except FatalError as err:
raise err
except Exception as err:
yellow_print(f'Failed to load CMake presets from {cmakepresets_file_name}, {str(err)}')
if not config_presets_info:
if preset_name:
raise FatalError(f"Preset '{preset_name}' specified, but no CMake presets found")
return
preset_names = [preset['name'] for preset in config_presets_info]
# Determine which preset to use
if not preset_name and DEFAULT_CMAKE_PRESET_NAME in preset_names:
yellow_print(
f"CMake presets file found but no preset name given; using '{DEFAULT_CMAKE_PRESET_NAME}' preset"
)
preset_name = DEFAULT_CMAKE_PRESET_NAME
elif not preset_name:
preset_name = preset_names[0]
yellow_print(f"CMake presets file found but no preset name given; using first preset: '{preset_name}'")
elif preset_name not in preset_names:
raise FatalError(f"No preset '{preset_name}' found in CMake presets")
selected_preset_info = next((p_info for p_info in config_presets_info if p_info['name'] == preset_name), None)
if selected_preset_info:
if selected_preset_info.get('inherits'):
yellow_print(f"Preset '{preset_name}' uses inheritance, which is not yet supported.")
# Set build directory from preset
binary_dir = selected_preset_info.get('binaryDir')
if binary_dir and not args.build_dir:
if not os.path.isabs(binary_dir):
binary_dir = os.path.join(args.project_dir, binary_dir)
args.build_dir = binary_dir
elif not binary_dir and not args.build_dir:
yellow_print(f'Warning: preset {preset_name} does not specify the build directory ("binaryDir")')
# Set generator from preset if specified
generator = selected_preset_info.get('generator', None)
if generator and not args.generator:
args.generator = generator
# Store preset name for cmake (we still need to pass --preset to cmake)
args.preset = preset_name
def idf_version_callback(ctx: Context, param: str, value: str) -> None:
if not value or ctx.resilient_parsing:

View File

@@ -34,9 +34,6 @@ SHELL_COMPLETE_VAR = '_IDF.PY_COMPLETE'
# was shell completion invoked?
SHELL_COMPLETE_RUN = SHELL_COMPLETE_VAR in os.environ
# If a CMake preset with this name exists, it will be used by default when no '--preset' argument is given.
DEFAULT_CMAKE_PRESET_NAME = 'default'
# The ctx dict "abuses" how python evaluates default parameter values.
# https://docs.python.org/3/reference/compound_stmts.html#function-definitions
@@ -49,7 +46,6 @@ def get_build_context(ctx: dict = {}) -> dict:
It returns dictionary with the following keys:
'proj_desc' - loaded project_description.json file
'presets' - loaded CMake presets dictionary
Please make sure that ensure_build_directory was called otherwise the build
context dictionary will be empty. Also note that it might not be thread-safe to
@@ -58,11 +54,11 @@ def get_build_context(ctx: dict = {}) -> dict:
return ctx
def _set_build_context_proj_desc(build_dir: str) -> None:
# private helper to save project description to global build context
def _set_build_context(args: 'PropertyDict') -> None:
# private helper to set global build context from ensure_build_directory
ctx = get_build_context()
proj_desc_fn = f'{build_dir}/project_description.json'
proj_desc_fn = f'{args.build_dir}/project_description.json'
try:
with open(proj_desc_fn, encoding='utf-8') as f:
ctx['proj_desc'] = json.load(f)
@@ -70,13 +66,6 @@ def _set_build_context_proj_desc(build_dir: str) -> None:
raise FatalError(f'Cannot load {proj_desc_fn}: {e}')
def _set_build_context_presets(presets: list[dict[str, Any]], preset_name: str | None = None) -> None:
# private helper to save CMake presets to global build context
ctx = get_build_context()
ctx['presets'] = presets
ctx['preset_name'] = preset_name
def executable_exists(args: list) -> bool:
try:
subprocess.check_output(args)
@@ -622,107 +611,6 @@ def _detect_cmake_generator(prog_name: str) -> Any:
raise FatalError(f"To use {prog_name}, either the 'ninja' or 'GNU make' build tool must be available in the PATH")
def load_cmake_presets(project_dir: str, preset_name: str | None = None) -> None:
"""
Try to load CMake presets from the project directory and determine the preset name to use.
If a preset name is provided, check that such a preset exists and select it.
If no preset name is provided, check if the "default" preset exists and select it.
Otherwise, select the first preset.
"""
# See if we have loaded the presets already
cached_presets = get_build_context().get('presets')
if cached_presets:
return
# Load CMake presets from the project directory
cmakepresets_file_names = ['CMakePresets.json', 'CMakeUserPresets.json']
config_presets_info = []
for cmakepresets_file_name in cmakepresets_file_names:
cmakepresets_file_path = os.path.join(project_dir, cmakepresets_file_name)
if not os.path.exists(cmakepresets_file_path):
continue
try:
with open(cmakepresets_file_path) as f_in:
presets_info = json.load(f_in)
for config_preset in presets_info['configurePresets']:
if not config_preset.get('name'):
yellow_print(
f'Found CMake preset without a name in {cmakepresets_file_name}, skipping the preset'
)
continue
# add more preset validations here, if needed
config_presets_info.append(config_preset)
except Exception as err:
yellow_print(f'Failed to load CMake presets from {cmakepresets_file_name}, {str(err)}')
preset_names = [preset['name'] for preset in config_presets_info]
print(preset_names)
if not preset_names:
if preset_name:
raise FatalError(f"Preset '{preset_name}' specified, but no CMake presets found")
return
if not preset_name and DEFAULT_CMAKE_PRESET_NAME in preset_names:
yellow_print(f"CMake presets file found but no preset name given; using '{DEFAULT_CMAKE_PRESET_NAME}' preset")
preset_name = DEFAULT_CMAKE_PRESET_NAME
elif not preset_name:
preset_name = preset_names[0]
yellow_print(f"CMake presets file found but no preset name given; using first preset: '{preset_name}'")
elif preset_name not in preset_names:
raise FatalError(f"No preset '{preset_name}' found in CMake presets")
# Stash the presets dictionary and the selected preset name in the global build context for future use
_set_build_context_presets(config_presets_info, preset_name)
def get_cmake_preset() -> dict[str, Any] | None:
"""
Get the selected CMake configuration preset, or None if no preset is selected.
See the description of load_cmake_presets for details on how the preset is selected.
"""
ctx = get_build_context()
if not ctx.get('presets') or not ctx.get('preset_name'):
return None
return next((p_info for p_info in ctx['presets'] if p_info['name'] == ctx['preset_name']), dict())
def get_build_directory_from_preset() -> str | None:
"""
Get build directory (binaryDir) from the selected CMake configuration preset.
See the description of load_cmake_presets for details on how the preset is selected.
"""
preset_info = get_cmake_preset()
preset_name = preset_info.get('name') # type: ignore
if not preset_info:
return None
if preset_info.get('inherits'):
# TODO: here we also need to check if the preset inherits from other presets,
# and possibly get the build directory from the inherited presets.
yellow_print(f"Preset '{preset_name}' uses inheritance, which is not yet supported. YMMV.")
binary_dir = preset_info.get('binaryDir', None)
if not binary_dir:
raise FatalError(f'Preset \'{preset_name}\' does not specify "binaryDir", this is not supported.')
return str(binary_dir)
def get_generator_from_preset() -> str | None:
"""Return the generator from the selected CMake configuration preset, if any.
Returns ``None`` when no preset is selected or the preset omits the generator.
"""
preset_info = get_cmake_preset()
if not preset_info:
return None
return preset_info.get('generator')
def ensure_build_directory(
args: 'PropertyDict', prog_name: str, always_run_cmake: bool = False, env: dict | None = None
) -> None:
@@ -767,28 +655,16 @@ def ensure_build_directory(
_check_idf_target(args, prog_name, cache, cache_cmdl, env)
if always_run_cmake or _new_cmakecache_entries(cache, cache_cmdl):
# Check if CMake preset specifies the generator
generator_defined_by_preset = False
if args.preset:
generator = get_generator_from_preset()
if generator:
# Won't try to auto-detect the generator and pass it to CMake, as that would
# override the choice made by the preset.
generator_defined_by_preset = True
# No generator specified in CLI or in preset, so auto-detect it
if args.generator is None and not generator_defined_by_preset:
if args.generator is None:
args.generator = _detect_cmake_generator(prog_name)
# Pass the generator to CMake if one was specified in CLI or auto-detected
generator_args = []
if args.generator:
generator_args = ['-G', args.generator]
try:
cmake_args = [
'cmake',
*generator_args,
'-G',
args.generator,
'-B',
args.build_dir,
'-DPYTHON_DEPS_CHECKED=1',
f'-DPYTHON={sys.executable}',
'-DESP_PLATFORM=1',
@@ -848,8 +724,8 @@ def ensure_build_directory(
except KeyError:
pass
# Load project metadata file into global build context
_set_build_context_proj_desc(args.build_dir)
# set global build context
_set_build_context(args)
def merge_action_lists(*action_lists: dict, custom_actions: dict[str, Any] | None = None) -> dict:

View File

@@ -1,5 +1,6 @@
# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import json
import logging
import os
import re
@@ -14,6 +15,7 @@ from test_build_system_helpers import append_to_file
from test_build_system_helpers import file_contains
from test_build_system_helpers import run_cmake
from test_build_system_helpers import run_cmake_and_build
from test_build_system_helpers import run_idf_py
# This test checks multiple targets in one test function. It would be better to have each target
@@ -63,7 +65,7 @@ def test_build_cmake_library_psram_workaround(test_app_copy: Path) -> None:
'-DSDKCONFIG_DEFAULTS={}'.format(test_app_copy / 'sdkconfig.defaults'),
str(idf_path / 'examples' / 'build_system' / 'cmake' / 'import_lib'),
)
with open((test_app_copy / 'build' / 'compile_commands.json'), 'r', encoding='utf-8') as f:
with open((test_app_copy / 'build' / 'compile_commands.json'), encoding='utf-8') as f:
data = f.read()
res = re.findall(r'.*\"command\".*', data)
for r in res:
@@ -85,7 +87,7 @@ def test_build_cmake_library_psram_strategies(idf_py: IdfPyFunc, test_app_copy:
)
)
idf_py('reconfigure')
with open((test_app_copy / 'build' / 'compile_commands.json'), 'r', encoding='utf-8') as f:
with open((test_app_copy / 'build' / 'compile_commands.json'), encoding='utf-8') as f:
data = f.read()
res = re.findall(r'.*\"command\".*', data)
for r in res:
@@ -110,7 +112,7 @@ def test_defaults_unspecified_build_args(idf_copy: Path) -> None:
'-DTARGET=esp32',
workdir=idf_as_lib_path,
)
assert 'Project directory: {}'.format(str(idf_as_lib_path.as_posix())) in ret.stderr
assert f'Project directory: {str(idf_as_lib_path.as_posix())}' in ret.stderr
def test_build_example_on_host(default_idf_env: EnvDict) -> None:
@@ -130,3 +132,202 @@ def test_build_example_on_host(default_idf_env: EnvDict) -> None:
run_cmake('--build', '.', workdir=idf_as_lib_path)
finally:
shutil.rmtree(idf_as_lib_path / 'build', ignore_errors=True)
def test_cmake_preset_basic_functionality(test_app_copy: Path, default_idf_env: EnvDict) -> None:
logging.info('Test basic CMake preset functionality with multiple presets')
# Create a CMakePresets.json file with multiple configurations
presets_content = {
'version': 3,
'configurePresets': [
{
'name': 'default',
'binaryDir': 'build/default',
'displayName': 'Default Configuration',
'cacheVariables': {'SDKCONFIG': './build/default/sdkconfig'},
},
{
'name': 'prod1',
'binaryDir': 'build/prod1',
'displayName': 'Production 1',
'cacheVariables': {
'SDKCONFIG_DEFAULTS': 'sdkconfig.prod_common;sdkconfig.prod1',
'SDKCONFIG': './build/prod1/sdkconfig',
},
},
{
'name': 'prod2',
'binaryDir': 'build/prod2',
'displayName': 'Production 2',
'cacheVariables': {
'SDKCONFIG_DEFAULTS': 'sdkconfig.prod_common;sdkconfig.prod2',
'SDKCONFIG': './build/prod2/sdkconfig',
},
},
],
}
# Write the presets file
presets_file = test_app_copy / 'CMakePresets.json'
with open(presets_file, 'w') as f:
json.dump(presets_content, f, indent=4)
(test_app_copy / 'sdkconfig.prod_common').write_text('CONFIG_LWIP_IPV6=y\n')
(test_app_copy / 'sdkconfig.prod1').write_text('CONFIG_ESP_TASK_WDT_TIMEOUT_S=10\n')
(test_app_copy / 'sdkconfig.prod2').write_text('CONFIG_ESP_TASK_WDT_TIMEOUT_S=20\n')
# Test default preset auto-selection
ret = run_idf_py('reconfigure')
assert "CMake presets file found but no preset name given; using 'default' preset" in ret.stderr
assert (test_app_copy / 'build' / 'default').is_dir()
assert (test_app_copy / 'build' / 'default' / 'sdkconfig').is_file()
# Verify that sdkconfig is NOT in the project root, even when no preset is specified but auto-selected
assert not (test_app_copy / 'sdkconfig').is_file(), (
'sdkconfig should not be in project root when preset specifies custom location'
)
# Test explicit preset selection
ret = run_idf_py('--preset', 'prod1', 'reconfigure')
assert (test_app_copy / 'build' / 'prod1').is_dir()
assert (test_app_copy / 'build' / 'prod1' / 'sdkconfig').is_file()
# Test env variable preset selection
env_with_preset = default_idf_env.copy()
env_with_preset['IDF_PRESET'] = 'prod2'
ret = run_idf_py('reconfigure', env=env_with_preset)
assert (test_app_copy / 'build' / 'prod2').is_dir()
assert (test_app_copy / 'build' / 'prod2' / 'sdkconfig').is_file()
def test_cmake_preset_build_directory_precedence(test_app_copy: Path) -> None:
logging.info('Test CMake preset build directory precedence - command line overrides preset')
presets_content = {
'version': 3,
'configurePresets': [{'name': 'test_preset', 'binaryDir': 'build/preset_dir', 'displayName': 'Test Preset'}],
}
presets_file = test_app_copy / 'CMakePresets.json'
with open(presets_file, 'w') as f:
json.dump(presets_content, f, indent=4)
ret = run_idf_py('--preset', 'test_preset', '-B', 'custom_build', 'reconfigure')
assert 'custom_build' in ret.stdout
assert 'build/preset_dir' not in ret.stdout
def test_cmake_preset_without_binary_dir(test_app_copy: Path) -> None:
logging.info('Test CMake preset without binaryDir specification')
presets_content = {
'version': 3,
'configurePresets': [{'name': 'no_binary_dir', 'displayName': 'Preset without binaryDir'}],
}
presets_file = test_app_copy / 'CMakePresets.json'
with open(presets_file, 'w') as f:
json.dump(presets_content, f, indent=4)
ret = run_idf_py('--preset', 'no_binary_dir', 'reconfigure', check=False)
assert 'does not specify the build directory ("binaryDir")' in ret.stderr
def test_cmake_preset_error_cases(test_app_copy: Path) -> None:
logging.info('Test CMake preset error cases and invalid inputs')
# Test with invalid JSON
presets_file = test_app_copy / 'CMakePresets.json'
presets_file.write_text('{ invalid json content')
ret = run_idf_py('--preset', 'nonexistent', 'reconfigure', check=False)
assert ret.returncode != 0
# Clean up and test with valid JSON but nonexistent preset
presets_content = {
'version': 3,
'configurePresets': [
{'name': 'existing_preset', 'binaryDir': 'build/existing', 'displayName': 'Existing Preset'}
],
}
with open(presets_file, 'w') as f:
json.dump(presets_content, f, indent=4)
ret = run_idf_py('--preset', 'nonexistent_preset', 'reconfigure', check=False)
assert ret.returncode != 0
assert "No preset 'nonexistent_preset' found" in ret.stderr
presets_content_no_name = {
'version': 3,
'configurePresets': [
{'binaryDir': 'build/no_name', 'displayName': 'Preset without name'},
{'name': 'valid_preset', 'binaryDir': 'build/valid', 'displayName': 'Valid Preset'},
],
}
with open(presets_file, 'w') as f:
json.dump(presets_content_no_name, f, indent=4)
ret = run_idf_py('reconfigure', check=False)
assert 'Found CMake preset without a name' in ret.stderr
assert ret.returncode != 0
presets_content_no_version = {
'configurePresets': [
{'name': 'no_version', 'binaryDir': 'build/no_version', 'displayName': 'Preset without version'},
],
}
with open(presets_file, 'w') as f:
json.dump(presets_content_no_version, f, indent=4)
ret = run_idf_py('reconfigure', check=False)
assert 'Found CMake preset file without a version' in ret.stderr
assert ret.returncode != 0
def test_cmake_preset_sdkconfig_defaults_integration(test_app_copy: Path) -> None:
logging.info('Test CMake preset integration with SDKCONFIG_DEFAULTS')
presets_content = {
'version': 3,
'configurePresets': [
{
'name': 'with_defaults',
'binaryDir': 'build/with_defaults',
'displayName': 'Preset with SDKCONFIG_DEFAULTS',
'cacheVariables': {
'SDKCONFIG_DEFAULTS': 'sdkconfig.preset1;sdkconfig.preset2',
'SDKCONFIG': './build/with_defaults/sdkconfig',
},
},
{
'name': 'prod1',
'binaryDir': 'build/prod1',
'displayName': 'Production 1',
'cacheVariables': {
'SDKCONFIG_DEFAULTS': 'sdkconfig.prod_common;sdkconfig.prod1',
'SDKCONFIG': './build/prod1/sdkconfig',
},
},
],
}
presets_file = test_app_copy / 'CMakePresets.json'
with open(presets_file, 'w') as f:
json.dump(presets_content, f, indent=4)
(test_app_copy / 'sdkconfig.preset1').write_text('CONFIG_LWIP_IPV6=y\n')
(test_app_copy / 'sdkconfig.preset2').write_text('CONFIG_ESP_TASK_WDT_TIMEOUT_S=15\n')
run_idf_py('--preset', 'with_defaults', 'reconfigure')
assert (test_app_copy / 'build' / 'with_defaults').is_dir()
sdkconfig_path = test_app_copy / 'build' / 'with_defaults' / 'sdkconfig'
assert sdkconfig_path.is_file()
sdkconfig_content = sdkconfig_path.read_text()
assert 'CONFIG_LWIP_IPV6=y' in sdkconfig_content
assert 'CONFIG_ESP_TASK_WDT_TIMEOUT_S=15' in sdkconfig_content