alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2025-12-15 19:34:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: adjust parallel count based on build item count

Browse Source
This commit is contained in:
Fu Hanxi
2022-01-29 14:38:56 +08:00
parent 30f098cd49
commit e761153cff
3 changed files with 57 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
tools/ci/python_packages/ttfw_idf/CIScanTests.py
View File

@@ -8,8 +8,8 @@ from copy import deepcopy
from typing import Any
from ci.idf_ci_utils import get_pytest_app_paths
from find_apps import find_apps
from find_build_apps import BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS
from find_apps import find_apps, find_builds_for_app
from find_build_apps import BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS, config_rules_from_str
from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
from ttfw_idf.IDFAssignTest import ExampleAssignTest, TestAppsAssignTest
@@ -29,6 +29,8 @@ BUILD_ALL_LABELS = [
'BOT_LABEL_WEEKEND_TEST',
]
BUILD_PER_JOB = 30 # each build takes 1 mins around
def _has_build_all_label(): # type: () -> bool
for label in BUILD_ALL_LABELS:
@@ -101,6 +103,19 @@ def main(): # type: () -> None
help='add this flag to combine all target jsons into one')
parser.add_argument('--except-targets', nargs='+',
help='only useful when "--combine-all-targets". Specified targets would be skipped.')
parser.add_argument(
'--config',
action='append',
help='Only useful when "--evaluate-parallel-count" is flagged.'
'Adds configurations (sdkconfig file names) to build. This can either be ' +
'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' +
'relative to the project directory, to be used. Optional NAME can be specified, ' +
'which can be used as a name of this configuration. FILEPATTERN is the name of ' +
'the sdkconfig file, relative to the project directory, with at most one wildcard. ' +
'The part captured by the wildcard is used as the name of the configuration.',
)
parser.add_argument('--evaluate-parallel-count', action='store_true',
help='suggest parallel count according to build items')
args = parser.parse_args()
build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all)
@@ -175,6 +190,7 @@ def main(): # type: () -> None
scan_info_dict[target]['standalone_apps'] = set()
test_case_apps_preserve_default = True if build_system == 'cmake' else False
output_files = []
build_items_total_count = 0
for target in SUPPORTED_TARGETS:
# get pytest apps paths
pytest_app_paths = set()
@@ -204,15 +220,35 @@ def main(): # type: () -> None
})
output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system))
with open(output_path, 'w') as fw:
if args.evaluate_parallel_count:
build_items = []
config_rules = config_rules_from_str(args.config or [])
for app in apps:
build_items += find_builds_for_app(
app['app_dir'],
app['app_dir'],
'build',
'',
app['target'],
app['build_system'],
config_rules,
app['preserve'],
)
print('Found {} builds'.format(len(build_items)))
if args.combine_all_targets:
if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \
or (not args.except_targets):
build_items_total_count += len(build_items)
else:
print(f'suggest set parallel count for target {target} to {len(build_items) // BUILD_PER_JOB + 1}')
fw.writelines([json.dumps(app) + '\n' for app in apps])
if args.combine_all_targets:
if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \
or (not args.except_targets):
output_files.append(output_path)
build_items_total_count += len(build_items)
else:
print(f'skipping combining target {target}')
if args.combine_all_targets:
if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \
or (not args.except_targets):
output_files.append(output_path)
else:
print(f'skipping combining target {target}')
if args.combine_all_targets:
scan_all_json = os.path.join(args.output_path, f'scan_all_{build_system}.json')
@@ -224,6 +260,9 @@ def main(): # type: () -> None
fw.writelines(lines)
print(f'combined into file: {scan_all_json}')
if args.evaluate_parallel_count:
print(f'Total build: {build_items_total_count}. Suggest set parallel count for all target to {build_items_total_count // BUILD_PER_JOB + 1}')
if __name__ == '__main__':
main()