alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-10 04:43:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove build_or_not attr.

Browse Source
This commit is contained in:
Fu Hanxi
2020-06-29 18:11:49 +08:00
parent 2c2f66aa21
commit 4087152305
5 changed files with 98 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tools/build_apps.py
View File

@@ -77,10 +77,9 @@ def main():
setup_logging(args) setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list] build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items: if not build_items:
logging.error("Empty build list!") logging.warning("Empty build list")
raise SystemExit(1) SystemExit(0)
num_builds = len(build_items) num_builds = len(build_items)
num_jobs = args.parallel_count num_jobs = args.parallel_count
@@ -109,10 +108,6 @@ def main():
failed_builds = [] failed_builds = []
for build_info in builds_for_current_job: for build_info in builds_for_current_job:
if not build_info.build:
logging.info("Skip building app {}".format(build_info.app_dir))
continue
logging.info("Running build {}: {}".format(build_info.index, repr(build_info))) logging.info("Running build {}: {}".format(build_info.index, repr(build_info)))
build_system_class = BUILD_SYSTEMS[build_info.build_system] build_system_class = BUILD_SYSTEMS[build_info.build_system]
try: try:

8
tools/ci/check_build_warnings.py
View File

@@ -70,16 +70,12 @@ def main():
setup_logging(args) setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list] build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items: if not build_items:
logging.error("Empty build list!") logging.warning("Empty build list")
raise SystemExit(1) SystemExit(0)
found_warnings = 0 found_warnings = 0
for build_item in build_items: for build_item in build_items:
if not build_item.build:
logging.debug("Skip checking build log for app {}".format(build_item.app_dir))
continue
if not build_item.build_log_path: if not build_item.build_log_path:
logging.debug("No log file for {}".format(build_item.work_dir)) logging.debug("No log file for {}".format(build_item.work_dir))
continue continue

163
tools/ci/python_packages/ttfw_idf/CIScanTests.py
View File

@@ -1,6 +1,7 @@
import argparse import argparse
import errno import errno
import json import json
import logging
import os import os
import re import re
from collections import defaultdict from collections import defaultdict
@@ -14,80 +15,73 @@ VALID_TARGETS = [
'esp32s2', 'esp32s2',
] ]
SPECIAL_REFS = [ TEST_LABELS = {
'master', 'example_test': 'BOT_LABEL_EXAMPLE_TEST',
re.compile(r'^release/v'), 'test_apps': 'BOT_LABEL_CUSTOM_TEST',
re.compile(r'^v\d+\.\d+'), }
BUILD_ALL_LABELS = [
'BOT_LABEL_BUILD_ALL_APPS',
'BOT_LABEL_REGULAR_TEST',
] ]
def _judge_build_all(args_build_all): def _has_build_all_label():
if args_build_all: for label in BUILD_ALL_LABELS:
return True if os.getenv(label):
if os.getenv('BUILD_ALL_APPS'): return True
return True
ref = os.getenv('CI_COMMIT_REF_NAME')
pipeline_src = os.getenv('CI_PIPELINE_SOURCE')
if not ref or not pipeline_src:
return False
# scheduled pipeline will build all
if pipeline_src == 'schedule':
return True
# master, release/v..., v1.2.3..., and will build all
for special_ref in SPECIAL_REFS:
if isinstance(special_ref, re._pattern_type):
if special_ref.match(ref):
return True
else:
if ref == special_ref:
return True
return False return False
def _judge_build_or_not(action, build_all): # type: (str, bool) -> (bool, bool)
"""
:return: (build_or_not_for_test_related_apps, build_or_not_for_non_related_apps)
"""
if build_all or _has_build_all_label() or (not os.getenv('BOT_TRIGGER_WITH_LABEL')):
logging.info('Build all apps')
return True, True
if os.getenv(TEST_LABELS[action]):
logging.info('Build test cases apps')
return True, False
else:
logging.info('Skip all')
return False, False
def output_json(apps_dict_list, target, build_system, output_dir):
output_path = os.path.join(output_dir, 'scan_{}_{}.json'.format(target.lower(), build_system))
with open(output_path, 'w') as fw:
fw.writelines([json.dumps(app) + '\n' for app in apps_dict_list])
def main(): def main():
parser = argparse.ArgumentParser(description='Scan the required build tests') parser = argparse.ArgumentParser(description='Scan the required build tests')
actions = parser.add_subparsers(dest='action') parser.add_argument('test_type',
choices=TEST_LABELS.keys(),
common = argparse.ArgumentParser(add_help=False) help='Scan test type')
common.add_argument('paths', parser.add_argument('paths',
nargs='+', nargs='+',
help="One or more app paths") help='One or more app paths')
common.add_argument('-b', '--build-system', parser.add_argument('-b', '--build-system',
choices=BUILD_SYSTEMS.keys(), choices=BUILD_SYSTEMS.keys(),
default=BUILD_SYSTEM_CMAKE) default=BUILD_SYSTEM_CMAKE)
common.add_argument('-c', '--ci-config-file', parser.add_argument('-c', '--ci-config-file',
required=True, required=True,
help="gitlab ci config target-test file") help="gitlab ci config target-test file")
common.add_argument('-o', '--output-path', parser.add_argument('-o', '--output-path',
required=True, required=True,
help="output path of the scan result") help="output path of the scan result")
common.add_argument("--exclude", parser.add_argument("--exclude",
action="append", action="append",
help="Ignore specified directory. Can be used multiple times.") help='Ignore specified directory. Can be used multiple times.')
common.add_argument('--preserve', action="store_true", parser.add_argument('--preserve', action="store_true",
help='add this flag to preserve artifacts for all apps') help='add this flag to preserve artifacts for all apps')
common.add_argument('--build-all', action="store_true", parser.add_argument('--build-all', action="store_true",
help='add this flag to build all apps') help='add this flag to build all apps')
actions.add_parser('example_test', parents=[common])
actions.add_parser('test_apps', parents=[common])
args = parser.parse_args() args = parser.parse_args()
build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all)
test_cases = []
for path in args.paths:
if args.action == 'example_test':
assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup)
elif args.action == 'test_apps':
CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+')
assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup)
else:
raise SystemExit(1) # which is impossible
test_cases.extend(assign.search_cases())
if not os.path.exists(args.output_path): if not os.path.exists(args.output_path):
try: try:
@@ -96,6 +90,23 @@ def main():
if e.errno != errno.EEXIST: if e.errno != errno.EEXIST:
raise e raise e
if (not build_standalone_apps) and (not build_test_case_apps):
for target in VALID_TARGETS:
output_json([], target, args.build_system, args.output_path)
SystemExit(0)
test_cases = []
for path in set(args.paths):
if args.test_type == 'example_test':
assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup)
elif args.test_type == 'test_apps':
CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+')
assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup)
else:
raise SystemExit(1) # which is impossible
test_cases.extend(assign.search_cases())
''' '''
{ {
<target>: { <target>: {
@@ -113,32 +124,37 @@ def main():
build_system = args.build_system.lower() build_system = args.build_system.lower()
build_system_class = BUILD_SYSTEMS[build_system] build_system_class = BUILD_SYSTEMS[build_system]
for target in VALID_TARGETS: if build_test_case_apps:
target_dict = scan_info_dict[target] for target in VALID_TARGETS:
test_case_apps = target_dict['test_case_apps'] = set() target_dict = scan_info_dict[target]
for case in test_cases: test_case_apps = target_dict['test_case_apps'] = set()
app_dir = case.case_info['app_dir'] for case in test_cases:
app_target = case.case_info['target'] app_dir = case.case_info['app_dir']
if app_target.lower() != target.lower(): app_target = case.case_info['target']
continue if app_target.lower() != target.lower():
test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower())) continue
exclude_apps.append(app_dir) test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower()))
exclude_apps.append(app_dir)
else:
for target in VALID_TARGETS:
scan_info_dict[target]['test_case_apps'] = set()
for target in VALID_TARGETS: if build_standalone_apps:
target_dict = scan_info_dict[target] for target in VALID_TARGETS:
standalone_apps = target_dict['standalone_apps'] = set() target_dict = scan_info_dict[target]
for path in args.paths: standalone_apps = target_dict['standalone_apps'] = set()
standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower())) for path in args.paths:
standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower()))
else:
for target in VALID_TARGETS:
scan_info_dict[target]['standalone_apps'] = set()
build_all = _judge_build_all(args.build_all)
test_case_apps_preserve_default = True if build_system == 'cmake' else False test_case_apps_preserve_default = True if build_system == 'cmake' else False
for target in VALID_TARGETS: for target in VALID_TARGETS:
apps = [] apps = []
for app_dir in scan_info_dict[target]['test_case_apps']: for app_dir in scan_info_dict[target]['test_case_apps']:
apps.append({ apps.append({
'app_dir': app_dir, 'app_dir': app_dir,
'build': True,
'build_system': args.build_system, 'build_system': args.build_system,
'target': target, 'target': target,
'preserve': args.preserve or test_case_apps_preserve_default 'preserve': args.preserve or test_case_apps_preserve_default
@@ -146,10 +162,9 @@ def main():
for app_dir in scan_info_dict[target]['standalone_apps']: for app_dir in scan_info_dict[target]['standalone_apps']:
apps.append({ apps.append({
'app_dir': app_dir, 'app_dir': app_dir,
'build': build_all if build_system == 'cmake' else True,
'build_system': args.build_system, 'build_system': args.build_system,
'target': target, 'target': target,
'preserve': (args.preserve and build_all) if build_system == 'cmake' else False 'preserve': args.preserve
}) })
output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system)) output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system))
with open(output_path, 'w') as fw: with open(output_path, 'w') as fw:

14
tools/find_apps.py
View File

@@ -49,8 +49,8 @@ def dict_from_sdkconfig(path):
def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg, def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
build_system, config_rules, build_or_not=True, preserve_artifacts=True): build_system, config_rules, preserve_artifacts=True):
# type: (str, str, str, str, str, str, typing.List[ConfigRule], bool, bool) -> typing.List[BuildItem] # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool) -> typing.List[BuildItem]
""" """
Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects
:param app_path: app directory (can be / usually will be a relative path) :param app_path: app directory (can be / usually will be a relative path)
@@ -63,7 +63,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
a different CONFIG_IDF_TARGET value. a different CONFIG_IDF_TARGET value.
:param build_system: name of the build system, index into BUILD_SYSTEMS dictionary :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary
:param config_rules: mapping of sdkconfig file name patterns to configuration names :param config_rules: mapping of sdkconfig file name patterns to configuration names
:param build_or_not: determine if it will build in build_apps.py
:param preserve_artifacts: determine if the built binary will be uploaded as artifacts. :param preserve_artifacts: determine if the built binary will be uploaded as artifacts.
:return: list of BuildItems representing build configuration of the app :return: list of BuildItems representing build configuration of the app
""" """
@@ -109,7 +108,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
sdkconfig_path, sdkconfig_path,
config_name, config_name,
build_system, build_system,
build_or_not,
preserve_artifacts, preserve_artifacts,
)) ))
@@ -125,7 +123,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
None, None,
default_config_name, default_config_name,
build_system, build_system,
build_or_not,
preserve_artifacts, preserve_artifacts,
) )
] ]
@@ -296,10 +293,10 @@ def main():
apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs] apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs]
if not apps: if not apps:
logging.critical("No apps found") logging.warning("No apps found")
raise SystemExit(1) SystemExit(0)
logging.info("Found {} apps".format(len(apps)))
logging.info("Found {} apps".format(len(apps)))
apps.sort(key=lambda x: x["app_dir"]) apps.sort(key=lambda x: x["app_dir"])
# Find compatible configurations of each app, collect them as BuildItems # Find compatible configurations of each app, collect them as BuildItems
@@ -314,7 +311,6 @@ def main():
args.target or app["target"], args.target or app["target"],
args.build_system or app["build_system"], args.build_system or app["build_system"],
config_rules, config_rules,
app["build"],
app["preserve"], app["preserve"],
) )
logging.info("Found {} builds".format(len(build_items))) logging.info("Found {} builds".format(len(build_items)))

4
tools/find_build_apps/common.py
View File

@@ -71,7 +71,6 @@ class BuildItem(object):
sdkconfig_path, sdkconfig_path,
config_name, config_name,
build_system, build_system,
build_or_not,
preserve_artifacts, preserve_artifacts,
): ):
# These internal variables store the paths with environment variables and placeholders; # These internal variables store the paths with environment variables and placeholders;
@@ -86,7 +85,6 @@ class BuildItem(object):
self.target = target self.target = target
self.build_system = build_system self.build_system = build_system
self.build = build_or_not
self.preserve = preserve_artifacts self.preserve = preserve_artifacts
self._app_name = os.path.basename(os.path.normpath(app_path)) self._app_name = os.path.basename(os.path.normpath(app_path))
@@ -160,7 +158,6 @@ class BuildItem(object):
"config": self.config_name, "config": self.config_name,
"target": self.target, "target": self.target,
"verbose": self.verbose, "verbose": self.verbose,
"build": self.build,
"preserve": self.preserve, "preserve": self.preserve,
}) })
@@ -179,7 +176,6 @@ class BuildItem(object):
config_name=d["config"], config_name=d["config"],
target=d["target"], target=d["target"],
build_system=d["build_system"], build_system=d["build_system"],
build_or_not=d["build"],
preserve_artifacts=d["preserve"] preserve_artifacts=d["preserve"]
) )
result.verbose = d["verbose"] result.verbose = d["verbose"]