diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 3b68f1531c..98181f28e0 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -147,3 +147,12 @@ check_version_tag: - .rules:tag:release-no_label script: - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1) + +check_tools_file_patterns: + extends: .pre_check_job_template + image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG + variables: + PYTHON_VER: 3.7.7 + script: + - python tools/ci/check_tools_files_patterns.py + allow_failure: true diff --git a/tools/ci/check_tools_files_patterns.py b/tools/ci/check_tools_files_patterns.py new file mode 100755 index 0000000000..1aee9815f8 --- /dev/null +++ b/tools/ci/check_tools_files_patterns.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# +# Copyright 2021 Espressif Systems (Shanghai) CO LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import fnmatch +import glob +import os +import sys + +import yaml +from idf_ci_utils import IDF_PATH, get_git_files, magic_check, magic_check_bytes, translate + +# Monkey patch starts + +# glob.glob will ignore all files starts with ``.`` +# don't ignore them here +# need to keep the same argument as glob._ishidden + + +def _ishidden(path): # pylint: disable=W0613 + return False + + +fnmatch.translate = translate + +glob.magic_check = magic_check +glob.magic_check_bytes = magic_check_bytes +glob._ishidden = _ishidden # pylint: disable=W0212 +# ends here + + +def check(pattern_yml, exclude_list): + rules_dict = yaml.load(open(pattern_yml), Loader=yaml.FullLoader) + rules_patterns_set = set() + for k, v in rules_dict.items(): + if k.startswith('.pattern') and isinstance(v, list): + rules_patterns_set.update(v) + rules_files_set = set() + for pat in rules_patterns_set: + rules_files_set.update(glob.glob(os.path.join(IDF_PATH, pat), recursive=True)) + + exclude_patterns_set = set() + exclude_patterns_set.update([path.split('#')[0].strip() for path in open(exclude_list).readlines() if path]) + exclude_files_set = set() + for pat in exclude_patterns_set: + exclude_files_set.update(glob.glob(os.path.join(IDF_PATH, pat), recursive=True)) + + missing_files = set() + git_files = get_git_files(os.path.join(IDF_PATH, 'tools'), full_path=True) + for f in git_files: + if f in rules_files_set or f in exclude_files_set: + continue + missing_files.add(os.path.relpath(f, IDF_PATH)) + + return missing_files, rules_patterns_set.intersection(exclude_patterns_set) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='check if all tools files are in rules patterns or exclude list') + parser.add_argument('-c', '--pattern-yml', + default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), + help='yml file path included file patterns') + parser.add_argument('-e', '--exclude-list', + default=os.path.join(IDF_PATH, 'tools', 'ci', 'exclude_check_tools_files.txt'), + help='exclude list path') + + args = parser.parse_args() + + res = 0 + not_included_files, dup_patterns = check(args.pattern_yml, args.exclude_list) + if not_included_files: + print('Missing Files: (please add to tools/ci/exclude_check_tools_files.txt') + for f in not_included_files: + print(f) + res = 1 + if dup_patterns: + print('Duplicated Patterns: (please check .gitlab/ci/rules.yml and tools/ci/exclude_check_tools_files.txt') + for pat in dup_patterns: + print(pat) + res = 1 + + sys.exit(res) diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt new file mode 100644 index 0000000000..3d8adc0218 --- /dev/null +++ b/tools/ci/exclude_check_tools_files.txt @@ -0,0 +1,32 @@ +tools/ble/**/* +tools/catch/**/* +tools/ci/build_template_app.sh +tools/ci/check_*.{py,txt,sh} # excluded because run in default pipeline pre-check stage +tools/ci/checkout_project_ref.py +tools/ci/ci_fetch_submodule.py +tools/ci/ci_get_mr_info.py +tools/ci/configure_ci_environment.sh +tools/ci/deploy_docs.py +tools/ci/envsubst.py +tools/ci/*exclude*.txt +tools/ci/executable-list.txt +tools/ci/fix_empty_prototypes.sh +tools/ci/get-full-sources.sh +tools/ci/idf_ci_utils.py +tools/ci/mirror-submodule-update.sh +tools/ci/multirun_with_pyenv.sh +tools/ci/normalize_clangtidy_path.py +tools/ci/push_to_github.sh +tools/ci/python_packages/wifi_tools.py +tools/ci/setup_python.sh +tools/ci/utils.sh +tools/eclipse-code-style.xml +tools/format-minimal.sh +tools/format.sh +tools/gen_esp_err_to_name.py +tools/kconfig/**/* +tools/set-submodules-to-github.sh +tools/templates/sample_component/CMakeLists.txt +tools/templates/sample_component/include/main.h +tools/templates/sample_component/main.c +tools/toolchain_versions.mk diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 1c446afa3e..55fe56476e 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -46,6 +46,7 @@ tools/ci/check_idf_version.sh tools/ci/check_kconfigs.py tools/ci/check_readme_links.py tools/ci/check_rom_apis.sh +tools/ci/check_tools_files_patterns.py tools/ci/check_ut_cmake_make.sh tools/ci/checkout_project_ref.py tools/ci/deploy_docs.py diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index ee92b5db0d..bcb12ea491 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -15,13 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # - +import functools import logging import os +import re import subprocess import sys -IDF_PATH = os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')) +IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))) def get_submodule_dirs(full_path=False): # type: (bool) -> list @@ -41,7 +42,7 @@ def get_submodule_dirs(full_path=False): # type: (bool) -> list dirs.append(os.path.join(IDF_PATH, path)) else: dirs.append(path) - except Exception as e: + except Exception as e: # pylint: disable=W0703 logging.warning(str(e)) return dirs @@ -67,5 +68,132 @@ def is_executable(full_path): # type: (str) -> bool """ if sys.platform == 'win32': return _check_git_filemode(full_path) - else: - return os.access(full_path, os.X_OK) + return os.access(full_path, os.X_OK) + + +def get_git_files(path=IDF_PATH, full_path=False): # type: (str, bool) -> list[str] + """ + Get the result of git ls-files + :param path: path to run git ls-files + :param full_path: return full path if set to True + :return: list of file paths + """ + try: + files = subprocess.check_output(['git', 'ls-files'], cwd=path).decode('utf8').strip().split('\n') + except Exception as e: # pylint: disable=W0703 + logging.warning(str(e)) + files = [] + return [os.path.join(path, f) for f in files] if full_path else files + + +# this function is a commit from +# https://github.com/python/cpython/pull/6299/commits/bfd63120c18bd055defb338c075550f975e3bec1 +# In order to solve python https://bugs.python.org/issue9584 +# glob pattern does not support brace expansion issue +def _translate(pat): + """Translate a shell PATTERN to a regular expression. + There is no way to quote meta-characters. + """ + i, n = 0, len(pat) + res = '' + while i < n: + c = pat[i] + i = i + 1 + if c == '*': + res = res + '.*' + elif c == '?': + res = res + '.' + elif c == '[': + j = i + if j < n and pat[j] == '!': + j = j + 1 + if j < n and pat[j] == ']': + j = j + 1 + while j < n and pat[j] != ']': + j = j + 1 + if j >= n: + res = res + '\\[' + else: + stuff = pat[i:j] + if '--' not in stuff: + stuff = stuff.replace('\\', r'\\') + else: + chunks = [] + k = i + 2 if pat[i] == '!' else i + 1 + while True: + k = pat.find('-', k, j) + if k < 0: + break + chunks.append(pat[i:k]) + i = k + 1 + k = k + 3 + chunks.append(pat[i:j]) + # Escape backslashes and hyphens for set difference (--). + # Hyphens that create ranges shouldn't be escaped. + stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') + for s in chunks) + # Escape set operations (&&, ~~ and ||). + stuff = re.sub(r'([&~|])', r'\\\1', stuff) + i = j + 1 + if stuff[0] == '!': + stuff = '^' + stuff[1:] + elif stuff[0] in ('^', '['): + stuff = '\\' + stuff + res = '%s[%s]' % (res, stuff) + elif c == '{': + # Handling of brace expression: '{PATTERN,PATTERN,...}' + j = 1 + while j < n and pat[j] != '}': + j = j + 1 + if j >= n: + res = res + '\\{' + else: + stuff = pat[i:j] + i = j + 1 + + # Find indices of ',' in pattern excluding r'\,'. + # E.g. for r'a\,a,b\b,c' it will be [4, 8] + indices = [m.end() for m in re.finditer(r'[^\\],', stuff)] + + # Splitting pattern string based on ',' character. + # Also '\,' is translated to ','. E.g. for r'a\,a,b\b,c': + # * first_part = 'a,a' + # * last_part = 'c' + # * middle_part = ['b,b'] + first_part = stuff[:indices[0] - 1].replace(r'\,', ',') + last_part = stuff[indices[-1]:].replace(r'\,', ',') + middle_parts = [ + stuff[st:en - 1].replace(r'\,', ',') + for st, en in zip(indices, indices[1:]) + ] + + # creating the regex from splitted pattern. Each part is + # recursivelly evaluated. + expanded = functools.reduce( + lambda a, b: '|'.join((a, b)), + (_translate(elem) for elem in [first_part] + middle_parts + [last_part]) + ) + res = '%s(%s)' % (res, expanded) + else: + res = res + re.escape(c) + return res + + +def translate(pat): + res = _translate(pat) + return r'(?s:%s)\Z' % res + + +magic_check = re.compile('([*?[{])') +magic_check_bytes = re.compile(b'([*?[{])') +# cpython github PR 6299 ends here + +# Here's the code block we're going to use to monkey patch ``glob`` module and ``fnmatch`` modules +# DO NOT monkey patch here, only patch where you really needs +# +# import glob +# import fnmatch +# from idf_ci_utils import magic_check, magic_check_bytes, translate +# glob.magic_check = magic_check +# glob.magic_check_bytes = magic_check_bytes +# fnmatch.translate = translate