ci(rules): auto generate rules.yml labels titles and rules

Also simplified the labels and ci

- Remove label regular_test, weekend_test

- Remove apply_job_filter

- check_submodule_sync: only run on protected branch

.gitlab-ci.yml

@@ -5,7 +5,6 @@ stages:
   - host_test
   - target_test
   - test_deploy
-  - post_check
   - deploy
   - post_deploy
 
@@ -50,7 +49,7 @@ variables:
   BOT_DOCKER_IMAGE_TAG: ":latest"
 
   # target test config file, used by assign test job
-  CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
+  CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
 
   # target test repo parameters
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
@@ -71,38 +70,27 @@ variables:
 before_script:
   - source tools/ci/utils.sh
   - source tools/ci/setup_python.sh
-  - apply_bot_filter
   - add_gitlab_ssh_keys
   - source tools/ci/configure_ci_environment.sh
   - *setup_tools_unless_target_test
   - fetch_submodules
 
 # used for check scripts which we want to run unconditionally
-.before_script_lesser_nofilter:
+.before_script_no_sync_submodule:
-  before_script:
-    - echo "Not setting up GitLab key, not fetching submodules, not applying bot filter"
-    - source tools/ci/utils.sh
-    - source tools/ci/setup_python.sh
-    - source tools/ci/configure_ci_environment.sh
-
-# used for everything else where we want to do no prep, except for bot filter
-.before_script_lesser:
   before_script:
     - echo "Not setting up GitLab key, not fetching submodules"
     - source tools/ci/utils.sh
     - source tools/ci/setup_python.sh
-    - apply_bot_filter
     - source tools/ci/configure_ci_environment.sh
 
-.before_script_slim:
+.before_script_minimal:
   before_script:
-    - echo "Only load utils.sh inside"
+    - echo "Only load utils.sh"
     - source tools/ci/utils.sh
 
 .before_script_macos:
   before_script:
     - source tools/ci/utils.sh
-    - apply_bot_filter
     - $IDF_PATH/tools/idf_tools.py install-python-env
     # On macOS, these tools need to be installed
     - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
@@ -116,13 +104,12 @@ before_script:
     - fetch_submodules
 
 include:
-  - '/tools/ci/config/rules.yml'
+  - '.gitlab/ci/rules.yml'
-  - '/tools/ci/config/docs.yml'
+  - '.gitlab/ci/docs.yml'
-  - '/tools/ci/config/static-code-analysis.yml'
+  - '.gitlab/ci/static-code-analysis.yml'
-  - '/tools/ci/config/pre_check.yml'
+  - '.gitlab/ci/pre_check.yml'
-  - '/tools/ci/config/build.yml'
+  - '.gitlab/ci/build.yml'
-  - '/tools/ci/config/assign-test.yml'
+  - '.gitlab/ci/assign-test.yml'
-  - '/tools/ci/config/host-test.yml'
+  - '.gitlab/ci/host-test.yml'
-  - '/tools/ci/config/target-test.yml'
+  - '.gitlab/ci/target-test.yml'
-  - '/tools/ci/config/post_check.yml'
+  - '.gitlab/ci/deploy.yml'
-  - '/tools/ci/config/deploy.yml'

.gitlab/ci/assign-test.yml

@@ -1,6 +1,5 @@
 assign_test:
-  extends:
+  extends: .rules:test:any_test
-    - .rules:assign_test:target_test-integration_test-weekend_test
   tags:
     - assign_test
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG

.gitlab/ci/build.yml

@@ -8,9 +8,7 @@
   dependencies: []
 
 .build_template_app_template:
-  extends:
+  extends: .build_template
-    - .build_template
-    - .rules:labels:build
   variables:
     LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
     BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
@@ -46,7 +44,7 @@
 fast_template_app:
   extends:
     - .build_template_app_template
-    - .rules:build_tests:target_test-weekend_test
+    - .rules:test:target_test
   stage: pre_check
   variables:
     BUILD_COMMAND_ARGS: "-p"
@@ -55,7 +53,7 @@ fast_template_app:
 .build_ssc_template:
   extends:
     - .build_template
-    - .rules:build_tests:integration_test
+    - .rules:build:integration_test
   artifacts:
     paths:
       - SSC/ssc_bin
@@ -84,10 +82,8 @@ build_ssc_esp32c3:
   variables:
     TARGET_NAME: "ESP32C3"
 
-.build_esp_idf_tests_cmake:
+.build_esp_idf_tests_cmake_template:
-  extends:
+  extends: .build_template
-    - .build_template
-    - .rules:build_tests:unit_test
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
@@ -117,29 +113,35 @@ build_ssc_esp32c3:
     - python tools/UnitTestParser.py ${BUILD_PATH}
 
 build_esp_idf_tests_cmake_esp32:
-  extends: .build_esp_idf_tests_cmake
+  extends:
+    - .build_esp_idf_tests_cmake_template
+    - .rules:build:unit_test-esp32
   variables:
     IDF_TARGET: esp32
 
 build_esp_idf_tests_cmake_esp32s2:
-  extends: .build_esp_idf_tests_cmake
+  extends:
+    - .build_esp_idf_tests_cmake_template
+    - .rules:build:unit_test-esp32s2
   variables:
     IDF_TARGET: esp32s2
 
 build_esp_idf_tests_cmake_esp32s3:
-  extends: .build_esp_idf_tests_cmake
+  extends:
+    - .build_esp_idf_tests_cmake_template
+    - .rules:build:unit_test-esp32s3
   variables:
     IDF_TARGET: esp32s3
 
 build_esp_idf_tests_cmake_esp32c3:
-  extends: .build_esp_idf_tests_cmake
+  extends:
+    - .build_esp_idf_tests_cmake_template
+    - .rules:build:unit_test-esp32c3
   variables:
     IDF_TARGET: esp32c3
 
 .build_examples_template:
-  extends:
+  extends: .build_template
-    - .build_template
-    - .rules:build_tests:example_test-weekend_test
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
@@ -162,6 +164,7 @@ build_esp_idf_tests_cmake_esp32c3:
 build_examples_make:
   extends:
     - .build_examples_template
+    - .rules:build:example_test-esp32
   # This is a workaround for a rarely encountered issue with building examples in CI.
   # Probably related to building of Kconfig in 'make clean' stage
   retry: 1
@@ -178,7 +181,7 @@ build_examples_make:
     IDF_TARGET: esp32  # currently we only support esp32
 
 # same as above, but for CMake
-.build_examples_cmake:
+.build_examples_cmake_template:
   extends: .build_examples_template
   artifacts:
     paths:
@@ -200,27 +203,31 @@ build_examples_make:
     BUILD_SYSTEM: cmake
 
 build_examples_cmake_esp32:
-  extends: .build_examples_cmake
+  extends:
+    - .build_examples_cmake_template
+    - .rules:build:example_test-esp32
   parallel: 10
   variables:
     IDF_TARGET: esp32
 
 build_examples_cmake_esp32s2:
-  extends: .build_examples_cmake
+  extends:
+    - .build_examples_cmake_template
+    - .rules:build:example_test-esp32s2
   parallel: 8
   variables:
     IDF_TARGET: esp32s2
 
 build_examples_cmake_esp32c3:
-  extends: .build_examples_cmake
+  extends:
+    - .build_examples_cmake_template
+    - .rules:build:example_test-esp32c3
   parallel: 8
   variables:
     IDF_TARGET: esp32c3
 
-.build_test_apps:
+.build_test_apps_template:
-  extends:
+  extends: .build_examples_cmake_template
-    - .build_examples_cmake
-    - .rules:build_tests:custom_test-weekend_test
   variables:
     TEST_PREFIX: test_apps
     TEST_RELATIVE_DIR: tools/test_apps
@@ -229,57 +236,68 @@ build_examples_cmake_esp32c3:
     - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
 
 build_test_apps_esp32:
-  extends: .build_test_apps
+  extends:
+    - .build_test_apps_template
+    - .rules:build:custom_test-esp32
   parallel: 8
   variables:
     IDF_TARGET: esp32
 
 build_test_apps_esp32s2:
-  extends: .build_test_apps
+  extends:
+    - .build_test_apps_template
+    - .rules:build:custom_test-esp32s2
   parallel: 8
   variables:
     IDF_TARGET: esp32s2
 
 build_test_apps_esp32s3:
-  extends: .build_test_apps
+  extends:
+    - .build_test_apps_template
+    - .rules:build:custom_test-esp32s3
   parallel: 8
   variables:
     IDF_TARGET: esp32s3
 
 build_test_apps_esp32c3:
-  extends: .build_test_apps
+  extends:
+    - .build_test_apps_template
+    - .rules:build:custom_test-esp32c3
   parallel: 8
   variables:
     IDF_TARGET: esp32c3
 
-.build_component_ut:
+.build_component_ut_template:
-  extends:
+  extends: .build_test_apps_template
-    - .build_test_apps
-    - .rules:build_tests:unit_test
   variables:
     TEST_PREFIX: component_ut
     TEST_RELATIVE_DIR: component_ut
 
 build_component_ut_esp32:
-  extends: .build_component_ut
+  extends:
+    - .build_component_ut_template
+    - .rules:build:component_ut-esp32
   variables:
     IDF_TARGET: esp32
 
 build_component_ut_esp32s2:
-  extends: .build_component_ut
+  extends:
+    - .build_component_ut_template
+    - .rules:build:component_ut-esp32s2
   variables:
     IDF_TARGET: esp32s2
 
 build_component_ut_esp32c3:
-  extends: .build_component_ut
+  extends:
+    - .build_component_ut_template
+    - .rules:build:component_ut-esp32c3
   variables:
     IDF_TARGET: esp32c3
 
 .test_build_system_template:
   extends:
     - .build_template
-    - .rules:build_tests:weekend_test
+    - .rules:build
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
@@ -304,7 +322,7 @@ test_build_system_cmake_macos:
   extends:
     - .test_build_system_template
     - .before_script_macos
-    - .rules:os:mac_os
+    - .rules:build:macos
   tags:
     - macos_shell
   variables:
@@ -312,8 +330,8 @@ test_build_system_cmake_macos:
 
 build_docker:
   extends:
-    - .before_script_slim
+    - .before_script_minimal
-    - .rules:protected-schedule
+    - .rules:build:docker
   stage: build
   image: espressif/docker-builder:1
   tags:
@@ -334,8 +352,8 @@ build_docker:
 
 .test-on-windows:
   extends:
-    - .before_script_slim
+    - .before_script_minimal
-    - .rules:protected-schedule
+    - .rules:build:windows
   stage: build
   image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
   tags:
@@ -366,14 +384,10 @@ build_cmdlinerunner:
     TEST_DIR: tools/windows/tool_setup/cmdlinerunner
 
 build_installer:
-  extends:
+  extends: .test-on-windows
-    - .before_script_slim
-    - .rules:protected-schedule
   # using a different stage here to be able to use artifacts from build_cmdlinerunner job
   stage: host_test
   image: $CI_DOCKER_REGISTRY/wine-innosetup:1
-  tags:
-    - build
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - build_cmdlinerunner
@@ -383,7 +397,9 @@ build_installer:
 
 # This job builds template app with permutations of targets and optimization levels
 build_template_app:
+  extends:
+    - .build_template_app_template
+    - .rules:build
   needs:
     - job: fast_template_app
       artifacts: false
-  extends: .build_template_app_template

.gitlab/ci/dependencies/dependencies.yml

@@ -0,0 +1,142 @@
+.all_targets: &all_targets
+  - esp32
+  - esp32s2
+  - esp32s3
+  - esp32c3
+
+.target_test: &target_test
+  - example_test
+  - custom_test
+  - unit_test
+  - component_ut
+
+"build:{0}-{1}":
+  matrix:
+    - *target_test
+    - *all_targets
+  labels:
+    - build
+  patterns:
+    - build
+
+"build:example_test-esp32":
+  labels:
+    - build
+    - weekend_test  # only have esp32 jobs
+    - iperf_stress_test  # only have esp32 jobs
+  patterns:
+    - build
+    - example_test
+
+"build:{0}":
+  matrix:
+    - [windows, docker]
+  labels:
+    - build
+    - "{0}"
+  patterns:
+    - build
+    - "{0}"
+
+"build:macos":
+  labels:
+    - build
+    - macos
+    - macos_test  # for backward compatibility
+  patterns:
+    - build
+    - macos
+
+"build:docs":
+  labels:
+    - build
+    - docs
+    - build_docs  # for backward compatibility
+  patterns:
+    - docs
+  deploy:
+    - preview
+    - production
+
+"build":
+  labels:
+    - build
+  patterns:
+    - build
+
+"test:{0}-{1}":
+  matrix:
+    - *target_test
+    - *all_targets
+  labels:
+    - "{0}"
+    - "{0}_{1}"
+  patterns:
+    - "{0}"
+    - build
+    - "build-{0}"
+  included_in:
+    - "build:{0}-{1}"
+    - test:target_test
+    - test:any_test
+
+"test:component_ut-{0}":
+  matrix:
+    - *all_targets
+  labels:
+    - component_ut
+    - "component_ut_{0}"
+    - unit_test
+    - "unit_test_{0}"
+  patterns:
+    - component_ut
+    - build
+    - "build-component_ut-{0}"
+  included_in:
+    - "build:component_ut-{0}"
+    - test:target_test
+    - test:any_test
+
+# due to the lack of runners, c3 tests will only be triggered by label
+"test:unit_test-esp32c3":
+  labels:
+    - unit_test_esp32c3
+  patterns:
+    - unit_test
+    - build
+    - "build-unit_test-esp32c3"
+  included_in:
+    - "build:unit_test-esp32c3"
+
+"test:integration_test":
+  labels:
+    - "integration_test"
+  patterns:
+    - "integration_test"
+  included_in:
+    - "build:integration_test"
+    - test:target_test
+    - test:any_test
+
+"test:host_test":
+  labels:
+    - host_test
+  patterns:
+    - host_test
+  included_in:
+    - test:any_test
+
+"labels:{0}":
+  matrix:
+    - [weekend_test, iperf_stress_test, nvs_coverage]
+  labels:
+    - "{0}"
+  included_in:
+    - test:any_test
+
+"labels:fuzzer_test-weekend_test":
+  labels:
+    - fuzzer_test
+    - weekend_test
+  included_in:
+    - test:any_test

.gitlab/ci/dependencies/generate_rules.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python
+#
+# Copyright 2021 Espressif Systems (Shanghai) CO LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import inspect
+import os
+import sys
+from collections import defaultdict
+from itertools import product
+
+try:
+    import pygraphviz as pgv
+except ImportError:  # used when pre-commit, skip generating image
+    pass
+
+import yaml
+
+IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..', '..')))
+
+
+def _list(str_or_list):
+    if isinstance(str_or_list, str):
+        return [str_or_list]
+    elif isinstance(str_or_list, list):
+        return str_or_list
+    else:
+        raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list)))
+
+
+def _format_nested_dict(_dict, f_tuple):
+    res = {}
+    for k, v in _dict.items():
+        k = k.split('__')[0]
+        if isinstance(v, dict):
+            v = _format_nested_dict(v, f_tuple)
+        elif isinstance(v, list):
+            v = _format_nested_list(v, f_tuple)
+        elif isinstance(v, str):
+            v = v.format(*f_tuple)
+        res[k.format(*f_tuple)] = v
+    return res
+
+
+def _format_nested_list(_list, f_tuple):
+    res = []
+    for item in _list:
+        if isinstance(item, list):
+            item = _format_nested_list(item, f_tuple)
+        elif isinstance(item, dict):
+            item = _format_nested_dict(item, f_tuple)
+        elif isinstance(item, str):
+            item = item.format(*f_tuple)
+        res.append(item)
+    return res
+
+
+class RulesWriter:
+    AUTO_GENERATE_MARKER = inspect.cleandoc(r'''
+    ##################
+    # Auto Generated #
+    ##################
+    ''')
+
+    LABEL_TEMPLATE = inspect.cleandoc(r'''
+    .if-label-{0}: &if-label-{0}
+      if: '$BOT_LABEL_{1}'
+    ''')
+    TITLE_TEMPLATE = inspect.cleandoc(r'''
+    .if-title-{0}: &if-title-{0}
+      if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:\w+,)*{0}(?:,\w+)*$/i || $CI_COMMIT_DESCRIPTION =~ /test labels?: (?:\w+[, ]+)*{0}(?:[, ]+\w+)*/i'
+    ''')
+
+    RULE_NORM = '    - <<: *if-protected'
+    RULE_PROD = '    - <<: *if-protected-no_label'
+    RULE_LABEL_TEMPLATE = '    - <<: *if-label-{0}'
+    RULE_TITLE_TEMPLATE = '    - <<: *if-title-{0}'
+    RULE_PATTERN_TEMPLATE = '    - <<: *if-dev-push\n' \
+                            '      changes: *patterns-{0}'
+    RULES_TEMPLATE = inspect.cleandoc(r"""
+    .rules:{0}:
+      rules:
+    {1}
+    """)
+
+    KEYWORDS = ['labels', 'patterns']
+
+    def __init__(self, rules_yml, depend_yml):  # type: (str, str) -> None
+        self.rules_yml = rules_yml
+        self.rules_cfg = yaml.load(open(rules_yml), Loader=yaml.FullLoader)
+
+        self.full_cfg = yaml.load(open(depend_yml), Loader=yaml.FullLoader)
+        self.cfg = {k: v for k, v in self.full_cfg.items() if not k.startswith('.')}
+        self.cfg = self.expand_matrices()
+        self.rules = self.expand_rules()
+
+        self.graph = None
+
+    def expand_matrices(self):  # type: () -> dict
+        """
+        Expand the matrix into different rules
+        """
+        res = {}
+        for k, v in self.cfg.items():
+            res.update(self._expand_matrix(k, v))
+
+        for k, v in self.cfg.items():
+            deploy = v.get('deploy')
+            if deploy:
+                for item in _list(deploy):
+                    res['{}-{}'.format(k, item)] = v
+        return res
+
+    @staticmethod
+    def _expand_matrix(name, cfg):  # type: (str, dict) -> dict
+        """
+        Expand matrix into multi keys
+        :param cfg: single rule dict
+        :return:
+        """
+        default = {name: cfg}
+        if not cfg:
+            return default
+        matrices = cfg.pop('matrix', None)
+        if not matrices:
+            return default
+
+        res = {}
+        for comb in product(*_list(matrices)):
+            res.update(_format_nested_dict(default, comb))
+        return res
+
+    def expand_rules(self):  # type: () -> dict[str, dict[str, list]]
+        res = defaultdict(lambda: defaultdict(set))  # type: dict[str, dict[str, set]]
+        for k, v in self.cfg.items():
+            for vk, vv in v.items():
+                if vk in self.KEYWORDS:
+                    res[k][vk] = set(_list(vv))
+                else:
+                    res[k][vk] = vv
+            for key in self.KEYWORDS:  # provide empty set for missing field
+                if key not in res[k]:
+                    res[k][key] = set()
+
+        for k, v in self.cfg.items():
+            if not v:
+                continue
+            if 'included_in' in v:
+                for item in _list(v['included_in']):
+                    if 'labels' in v:
+                        res[item]['labels'].update(_list(v['labels']))
+                    if 'patterns' in v:
+                        for _pat in _list(v['patterns']):
+                            # Patterns must be pre-defined
+                            if '.patterns-{}'.format(_pat) not in self.rules_cfg:
+                                print('WARNING: pattern {} not exists'.format(_pat))
+                                continue
+                            res[item]['patterns'].add(_pat)
+
+        sorted_res = defaultdict(lambda: defaultdict(list))  # type: dict[str, dict[str, list]]
+        for k, v in res.items():
+            for vk, vv in v.items():
+                sorted_res[k][vk] = sorted(vv)
+        return sorted_res
+
+    def new_labels_titles_str(self):  # type: () -> str
+        _labels = set([])
+        for k, v in self.cfg.items():
+            if not v:
+                continue  # shouldn't be possible
+            labels = v.get('labels')
+            if not labels:
+                continue
+            _labels.update(_list(labels))
+        labels = sorted(_labels)
+
+        res = ''
+        res += '\n\n'.join([self._format_label(_label) for _label in labels])
+        res += '\n\n'
+        res += '\n\n'.join([self._format_title(_label) for _label in labels])
+        return res
+
+    @classmethod
+    def _format_label(cls, label):  # type: (str) -> str
+        return cls.LABEL_TEMPLATE.format(label, cls.bot_label_str(label))
+
+    @staticmethod
+    def bot_label_str(label):  # type: (str) -> str
+        return label.upper().replace('-', '_')
+
+    @classmethod
+    def _format_title(cls, title):  # type: (str) -> str
+        return cls.TITLE_TEMPLATE.format(title)
+
+    def new_rules_str(self):  # type: () -> str
+        res = []
+        for k, v in sorted(self.rules.items()):
+            res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
+        return '\n\n'.join(res)
+
+    def _format_rule(self, name, cfg):  # type: (str, dict) -> str
+        _rules = []
+        if name.endswith('-production'):
+            _rules.append(self.RULE_PROD)
+        else:
+            if not name.endswith('-preview'):
+                _rules.append(self.RULE_NORM)
+            for label in cfg['labels']:
+                _rules.append(self.RULE_LABEL_TEMPLATE.format(label))
+                _rules.append(self.RULE_TITLE_TEMPLATE.format(label))
+            for pattern in cfg['patterns']:
+                if '.patterns-{}'.format(pattern) in self.rules_cfg:
+                    _rules.append(self.RULE_PATTERN_TEMPLATE.format(pattern))
+                else:
+                    print('WARNING: pattern {} not exists'.format(pattern))
+        return '\n'.join(_rules)
+
+    def update_rules_yml(self):  # type: () -> bool
+        with open(self.rules_yml) as fr:
+            file_str = fr.read()
+
+        auto_generate_str = '\n{}\n\n{}\n'.format(self.new_labels_titles_str(), self.new_rules_str())
+        rest, marker, old = file_str.partition(self.AUTO_GENERATE_MARKER)
+        if old == auto_generate_str:
+            return False
+        else:
+            print(self.rules_yml, 'has been modified. Please check')
+            with open(self.rules_yml, 'w') as fw:
+                fw.write(rest + marker + auto_generate_str)
+            return True
+
+
+LABEL_COLOR = 'green'
+PATTERN_COLOR = 'cyan'
+RULE_COLOR = 'blue'
+
+
+def build_graph(rules_dict):  # type: (dict[str, dict[str, list]]) -> pgv.AGraph
+    graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True)
+
+    for k, v in rules_dict.items():
+        if not v:
+            continue
+        included_in = v.get('included_in')
+        if included_in:
+            for item in _list(included_in):
+                graph.add_node(k, color=RULE_COLOR)
+                graph.add_node(item, color=RULE_COLOR)
+                graph.add_edge(k, item, color=RULE_COLOR)
+        labels = v.get('labels')
+        if labels:
+            for _label in labels:
+                graph.add_node('label:{}'.format(_label), color=LABEL_COLOR)
+                graph.add_edge('label:{}'.format(_label), k, color=LABEL_COLOR)
+        patterns = v.get('patterns')
+        if patterns:
+            for _pat in patterns:
+                graph.add_node('pattern:{}'.format(_pat), color=PATTERN_COLOR)
+                graph.add_edge('pattern:{}'.format(_pat), k, color=PATTERN_COLOR)
+
+    return graph
+
+
+def output_graph(graph, output_path='output.png'):  # type: (pgv.AGraph, str) -> None
+    graph.layout('dot')
+    if output_path.endswith('.png'):
+        img_path = output_path
+    else:
+        img_path = os.path.join(output_path, 'output.png')
+    graph.draw(img_path)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'),
+                        help='rules.yml file path')
+    parser.add_argument('dependencies_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies',
+                                                                            'dependencies.yml'),
+                        help='dependencies.yml file path')
+    parser.add_argument('--graph',
+                        help='Specify PNG image output path. Use this argument to generate dependency graph')
+    args = parser.parse_args()
+
+    writer = RulesWriter(args.rules_yml, args.dependencies_yml)
+    file_modified = writer.update_rules_yml()
+
+    if args.graph:
+        dep_tree_graph = build_graph(writer.rules)
+        output_graph(dep_tree_graph)
+
+    sys.exit(file_modified)

.gitlab/ci/deploy.yml

@@ -1,15 +1,15 @@
 .deploy_job_template:
+  extends: .before_script_no_sync_submodule
   stage: deploy
   image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - deploy
+  dependencies: []
 
 push_to_github:
   extends:
     - .deploy_job_template
-    - .before_script_lesser
     - .rules:protected-no_label
-  dependencies: []
   script:
     - add_github_ssh_keys
     - git remote remove github &>/dev/null || true
@@ -19,7 +19,6 @@ push_to_github:
 deploy_test_result:
   extends:
     - .deploy_job_template
-    - .before_script_slim
     - .rules:ref:master-schedule-always
   image: $CI_DOCKER_REGISTRY/bot-env
   tags:
@@ -51,3 +50,24 @@ deploy_test_result:
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
     # update test results
     - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS ${CI_PROJECT_DIR}/${CI_COMMIT_SHA} --pipeline_url ${CI_PIPELINE_URL}
+
+check_submodule_sync:
+  extends:
+    - .deploy_job_template
+    - .rules:protected
+  tags:
+    - github_sync
+  retry: 2
+  variables:
+    GIT_STRATEGY: clone
+    SUBMODULES_TO_FETCH: "none"
+    PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
+  script:
+    - git submodule deinit --force .
+    # setting the default remote URL to the public one, to resolve relative location URLs
+    - git config remote.origin.url ${PUBLIC_IDF_URL}
+    # check if all submodules are correctly synced to public repository
+    - git submodule init
+    - git config --get-regexp '^submodule\..*\.url$' || true
+    - git submodule update --recursive
+    - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"

.gitlab/ci/docs.yml

@@ -1,7 +1,26 @@
+# stage: pre_check
+check_readme_links:
+  extends:
+    - .pre_check_job_template
+    - .rules:build:docs
+  tags: ["build", "amd64", "internet"]
+  allow_failure: true
+  script:
+    - python ${IDF_PATH}/tools/ci/check_readme_links.py
+
+check_docs_lang_sync:
+  extends:
+    - .pre_check_job_template
+    - .rules:build:docs
+  script:
+    - cd docs
+    - ./check_lang_folder_sync.sh
+
 .build_docs_template:
   image: $ESP_IDF_DOC_ENV_IMAGE
   tags:
     - build_docs
+  dependencies: []
   script:
     - cd docs
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
@@ -9,37 +28,13 @@
   parallel:
     matrix:
       - DOCLANG: ["en", "zh_CN"]
-        DOCTGT: ["esp32", "esp32s2", "esp32c3"]
+        DOCTGT: ["esp32", "esp32s2"]
-
-# stage: pre_check
-check_readme_links:
-  extends:
-    - .pre_check_job_template
-    - .rules:patterns:docs
-  tags: ["internet"]
-  allow_failure: true
-  variables:
-    PYTHON_VER: 3
-  script:
-    - python ${IDF_PATH}/tools/ci/check_readme_links.py
-
-check_docs_lang_sync:
-  extends:
-    - .pre_check_job_template
-    - .rules:patterns:docs
-  variables:
-    SUBMODULES_TO_FETCH: "none"
-  script:
-    - cd docs
-    - ./check_lang_folder_sync.sh
 
 check_docs_gh_links:
   extends:
+    - .pre_check_job_template
     - .build_docs_template
-    - .rules:patterns:docs
+    - .rules:build:docs
-  stage: pre_check
-  variables:
-    SUBMODULES_TO_FETCH: "none"
   script:
     - cd docs
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
@@ -49,11 +44,13 @@ check_docs_gh_links:
 .build_docs_build_stage_template:
   extends:
     - .build_docs_template
-    - .rules:patterns:docs
+    - .rules:build:docs
   stage: build
   needs:
-    - check_docs_lang_sync
+    - job: check_docs_lang_sync
-    - check_docs_gh_links
+      artifacts: false
+    - job: check_docs_gh_links
+      artifacts: false
 
 build_docs_html:
   extends:
@@ -80,15 +77,16 @@ build_docs_pdf:
 
 .deploy_docs_template:
   extends:
-    - .before_script_lesser
+    - .before_script_no_sync_submodule
-    - .rules:patterns:docs
   image: $ESP_IDF_DOC_ENV_IMAGE
+  stage: test_deploy
   tags:
     - deploy
     - shiny
   variables:
     DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
     PYTHONUNBUFFERED: 1
+  dependencies: []
   script:
     - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
     - export GIT_VER=$(git describe --always)
@@ -98,8 +96,8 @@ build_docs_pdf:
 deploy_docs_preview:
   extends:
     - .deploy_docs_template
-    - .rules:patterns:docs-preview
+    - .rules:build:docs-preview
-  stage: test_deploy
+  dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - build_docs_html
     - build_docs_pdf
@@ -117,12 +115,14 @@ deploy_docs_production:
   # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
   extends:
     - .deploy_docs_template
-    - .rules:protected-no_label
+    - .rules:build:docs-production
   stage: post_deploy
+  dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs: # ensure runs after push_to_github succeeded
     - build_docs_html
     - build_docs_pdf
-    - push_to_github
+    - job: push_to_github
+      artifacts: false
   variables:
     TYPE: "preview"
     DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
@@ -144,7 +144,6 @@ check_doc_links:
       - docs/_build/*/*/linkcheck/*.txt
     expire_in: 1 week
   allow_failure: true
-  dependencies: []
   script:
     - cd docs
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt

.gitlab/ci/host-test.yml

@@ -1,30 +1,11 @@
 .host_test_template:
-  extends: .rules:labels:host_test
+  extends: .rules:test:host_test
   stage: host_test
   image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - host_test
   dependencies: []
-
+  needs: []  # run host_test jobs immediately
-.host_fuzzer_test_template:
-  extends:
-    - .host_test_template
-    - .rules:labels:fuzzer_test-weekend_test-only
-  image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
-  artifacts:
-    when: always
-    paths:
-      - ${FUZZER_TEST_DIR}/out/crashes
-      - ${FUZZER_TEST_DIR}/fuzz_output.log
-    expire_in: 1 week
-  script:
-    - export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
-    - cd ${FUZZER_TEST_DIR}
-    # run AFL fuzzer for one hour
-    - ( ( make ${FUZZER_PARAMS} fuzz | tee fuzz_output.log | grep -v '\(Fuzzing test case\|Entering queue cycle\)' ) || pkill sleep ) &
-    - ( sleep 3600 || mkdir -p out/crashes/env_failed ) && pkill afl-fuz
-    # check no crashes found
-    - test -z "$(ls out/crashes/)" || exit 1
 
 test_nvs_on_host:
   extends: .host_test_template
@@ -35,7 +16,7 @@ test_nvs_on_host:
 test_nvs_coverage:
   extends:
     - .host_test_template
-    - .rules:labels:nvs_coverage-only
+    - .rules:labels:nvs_coverage
   artifacts:
     paths:
       - components/nvs_flash/test_nvs_host/coverage_report
@@ -79,6 +60,26 @@ test_ldgen_on_host:
   variables:
     LC_ALL: C.UTF-8
 
+.host_fuzzer_test_template:
+  extends:
+    - .host_test_template
+    - .rules:labels:fuzzer_test-weekend_test
+  image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
+  artifacts:
+    when: always
+    paths:
+      - ${FUZZER_TEST_DIR}/out/crashes
+      - ${FUZZER_TEST_DIR}/fuzz_output.log
+    expire_in: 1 week
+  script:
+    - export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
+    - cd ${FUZZER_TEST_DIR}
+    # run AFL fuzzer for one hour
+    - ( ( make ${FUZZER_PARAMS} fuzz | tee fuzz_output.log | grep -v '\(Fuzzing test case\|Entering queue cycle\)' ) || pkill sleep ) &
+    - ( sleep 3600 || mkdir -p out/crashes/env_failed ) && pkill afl-fuz
+    # check no crashes found
+    - test -z "$(ls out/crashes/)" || exit 1
+
 test_mdns_fuzzer_on_host:
   extends: .host_fuzzer_test_template
   variables:

.gitlab/ci/post_check.yml

@@ -1,73 +0,0 @@
-# copy from .gitlab-ci.yml as anchor is not global
-.show_submodule_urls: &show_submodule_urls |
-  git config --get-regexp '^submodule\..*\.url$' || true
-
-.post_check_base_template:
-  stage: post_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - host_test
-  dependencies: []
-
-.post_check_job_template:
-  extends:
-    - .post_check_base_template
-    - .before_script_lesser_nofilter
-
-.post_check_job_template_with_filter:
-  extends:
-    - .post_check_base_template
-    - .before_script_lesser
-
-check_submodule_sync:
-  extends:
-    - .before_script_slim
-    - .post_check_job_template
-  tags:
-    - github_sync
-  retry: 2
-  variables:
-    GIT_STRATEGY: clone
-    SUBMODULES_TO_FETCH: "none"
-    PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
-  script:
-    - git submodule deinit --force .
-    # setting the default remote URL to the public one, to resolve relative location URLs
-    - git config remote.origin.url ${PUBLIC_IDF_URL}
-    # check if all submodules are correctly synced to public repository
-    - git submodule init
-    - *show_submodule_urls
-    - git submodule update --recursive
-    - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
-
-check_ut_cmake_make:
-  extends:
-    - .post_check_job_template_with_filter
-    - .rules:dev
-  tags:
-    - build
-  script:
-    - tools/ci/check_ut_cmake_make.sh
-
-check_artifacts_expire_time:
-  extends: .post_check_job_template
-  script:
-    # check if we have set expire time for all artifacts
-    - python tools/ci/check_artifacts_expire_time.py
-
-check_pipeline_triggered_by_label:
-  extends:
-    - .post_check_job_template
-    - .rules:dev
-  script:
-    # If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added.
-    # We want to make sure some jobs are always executed to detect regression.
-    - test "$BOT_LABEL_REGULAR_TEST" = "true" || { echo "CI can only pass if 'regular_test' label is included"; exit -1; }
-
-check_commit_msg:
-  extends: .post_check_job_template
-  script:
-    - git status
-    - git log -n10 --oneline
-    # commit start with "WIP: " need to be squashed before merge
-    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'

.gitlab/ci/pre_check.yml

@@ -8,16 +8,10 @@
 .pre_check_job_template:
   extends:
     - .pre_check_base_template
-    - .before_script_lesser_nofilter
+    - .before_script_no_sync_submodule
-
-.pre_check_job_template_with_filter:
-  extends:
-    - .pre_check_base_template
-    - .before_script_lesser
 
 .check_pre_commit_template:
   extends: .pre_check_job_template
-  stage: pre_check
   image: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
   before_script:
     - source tools/ci/utils.sh
@@ -48,14 +42,12 @@ check_version:
     - tools/ci/check_idf_version.sh
 
 check_examples_cmake_make:
-  extends:
+  extends: .pre_check_job_template
-    - .pre_check_job_template_with_filter
-    - .rules:dev
   script:
   - python ${IDF_PATH}/tools/ci/check_examples_cmake_make.py
 
 check_rom_api_header:
-  extends: .pre_check_job_template_with_filter
+  extends: .pre_check_job_template
   script:
     - tools/ci/check_examples_rom_header.sh
     - tools/ci/check_rom_apis.sh
@@ -72,8 +64,8 @@ check_python_style:
   script:
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
 
-check_kconfigs:
+test_check_kconfigs:
-  extends: .pre_check_job_template_with_filter
+  extends: .pre_check_job_template
   artifacts:
     when: on_failure
     paths:
@@ -86,7 +78,6 @@ check_kconfigs:
     expire_in: 1 week
   script:
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/ci/test_check_kconfigs.py
-    - ${IDF_PATH}/tools/ci/check_kconfigs.py
 
 check_wifi_lib_md5:
   extends: .pre_check_base_template
@@ -101,25 +92,19 @@ check_wifi_lib_md5:
 check_public_headers:
   extends:
     - .pre_check_base_template
-    - .rules:labels:build
+    - .rules:build
   tags:
     - build
   script:
     - python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
 
-.scan_build_tests:
+scan_tests:
-  stage: pre_check
+  extends:
+    - .pre_check_base_template
+    - .rules:test:target_test
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - assign_test
-  variables:
-    CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
-    TEST_CONFIG_FILE: ${CI_PROJECT_DIR}/tools/ci/config/target-test.yml
-
-scan_tests:
-  extends:
-    - .scan_build_tests
-    - .rules:build_tests:target_test-weekend_test
   artifacts:
     paths:
       - $EXAMPLE_TEST_OUTPUT_DIR
@@ -131,23 +116,44 @@ scan_tests:
     TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
     TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs
     COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
-    PYTHON_VER: 3
+    CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
   script:
     - set_component_ut_vars
-    - python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b make --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
+    - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b make --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
-    - python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
+    - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
-    - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
+    - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
-    - python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR
+    - run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR
 
 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'
 check_version_tag:
   extends:
     - .pre_check_job_template
-    - .rules:tag:release-no_label
+    - .rules:tag:release
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
 
+check_ut_cmake_make:
+  extends: .pre_check_job_template
+  tags:
+    - build
+  script:
+    - tools/ci/check_ut_cmake_make.sh
+
+check_artifacts_expire_time:
+  extends: .pre_check_job_template
+  script:
+    # check if we have set expire time for all artifacts
+    - python tools/ci/check_artifacts_expire_time.py
+
+check_commit_msg:
+  extends: .pre_check_job_template
+  script:
+    - git status
+    - git log -n10 --oneline
+    # commit start with "WIP: " need to be squashed before merge
+    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
+
 check_tools_file_patterns:
   extends: .pre_check_job_template
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG

.gitlab/ci/static-code-analysis.yml

@@ -1,4 +1,5 @@
-.clang_tidy_check_template:
+# pre_check stage
+clang_tidy_check:
   extends:
     - .pre_check_base_template
     - .rules:patterns:clang_tidy
@@ -16,24 +17,6 @@
     - export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
     - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml
 
-# pre_check stage
-clang_tidy_check:
-  extends: .clang_tidy_check_template
-  variables:
-    BOT_NEEDS_TRIGGER_BY_NAME: 1
-    TRIGGERED_RELATIVE: 1
-
-clang_tidy_check_regular:
-  extends:
-    - .clang_tidy_check_template
-    - .rules:patterns:clang_tidy-preview
-
-clang_tidy_check_all:
-  extends: .clang_tidy_check_template
-  variables:
-    BOT_NEEDS_TRIGGER_BY_NAME: 1
-    TRIGGERED_ABSOLUTE: 1
-
 # build stage
 # Sonarqube related jobs put here for this reason:
 # Here we have two jobs. code_quality_check and code_quality_report.
@@ -69,7 +52,7 @@ clang_tidy_check_all:
   tags:
     - host_test
   dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
-    - clang_tidy_check_regular
+    - clang_tidy_check
 
 code_quality_check:
   extends:
@@ -104,7 +87,7 @@ code_quality_check:
 code_quality_report:
   extends:
     - .sonar_scan_template
-    - .rules:protected-schedule
+    - .rules:protected
   script:
     - sonar-scanner
       -Dsonar.branch.name=$CI_COMMIT_REF_NAME
@@ -124,10 +107,12 @@ code_quality_report:
       -Dsonar.sources=$CI_PROJECT_DIR
 
 # deploy stage
-.clang_tidy_deploy_template:
+clang_tidy_deploy:
   extends:
     - .deploy_job_template
     - .rules:patterns:clang_tidy
+  needs:
+    - clang_tidy_check
   tags:
     - deploy
     - shiny
@@ -144,20 +129,3 @@ code_quality_report:
     # add link to view the report
     - echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html"
     - test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; }
-
-clang_tidy_deploy:
-  extends: .clang_tidy_deploy_template
-  # Override default stage to happen before the post_check
-  stage: test_deploy
-  needs:
-    - clang_tidy_check
-    - clang_tidy_check_all
-  variables:
-    BOT_NEEDS_TRIGGER_BY_NAME: 1
-
-clang_tidy_deploy_regular:
-  extends:
-    - .clang_tidy_deploy_template
-    - .rules:patterns:clang_tidy-preview
-  needs:
-    - clang_tidy_check_regular

tools/ci/apply_bot_filter.py

@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-
-# internal use only
-# called by CI jobs to determine if it need to be executed
-
-import json
-import os
-import re
-import sys
-
-RE_FILTER_PATTERN = re.compile(r'^r"(.+)?"$')
-
-RE_TYPE = type(re.compile('', 0))
-
-
-def parse_filter(filter_name):
-    filter_raw = os.getenv(filter_name)
-    filters = []
-    if filter_raw:
-        filter_data = json.loads(filter_raw)
-        for _filter in filter_data:
-            match = RE_FILTER_PATTERN.search(_filter)
-            if match:
-                filters.append(re.compile(match.group(1)))
-            else:
-                filters.append(_filter)
-    return filters
-
-
-def process_filter(execute_by_default, filter_name, ci_name):
-    execute = execute_by_default
-
-    # bot message is case insensitive (processed with lower case). so we also convert ci_name to lower case.
-    ci_name = ci_name.lower()
-
-    filter_list = parse_filter(filter_name)
-
-    for _filter in filter_list:
-        if isinstance(_filter, RE_TYPE):
-            match = _filter.search(ci_name) is not None
-        else:
-            match = _filter == ci_name
-
-        if match:
-            execute = True
-            break
-        else:
-            execute = False
-    return execute
-
-
-if __name__ == '__main__':
-    execute_by_default = True
-    if os.getenv('BOT_NEEDS_TRIGGER_BY_NAME', '0') == '1':
-        execute_by_default = False
-
-    need_to_execute = process_filter(True, 'BOT_STAGE_FILTER', os.getenv('CI_JOB_STAGE')) and process_filter(execute_by_default,
-                                                                                                             'BOT_JOB_FILTER', os.getenv('CI_JOB_NAME'))
-    if need_to_execute:
-        sys.exit(0)
-    else:
-        print("Skip this job as it doesn't fit @bot's filter")
-        sys.exit(-1)

tools/ci/check_tools_files_patterns.py

@@ -24,7 +24,6 @@ import yaml
 from idf_ci_utils import IDF_PATH, get_git_files, magic_check, magic_check_bytes, translate
 
 # Monkey patch starts
-
 # glob.glob will ignore all files starts with ``.``
 # don't ignore them here
 # need to keep the same argument as glob._ishidden
@@ -83,13 +82,13 @@ if __name__ == '__main__':
     not_included_files, dup_patterns = check(args.pattern_yml, args.exclude_list)
     if not_included_files:
         print('Missing Files: (please add to tools/ci/exclude_check_tools_files.txt')
-        for f in not_included_files:
+        for file in not_included_files:
-            print(f)
+            print(file)
         res = 1
     if dup_patterns:
         print('Duplicated Patterns: (please check .gitlab/ci/rules.yml and tools/ci/exclude_check_tools_files.txt')
-        for pat in dup_patterns:
+        for pattern in dup_patterns:
-            print(pat)
+            print(pattern)
         res = 1
 
     sys.exit(res)

tools/ci/executable-list.txt

@@ -1,3 +1,4 @@
+.gitlab/ci/dependencies/generate_rules.py
 components/app_update/otatool.py
 components/efuse/efuse_table_gen.py
 components/efuse/test_efuse_host/efuse_tests.py
@@ -33,7 +34,6 @@ install.fish
 install.sh
 tools/build_apps.py
 tools/check_python_dependencies.py
-tools/ci/apply_bot_filter.py
 tools/ci/build_template_app.sh
 tools/ci/check_build_warnings.py
 tools/ci/check_callgraph.py
@@ -46,6 +46,7 @@ tools/ci/check_idf_version.sh
 tools/ci/check_kconfigs.py
 tools/ci/check_readme_links.py
 tools/ci/check_rom_apis.sh
+tools/ci/check_rules_yml.py
 tools/ci/check_tools_files_patterns.py
 tools/ci/check_ut_cmake_make.sh
 tools/ci/checkout_project_ref.py

tools/ci/utils.sh

@@ -1,10 +1,5 @@
 # Modified from https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/utils.sh
 
-# before each job, we need to check if this job is filtered by bot stage/job filter
-function apply_bot_filter() {
-  python "${IDF_PATH}"/tools/ci/apply_bot_filter.py || exit 0
-}
-
 function add_ssh_keys() {
   local key_string="${1}"
   mkdir -p ~/.ssh