ci: apply idf-ci gitlab dynamic-pipeline-variables

Renamed env vars:

- TEST_CASE_FILTERS -> IDF_CI_SELECT_BY_FILTER_EXPR
- BUILD_AND_TEST_ALL_APPS -> IDF_CI_SELECT_ALL_PYTEST_CASES

Removed env vars;

- IS_MR_PIPELINE (could use CI_MERGE_REQUEST_IID)
- REPORT_EXIT_CODE -> IDF_CI_IS_DEBUG_PIPELINE (fail fast)
- PYTEST_IGNORE_COLLECT_IMPORT_ERROR -> always skip import error

.gitlab-ci.yml

@@ -8,11 +8,9 @@ workflow:
     - if: $CI_OPEN_MERGE_REQUESTS != null
       variables:
         PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
-        IS_MR_PIPELINE: 1
     - if: $CI_OPEN_MERGE_REQUESTS == null
       variables:
         PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA
-        IS_MR_PIPELINE: 0
     - when: always
 
 # Place the default settings in `.gitlab/ci/common.yml` instead

.gitlab/ci/build.yml

@@ -307,12 +307,9 @@ build_child_pipeline:
     - pipeline_variables
     - generate_build_child_pipeline
   variables:
-    IS_MR_PIPELINE: $IS_MR_PIPELINE
     MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS
     MR_MODIFIED_FILES: $MR_MODIFIED_FILES
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS
-    REPORT_EXIT_CODE: $REPORT_EXIT_CODE
   # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
   inherit:
     variables: false

.gitlab/ci/pre_check.yml

@@ -140,11 +140,10 @@ pipeline_variables:
     # MODIFIED_FILES is a list of files that changed, could be used everywhere
     - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
     - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
-    - echo "REPORT_EXIT_CODE=0" >> pipeline.env
     # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
     # for non MR pipeline, these are empty lists
     - |
-      if [ $IS_MR_PIPELINE == "0" ]; then
+      if [ -z "$CI_MERGE_REQUEST_IID" ]; then
         echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env
         echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env
       else
@@ -154,18 +153,9 @@ pipeline_variables:
         MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';')
         echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env
       fi
-    - |
-      if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)BUILD_AND_TEST_ALL_APPS(,|$)"; then
-        echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
-      fi
-    # run full pipeline if testing constraint branch
-    - |
-      if [ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]; then
-        echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
-      fi
     - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
     - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
-    - python tools/ci/ci_process_description.py
+    - idf-ci gitlab pipeline-variables >> pipeline.env
     - cat pipeline.env
     - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report
   artifacts:

tools/ci/ci_build_apps.py

@@ -3,6 +3,7 @@
 """
 This file is used in CI generate binary files for different kinds of apps
 """
+
 import argparse
 import os
 import sys
@@ -259,7 +260,7 @@ if __name__ == '__main__':
             os.environ[_k] = _v  # type: ignore
             print(f'env var {_k} set to "{_v}"')
 
-    if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1':
+    if not os.getenv('CI_MERGE_REQUEST_IID') or os.getenv('IDF_CI_SELECT_ALL_PYTEST_CASES') == '1':
         print('Build and run all test cases, and compile all cmake apps')
         arguments.modified_components = None
         arguments.modified_files = None

tools/ci/ci_process_description.py

@@ -1,28 +0,0 @@
-# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-import os
-import re
-import sys
-
-import yaml
-
-
-if __name__ == '__main__':
-    description = os.getenv('CI_MERGE_REQUEST_DESCRIPTION', '')
-    if not description:
-        sys.exit(0)
-    pattern = r'^## Dynamic Pipeline Configuration(?:[^`]*?)```(?:\w+)(.*?)```'
-    result = re.search(pattern, description, re.DOTALL | re.MULTILINE)
-    if not result:
-        sys.exit(0)
-
-    data = yaml.safe_load(result.group(1))
-    raise_report_exit_code = False
-    with open('pipeline.env', 'a+') as f:
-        if 'Test Case Filters' in data:
-            raise_report_exit_code = True
-            test_case_filters = ' or '.join(data.get('Test Case Filters'))
-            f.write(f'TEST_CASE_FILTERS={test_case_filters}\n')
-
-        if raise_report_exit_code:
-            f.write('REPORT_EXIT_CODE=30\n')

tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py

@@ -223,10 +223,10 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
 
-    if test_case_filters := os.getenv('TEST_CASE_FILTERS', None):
+    if test_case_filters := os.getenv('IDF_CI_SELECT_BY_FILTER_EXPR', None):
         args.filter_expr = test_case_filters
 
-    if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1':
+    if not os.getenv('CI_MERGE_REQUEST_IID') or os.getenv('IDF_CI_SELECT_ALL_PYTEST_CASES') == '1':
         print('Build and run all test cases, and compile all cmake apps')
         args.modified_components = None
         args.modified_files = None

tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py

@@ -109,8 +109,7 @@ def get_target_test_jobs(
     else:
         extra_include_yml = ['tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml']
 
-    fast_pipeline_flag = int(os.getenv('REPORT_EXIT_CODE', 0)) == 30
-    if fast_pipeline_flag:
+    if os.getenv('IDF_CI_IS_DEBUG_PIPELINE') == '1':
         extra_include_yml = ['tools/ci/dynamic_pipelines/templates/fast_pipeline.yml']
 
     no_env_marker_test_cases.sort()

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -20,6 +20,8 @@
   needs:
     - pipeline: $PARENT_PIPELINE_ID
       job: generate_build_child_pipeline
+    - pipeline: $PARENT_PIPELINE_ID
+      job: pipeline_variables
   artifacts:
     paths:
       # The other artifacts patterns are defined under tools/ci/artifacts_handler.py
@@ -49,6 +51,9 @@
   image: $TARGET_TEST_ENV_IMAGE
   stage: target_test
   timeout: 1 hour
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: pipeline_variables
   variables:
     SUBMODULES_TO_FETCH: "none"
     # set while generating the pipeline
@@ -70,10 +75,10 @@
     paths:
       - XUNIT_RESULT*.xml
       - pytest-embedded/
-# Child pipeline reports won't be collected in the main one
-# https://gitlab.com/groups/gitlab-org/-/epics/8205
-#   reports:
-#     junit: XUNIT_RESULT.xml
+    # Child pipeline reports won't be collected in the main one
+    # https://gitlab.com/groups/gitlab-org/-/epics/8205
+    #   reports:
+    #     junit: XUNIT_RESULT.xml
     when: always
     expire_in: 1 week
   script:

tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml

@@ -39,7 +39,6 @@ Pytest Target Test Jobs:
     - generate_pytest_child_pipeline
   variables:
     PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID
-    REPORT_EXIT_CODE: $REPORT_EXIT_CODE
   # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
   inherit:
     variables: false

tools/ci/exclude_check_tools_files.txt

@@ -14,7 +14,6 @@ tools/ci/check_test_files.py
 tools/ci/checkout_project_ref.py
 tools/ci/ci_fetch_submodule.py
 tools/ci/ci_get_mr_info.py
-tools/ci/ci_process_description.py
 tools/ci/cleanup_ignore_lists.py
 tools/ci/configure_ci_environment.sh
 tools/ci/deploy_docs.py

tools/requirements/requirements.ci.txt

@@ -6,12 +6,8 @@
 # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-tools.html
 
 # ci
+idf-ci==0.1.17
+
 coverage
-idf-build-apps
 jsonschema
-junit_xml
-python-gitlab
-pyyaml
-SimpleWebSocketServer
-minio
 prettytable