CI: support only download artifacts by request:

use CI dependencies could waste a lot bandwidth for target test jobs, as
example binary artifacts are very large. Now we will parse required
artifacts first, then use API to download required files in artifacts.

tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py

@@ -18,14 +18,23 @@ Command line tool to assign example tests to CI test jobs.
 
 # TODO: Need to handle running examples on different chips
 import os
-import sys
 import re
 import argparse
+import json
 
 import gitlab_api
 from tiny_test_fw.Utility import CIAssignTest
 
 
+EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
+IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
+if IDF_PATH_FROM_ENV:
+    ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV,
+                                       "build_examples", "artifact_index.json")
+else:
+    ARTIFACT_INDEX_FILE = "artifact_index.json"
+
+
 class ExampleGroup(CIAssignTest.Group):
     SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
 
@@ -34,15 +43,33 @@ class CIExampleAssignTest(CIAssignTest.AssignTest):
     CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
 
 
-class ArtifactFile(object):
-    def __init__(self, project_id, job_name, artifact_file_path):
-        self.gitlab_api = gitlab_api.Gitlab(project_id)
+def create_artifact_index_file(project_id=None, pipeline_id=None):
+    if project_id is None:
+        project_id = os.getenv("CI_PROJECT_ID")
+    if pipeline_id is None:
+        pipeline_id = os.getenv("CI_PIPELINE_ID")
+    gitlab_inst = gitlab_api.Gitlab(project_id)
+    artifact_index_list = []
 
-    def process(self):
+    def format_build_log_path():
+        return "build_examples/list_job_{}.json".format(job_info["parallel_num"])
+
+    for build_job_name in EXAMPLE_BUILD_JOB_NAMES:
+        job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
+        for job_info in job_info_list:
+            raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
+            build_info_list = [json.loads(line) for line in raw_data.splitlines()]
+            for build_info in build_info_list:
+                build_info["ci_job_id"] = job_info["id"]
+                artifact_index_list.append(build_info)
+    try:
+        os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE))
+    except OSError:
+        # already created
         pass
 
-    def output(self):
-        pass
+    with open(ARTIFACT_INDEX_FILE, "w") as f:
+        json.dump(artifact_index_list, f)
 
 
 if __name__ == '__main__':
@@ -53,8 +80,11 @@ if __name__ == '__main__':
                         help="gitlab ci config file")
     parser.add_argument("output_path",
                         help="output path of config files")
+    parser.add_argument("--pipeline_id", "-p", type=int, default=None,
+                        help="pipeline_id")
     args = parser.parse_args()
 
     assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
     assign_test.assign_cases()
     assign_test.output_configs(args.output_path)
+    create_artifact_index_file()