alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-27 02:16:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: improve rules for deploy jobs

Browse Source
This commit is contained in:
Fu Hanxi
2025-05-13 09:40:23 +02:00
parent 334cbe0dab
commit 9a9438b79a
7 changed files with 89 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
tools/ci/generate_rules.py
View File

@@ -1,30 +1,24 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import inspect
import os
import sys
import typing as t
from collections import defaultdict
from itertools import product
import yaml
from idf_ci_utils import GitlabYmlConfig
from idf_ci_utils import IDF_PATH
from idf_ci_utils import GitlabYmlConfig
try:
if t.TYPE_CHECKING:
import pygraphviz as pgv
except ImportError: # used when pre-commit, skip generating image
pass
try:
from typing import Union
except ImportError: # used for type hint
pass
def _list(str_or_list): # type: (Union[str, list]) -> list
def _list(str_or_list: t.Union[str, t.List]) -> t.List:
if isinstance(str_or_list, str):
return [str_or_list]
elif isinstance(str_or_list, list):
@@ -33,7 +27,7 @@ def _list(str_or_list): # type: (Union[str, list]) -> list
raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list)))
def _format_nested_dict(_dict, f_tuple): # type: (dict[str, dict], tuple[str, ...]) -> dict[str, dict]
def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) -> t.Dict[str, t.Dict]:
res = {}
for k, v in _dict.items():
k = k.split('__')[0]
@@ -47,7 +41,7 @@ def _format_nested_dict(_dict, f_tuple): # type: (dict[str, dict], tuple[str, .
return res
def _format_nested_list(_list, f_tuple): # type: (list[str], tuple[str, ...]) -> list[str]
def _format_nested_list(_list: t.List[str], f_tuple: t.Tuple[str, ...]) -> t.List[str]:
res = []
for item in _list:
if isinstance(item, list):
@@ -61,26 +55,23 @@ def _format_nested_list(_list, f_tuple): # type: (list[str], tuple[str, ...]) -
class RulesWriter:
AUTO_GENERATE_MARKER = inspect.cleandoc(r'''
AUTO_GENERATE_MARKER = inspect.cleandoc(r"""
##################
# Auto Generated #
##################
''')
""")
LABEL_TEMPLATE = inspect.cleandoc(r'''
LABEL_TEMPLATE = inspect.cleandoc(r"""
.if-label-{0}: &if-label-{0}
if: '$BOT_LABEL_{1} || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*{0}(?:,[^,\n\r]+)*$/i'
''')
""")
RULE_PROTECTED = ' - <<: *if-protected'
RULE_PROTECTED_NO_LABEL = ' - <<: *if-protected-no_label'
RULE_BUILD_ONLY = ' - <<: *if-label-build-only\n' \
' when: never'
RULE_REVERT_BRANCH = ' - <<: *if-revert-branch\n' \
' when: never'
RULE_PROTECTED_CHECK = ' - <<: *if-protected-check'
RULE_PROTECTED_PUSH = ' - <<: *if-protected-push'
RULE_BUILD_ONLY = ' - <<: *if-label-build-only\n when: never'
RULE_REVERT_BRANCH = ' - <<: *if-revert-branch\n when: never'
RULE_LABEL_TEMPLATE = ' - <<: *if-label-{0}'
RULE_PATTERN_TEMPLATE = ' - <<: *if-dev-push\n' \
' changes: *patterns-{0}'
RULE_PATTERN_TEMPLATE = ' - <<: *if-dev-push\n changes: *patterns-{0}'
SPECIFIC_RULE_TEMPLATE = ' - <<: *{0}'
RULES_TEMPLATE = inspect.cleandoc(r"""
.rules:{0}:
@@ -120,7 +111,7 @@ class RulesWriter:
return res
@staticmethod
def _expand_matrix(name, cfg): # type: (str, dict) -> dict
def _expand_matrix(name: str, cfg: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
"""
Expand matrix into multi keys
:param cfg: single rule dict
@@ -138,7 +129,7 @@ class RulesWriter:
res.update(_format_nested_dict(default, comb))
return res
def expand_rules(self): # type: () -> dict[str, dict[str, list]]
def expand_rules(self) -> t.Dict[str, t.Dict[str, t.List[str]]]:
res = defaultdict(lambda: defaultdict(set)) # type: dict[str, dict[str, set]]
for k, v in self.cfg.items():
if not v:
@@ -169,13 +160,13 @@ class RulesWriter:
continue
res[item]['patterns'].add(_pat)
sorted_res = defaultdict(lambda: defaultdict(list)) # type: dict[str, dict[str, list]]
sorted_res = defaultdict(lambda: defaultdict(list)) # type: t.Dict[str, t.Dict[str, t.List[str]]]
for k, v in res.items():
for vk, vv in v.items():
sorted_res[k][vk] = sorted(vv)
return sorted_res
def new_labels_str(self): # type: () -> str
def new_labels_str(self) -> str:
_labels = set([])
for k, v in self.cfg.items():
if not v:
@@ -191,14 +182,14 @@ class RulesWriter:
return res
@classmethod
def _format_label(cls, label): # type: (str) -> str
def _format_label(cls, label: str) -> str:
return cls.LABEL_TEMPLATE.format(label, cls.bot_label_str(label))
@staticmethod
def bot_label_str(label): # type: (str) -> str
def bot_label_str(label: str) -> str:
return label.upper().replace('-', '_')
def new_rules_str(self): # type: () -> str
def new_rules_str(self) -> str:
res = []
for k, v in sorted(self.rules.items()):
if k.startswith('pattern'):
@@ -211,13 +202,13 @@ class RulesWriter:
res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
return '\n\n'.join(res)
def _format_rule(self, name, cfg): # type: (str, dict) -> str
def _format_rule(self, name: str, cfg: t.Dict[str, t.Any]) -> str:
_rules = [self.RULE_REVERT_BRANCH]
if name.endswith('-production'):
_rules.append(self.RULE_PROTECTED_NO_LABEL)
_rules.append(self.RULE_PROTECTED_PUSH)
else:
if not (name.endswith('-preview') or name.startswith('labels:')):
_rules.append(self.RULE_PROTECTED)
_rules.append(self.RULE_PROTECTED_CHECK)
if name.startswith('test:'):
_rules.append(self.RULE_BUILD_ONLY)
@@ -235,7 +226,7 @@ class RulesWriter:
print('WARNING: pattern {} not exists'.format(pattern))
return '\n'.join(_rules)
def update_rules_yml(self): # type: () -> bool
def update_rules_yml(self) -> bool:
with open(self.rules_yml) as fr:
file_str = fr.read()
@@ -255,7 +246,9 @@ PATTERN_COLOR = 'cyan'
RULE_COLOR = 'blue'
def build_graph(rules_dict): # type: (dict[str, dict[str, list]]) -> pgv.AGraph
def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGraph':
from pygraphviz import pgv
graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True)
for k, v in rules_dict.items():
@@ -281,7 +274,7 @@ def build_graph(rules_dict): # type: (dict[str, dict[str, list]]) -> pgv.AGraph
return graph
def output_graph(graph, output_path='output.png'): # type: (pgv.AGraph, str) -> None
def output_graph(graph: 'pgv.AGraph', output_path: str = 'output.png') -> None:
graph.layout('dot')
if output_path.endswith('.png'):
img_path = output_path
@@ -292,13 +285,16 @@ def output_graph(graph, output_path='output.png'): # type: (pgv.AGraph, str) ->
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'),
help='rules.yml file path')
parser.add_argument('dependencies_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies',
'dependencies.yml'),
help='dependencies.yml file path')
parser.add_argument('--graph',
help='Specify PNG image output path. Use this argument to generate dependency graph')
parser.add_argument(
'rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), help='rules.yml file path'
)
parser.add_argument(
'dependencies_yml',
nargs='?',
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies', 'dependencies.yml'),
help='dependencies.yml file path',
)
parser.add_argument('--graph', help='Specify PNG image output path. Use this argument to generate dependency graph')
args = parser.parse_args()
writer = RulesWriter(args.rules_yml, args.dependencies_yml)