CI: Add pre-commit for esp-idf project.

add tools/ci/python_packages/idf_ci.py for some util functions used in
ci and needs multi-os solution

tools/ci/check_codeowners.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+#
+# Utility script for ESP-IDF developers to work with the CODEOWNERS file.
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+from idf_ci_utils import IDF_PATH
+
+CODEOWNERS_PATH = os.path.join(IDF_PATH, ".gitlab", "CODEOWNERS")
+CODEOWNER_GROUP_PREFIX = "@esp-idf-codeowners/"
+
+
+def get_all_files():
+    """
+    Get list of all file paths in the repository.
+    """
+    # only split on newlines, since file names may contain spaces
+    return subprocess.check_output(["git", "ls-files"], cwd=IDF_PATH).decode("utf-8").strip().split('\n')
+
+
+def pattern_to_regex(pattern):
+    """
+    Convert the CODEOWNERS path pattern into a regular expression string.
+    """
+    orig_pattern = pattern  # for printing errors later
+
+    # Replicates the logic from normalize_pattern function in Gitlab ee/lib/gitlab/code_owners/file.rb:
+    if not pattern.startswith('/'):
+        pattern = '/**/' + pattern
+    if pattern.endswith('/'):
+        pattern = pattern + '**/*'
+
+    # Convert the glob pattern into a regular expression:
+    # first into intermediate tokens
+    pattern = (pattern.replace('**/', ':REGLOB:')
+                      .replace('**', ':INVALID:')
+                      .replace('*', ':GLOB:')
+                      .replace('.', ':DOT:')
+                      .replace('?', ':ANY:'))
+
+    if pattern.find(':INVALID:') >= 0:
+        raise ValueError("Likely invalid pattern '{}': '**' should be followed by '/'".format(orig_pattern))
+
+    # then into the final regex pattern:
+    re_pattern = (pattern.replace(':REGLOB:', '(?:.*/)?')
+                         .replace(':GLOB:', '[^/]*')
+                         .replace(':DOT:', '[.]')
+                         .replace(':ANY:', '.') + '$')
+    if re_pattern.startswith('/'):
+        re_pattern = '^' + re_pattern
+
+    return re_pattern
+
+
+def files_by_regex(all_files, regex):
+    """
+    Return all files in the repository matching the given regular expresion.
+    """
+    return [file for file in all_files if regex.search('/' + file)]
+
+
+def files_by_pattern(all_files, pattern=None):
+    """
+    Return all the files in the repository matching the given CODEOWNERS pattern.
+    """
+    if not pattern:
+        return all_files
+
+    return files_by_regex(all_files, re.compile(pattern_to_regex(pattern)))
+
+
+def action_identify(args):
+    best_match = []
+    all_files = get_all_files()
+    with open(CODEOWNERS_PATH) as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            tokens = line.split()
+            path_pattern = tokens[0]
+            owners = tokens[1:]
+            files = files_by_pattern(all_files, path_pattern)
+            if args.path in files:
+                best_match = owners
+    for owner in best_match:
+        print(owner)
+
+
+def action_test_pattern(args):
+    re_pattern = pattern_to_regex(args.pattern)
+
+    if args.regex:
+        print(re_pattern)
+        return
+
+    files = files_by_regex(get_all_files(), re.compile(re_pattern))
+    for f in files:
+        print(f)
+
+
+def action_ci_check(args):
+    errors = []
+
+    def add_error(msg):
+        errors.append("{}:{}: {}".format(CODEOWNERS_PATH, line_no, msg))
+
+    all_files = get_all_files()
+    prev_path_pattern = ""
+    with open(CODEOWNERS_PATH) as f:
+        for line_no, line in enumerate(f, start=1):
+            # Skip empty lines and comments
+            line = line.strip()
+            if line.startswith("# sort-order-reset"):
+                prev_path_pattern = ""
+
+            if not line or line.startswith("#"):
+                continue
+
+            # Each line has a form of "<path> <owners>+"
+            tokens = line.split()
+            path_pattern = tokens[0]
+            owners = tokens[1:]
+            if not owners:
+                add_error("no owners specified for {}".format(path_pattern))
+
+            # Check that the file is sorted by path patterns
+            path_pattern_for_cmp = path_pattern.replace("-", "_")  # ignore difference between _ and - for ordering
+            if prev_path_pattern and path_pattern_for_cmp < prev_path_pattern:
+                add_error("file is not sorted: {} < {}".format(path_pattern_for_cmp, prev_path_pattern))
+            prev_path_pattern = path_pattern_for_cmp
+
+            # Check that the pattern matches at least one file
+            files = files_by_pattern(all_files, path_pattern)
+            if not files:
+                add_error("no files matched by pattern {}".format(path_pattern))
+
+            for o in owners:
+                # Sanity-check the owner group name
+                if not o.startswith(CODEOWNER_GROUP_PREFIX):
+                    add_error("owner {} doesn't start with {}".format(o, CODEOWNER_GROUP_PREFIX))
+
+    if not errors:
+        print("No errors found.")
+    else:
+        print("Errors found!")
+        for e in errors:
+            print(e)
+        raise SystemExit(1)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        sys.argv[0], description="Internal helper script for working with the CODEOWNERS file."
+    )
+    subparsers = parser.add_subparsers(dest="action")
+
+    identify = subparsers.add_parser(
+        "identify",
+        help="List the owners of the specified path within IDF."
+        "This command doesn't support files inside submodules, or files not added to git repository.",
+    )
+    identify.add_argument("path", help="Path of the file relative to the root of the repository")
+
+    subparsers.add_parser(
+        "ci-check",
+        help="Check CODEOWNERS file: every line should match at least one file, sanity-check group names, "
+        "check that the file is sorted by paths",
+    )
+
+    test_pattern = subparsers.add_parser(
+        "test-pattern",
+        help="Print files in the repository for a given CODEOWNERS pattern. Useful when adding new rules."
+    )
+    test_pattern.add_argument("--regex", action="store_true", help="Print the equivalent regular expression instead of the file list.")
+    test_pattern.add_argument("pattern", help="Path pattern to get the list of files for")
+
+    args = parser.parse_args()
+
+    if args.action is None:
+        parser.print_help()
+        parser.exit(1)
+
+    action_func_name = "action_" + args.action.replace("-", "_")
+    action_func = globals()[action_func_name]
+    action_func(args)
+
+
+if __name__ == "__main__":
+    main()