tools: Installing tools for given IDF_TARGET

Allow user to select specific ESP_TARGET while setting up ESD_IDF.
Only necessary tools for given target will be downloaded and installed.

Closes https://github.com/espressif/esp-idf/issues/5113

tools/idf_tools.py

@@ -60,6 +60,7 @@ from collections import OrderedDict, namedtuple
 
 try:
     import typing  # noqa: F401
+    from typing import IO, Any, Callable, Optional, Tuple, Union  # noqa: F401
 except ImportError:
     pass
 
@@ -79,6 +80,7 @@ except ImportError:
 TOOLS_FILE = 'tools/tools.json'
 TOOLS_SCHEMA_FILE = 'tools/tools_schema.json'
 TOOLS_FILE_NEW = 'tools/tools.new.json'
+IDF_ENV_FILE = 'idf-env.json'
 TOOLS_FILE_VERSION = 1
 IDF_TOOLS_PATH_DEFAULT = os.path.join('~', '.espressif')
 UNKNOWN_VERSION = 'unknown'
@@ -479,7 +481,8 @@ OPTIONS_LIST = ['version_cmd',
                 'install',
                 'info_url',
                 'license',
-                'strip_container_dirs']
+                'strip_container_dirs',
+                'supported_targets']
 
 IDFToolOptions = namedtuple('IDFToolOptions', OPTIONS_LIST)
 
@@ -490,8 +493,9 @@ class IDFTool(object):
     INSTALL_ON_REQUEST = 'on_request'
     INSTALL_NEVER = 'never'
 
-    def __init__(self, name, description, install, info_url, license, version_cmd, version_regex, version_regex_replace=None,
+    def __init__(self, name, description, install, info_url, license, version_cmd, version_regex, supported_targets, version_regex_replace=None,
                  strip_container_dirs=0):
+        # type: (str, str, str, str, str, list[str], str, list[str], Optional[str], int) -> None
         self.name = name
         self.description = description
         self.versions = OrderedDict()  # type: typing.Dict[str, IDFToolVersion]
@@ -500,8 +504,8 @@ class IDFTool(object):
         if version_regex_replace is None:
             version_regex_replace = VERSION_REGEX_REPLACE_DEFAULT
         self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace,
-                                      [], OrderedDict(), install, info_url, license, strip_container_dirs)
-        self.platform_overrides = []
+                                      [], OrderedDict(), install, info_url, license, strip_container_dirs, supported_targets)  # type: ignore
+        self.platform_overrides = []  # type: list[dict[str, str]]
         self._platform = CURRENT_PLATFORM
         self._update_current_options()
 
@@ -580,7 +584,10 @@ class IDFTool(object):
     def get_install_type(self):
         return self._current_options.install
 
-    def compatible_with_platform(self):
+    def get_supported_targets(self):  # type: ()  -> list[str]
+        return self._current_options.supported_targets  # type: ignore
+
+    def compatible_with_platform(self):  # type: () -> bool
         return any([v.compatible_with_platform() for v in self.versions.values()])
 
     def get_supported_platforms(self):  # type: () -> typing.Set[str]
@@ -793,10 +800,14 @@ class IDFTool(object):
         if type(overrides_list) is not list:
             raise RuntimeError('platform_overrides for tool %s is not a list' % tool_name)
 
+        supported_targets = tool_dict.get('supported_targets')
+        if not isinstance(supported_targets, list):
+            raise RuntimeError('supported_targets for tool %s is not a list of strings' % tool_name)
+
         # Create the object
-        tool_obj = cls(tool_name, description, install, info_url, license,
-                       version_cmd, version_regex, version_regex_replace,
-                       strip_container_dirs)
+        tool_obj = cls(tool_name, description, install, info_url, license,  # type: ignore
+                       version_cmd, version_regex, supported_targets, version_regex_replace,  # type: ignore
+                       strip_container_dirs)  # type: ignore
 
         for path in export_paths:
             tool_obj.options.export_paths.append(path)
@@ -902,6 +913,7 @@ class IDFTool(object):
             'license': self.options.license,
             'version_cmd': self.options.version_cmd,
             'version_regex': self.options.version_regex,
+            'supported_targets': self.options.supported_targets,
             'versions': versions_array,
         }
         if self.options.version_regex_replace != VERSION_REGEX_REPLACE_DEFAULT:
@@ -1009,7 +1021,99 @@ def get_python_env_path():
     return idf_python_env_path, idf_python_export_path, virtualenv_python
 
 
-def action_list(args):
+def get_idf_env():  # type: () -> Any
+    try:
+        idf_env_file_path = os.path.join(global_idf_tools_path, IDF_ENV_FILE)  # type: ignore
+        with open(idf_env_file_path, 'r') as idf_env_file:
+            return json.load(idf_env_file)
+    except (IOError, OSError):
+        if not os.path.exists(idf_env_file_path):
+            warn('File {} was not found. '.format(idf_env_file_path))
+        else:
+            filename, ending = os.path.splitext(os.path.basename(idf_env_file_path))
+            warn('File {} can not be opened, renaming to {}'.format(idf_env_file_path,filename + '_failed' + ending))
+            os.rename(idf_env_file_path, os.path.join(os.path.dirname(idf_env_file_path), (filename + '_failed' + ending)))
+
+        info('Creating {}' .format(idf_env_file_path))
+        return {'idfSelectedId': 'sha', 'idfInstalled': {'sha': {'targets': {}}}}
+
+
+def export_targets_to_idf_env_json(targets):  # type: (list[str]) -> None
+    idf_env_json = get_idf_env()
+    targets = list(set(targets + get_user_defined_targets()))
+
+    for env in idf_env_json['idfInstalled']:
+        if env == idf_env_json['idfSelectedId']:
+            idf_env_json['idfInstalled'][env]['targets'] = targets
+            break
+
+    try:
+        with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'w') as w:  # type: ignore
+            json.dump(idf_env_json, w, indent=4)
+    except (IOError, OSError):
+        warn('File {} can not be created. '.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE)))  # type: ignore
+
+
+def clean_targets(targets_str):  # type: (str) -> list[str]
+    targets_from_tools_json = get_all_targets_from_tools_json()
+    invalid_targets = []
+
+    targets_str = targets_str.lower()
+    targets = targets_str.replace('-', '').split(',')
+    if targets != ['all']:
+        invalid_targets = [t for t in targets if t not in targets_from_tools_json]
+        if invalid_targets:
+            warn('Targets: "{}" are not supported. Only allowed options are: {}.'.format(', '.join(invalid_targets), ', '.join(targets_from_tools_json)))
+            raise SystemExit(1)
+        # removing duplicates
+        targets = list(set(targets))
+        export_targets_to_idf_env_json(targets)
+    else:
+        export_targets_to_idf_env_json(targets_from_tools_json)
+    return targets
+
+
+def get_user_defined_targets():  # type: () -> list[str]
+    try:
+        with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'r') as idf_env_file:  # type: ignore
+            idf_env_json = json.load(idf_env_file)
+    except (OSError, IOError):
+        # warn('File {} was not found. Installing tools for all esp targets.'.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE)))  # type: ignore
+        return []
+
+    targets = []
+    for env in idf_env_json['idfInstalled']:
+        if env == idf_env_json['idfSelectedId']:
+            targets = idf_env_json['idfInstalled'][env]['targets']
+            break
+    return targets
+
+
+def get_all_targets_from_tools_json():  # type: () -> list[str]
+    tools_info = load_tools_info()
+    targets_from_tools_json = []  # type: list[str]
+
+    for _, v in tools_info.items():
+        targets_from_tools_json.extend(v.get_supported_targets())
+    # remove duplicates
+    targets_from_tools_json = list(set(targets_from_tools_json))
+    if 'all' in targets_from_tools_json:
+        targets_from_tools_json.remove('all')
+    return sorted(targets_from_tools_json)
+
+
+def filter_tools_info(tools_info):  # type: (OrderedDict[str, IDFTool]) -> OrderedDict[str,IDFTool]
+    targets = get_user_defined_targets()
+    if not targets:
+        return tools_info
+    else:
+        filtered_tools_spec = {k:v for k, v in tools_info.items() if
+                               (v.get_install_type() == IDFTool.INSTALL_ALWAYS or v.get_install_type() == IDFTool.INSTALL_ON_REQUEST) and
+                               (any(item in targets for item in v.get_supported_targets()) or v.get_supported_targets() == ['all'])}
+        return OrderedDict(filtered_tools_spec)
+
+
+def action_list(args):  # type: ignore
     tools_info = load_tools_info()
     for name, tool in tools_info.items():
         if tool.get_install_type() == IDFTool.INSTALL_NEVER:
@@ -1030,6 +1134,7 @@ def action_list(args):
 
 def action_check(args):
     tools_info = load_tools_info()
+    tools_info = filter_tools_info(tools_info)
     not_found_list = []
     info('Checking for installed tools...')
     for name, tool in tools_info.items():
@@ -1056,6 +1161,7 @@ def action_check(args):
 
 def action_export(args):
     tools_info = load_tools_info()
+    tools_info = filter_tools_info(tools_info)
     all_tools_found = True
     export_vars = {}
     paths_to_export = []
@@ -1237,6 +1343,10 @@ def apply_github_assets_option(tool_download_obj):
 def action_download(args):
     tools_info = load_tools_info()
     tools_spec = args.tools
+    targets = []  # type: list[str]
+    # Installing only single tools, no targets are specified.
+    if 'required' in tools_spec:
+        targets = clean_targets(args.targets)
 
     if args.platform not in PLATFORM_FROM_NAME:
         fatal('unknown platform: {}' % args.platform)
@@ -1249,8 +1359,17 @@ def action_download(args):
         tools_info_for_platform[name] = tool_for_platform
 
     if not tools_spec or 'required' in tools_spec:
+        # Downloading tools for all ESP_targets required by the operating system.
         tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
+        # Filtering tools user defined list of ESP_targets
+        if 'all' not in targets:
+            def is_tool_selected(tool):  # type: (IDFTool) -> bool
+                supported_targets = tool.get_supported_targets()
+                return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
+            tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])]
         info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec)))
+
+    # Downloading tools for all ESP_targets (MacOS, Windows, Linux)
     elif 'all' in tools_spec:
         tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
         info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec)))
@@ -1283,10 +1402,25 @@ def action_download(args):
 
 def action_install(args):
     tools_info = load_tools_info()
-    tools_spec = args.tools
+    tools_spec = args.tools  # type: ignore
+    targets = []  # type: list[str]
+    # Installing only single tools, no targets are specified.
+    if 'required' in tools_spec:
+        targets = clean_targets(args.targets)
+        info('Selected targets are: {}' .format(', '.join(get_user_defined_targets())))
+
     if not tools_spec or 'required' in tools_spec:
+        # Installing tools for all ESP_targets required by the operating system.
         tools_spec = [k for k, v in tools_info.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
+        # Filtering tools user defined list of ESP_targets
+        if 'all' not in targets:
+            def is_tool_selected(tool):  # type: (IDFTool) -> bool
+                supported_targets = tool.get_supported_targets()
+                return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
+            tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])]
         info('Installing tools: {}'.format(', '.join(tools_spec)))
+
+    # Installing tools for all ESP_targets (MacOS, Windows, Linux)
     elif 'all' in tools_spec:
         tools_spec = [k for k, v in tools_info.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
         info('Installing tools: {}'.format(', '.join(tools_spec)))
@@ -1573,6 +1707,8 @@ def main(argv):
                          'To install a specific version use <tool_name>@<version> syntax. ' +
                          'Use empty or \'required\' to install required tools, not optional ones. ' +
                          'Use \'all\' to install all tools, including the optional ones.')
+    install.add_argument('--targets', default='all', help='A comma separated list of desired chip targets for installing.' +
+                         ' It defaults to installing all supported targets.')
 
     download = subparsers.add_parser('download', help='Download the tools into the dist directory')
     download.add_argument('--platform', help='Platform to download the tools for')
@@ -1581,6 +1717,8 @@ def main(argv):
                           'To download a specific version use <tool_name>@<version> syntax. ' +
                           'Use empty or \'required\' to download required tools, not optional ones. ' +
                           'Use \'all\' to download all tools, including the optional ones.')
+    download.add_argument('--targets', default='all', help='A comma separated list of desired chip targets for installing.' +
+                          ' It defaults to installing all supported targets.')
 
     if IDF_MAINTAINER:
         for subparser in [download, install]: