alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-07 20:00:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tools: Improve the Python package system

Browse Source 
Introduce features into the Python package management system & manage
package versions outside of ESP-IDF repo.
This commit is contained in:
Roland Dobai
2021-09-16 16:48:03 +02:00
parent 9fc95216da
commit b28d7e6850
25 changed files with 441 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
tools/check_python_dependencies.py
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
@@ -8,69 +8,80 @@ import os
import re
import sys
PYTHON_PACKAGE_RE = re.compile(r'[^<>=~]+')
try:
import pkg_resources
except Exception:
except ImportError:
print('pkg_resources cannot be imported probably because the pip package is not installed and/or using a '
'legacy Python interpreter. Please refer to the Get Started section of the ESP-IDF Programming Guide for '
'setting up the required packages.')
sys.exit(1)
def escape_backslash(path):
if sys.platform == 'win32':
# escaped backslashes are necessary in order to be able to copy-paste the printed path
return path.replace('\\', '\\\\')
else:
return path
if __name__ == '__main__':
idf_path = os.getenv('IDF_PATH')
default_requirements_path = os.path.join(idf_path, 'requirements.txt') # type: ignore
parser = argparse.ArgumentParser(description='ESP-IDF Python package dependency checker')
parser.add_argument('--requirements', '-r',
help='Path to the requirements file',
default=default_requirements_path)
help='Path to a requirements file (can be used multiple times)',
action='append', default=[])
parser.add_argument('--constraints', '-c', default=[],
help='Path to a constraints file (can be used multiple times)',
action='append')
args = parser.parse_args()
required_set = set()
for req_path in args.requirements:
with open(req_path) as f:
required_set |= set(i for i in map(str.strip, f.readlines()) if len(i) > 0 and not i.startswith('#'))
constr_dict = {} # for example package_name -> package_name==1.0
for const_path in args.constraints:
with open(const_path) as f:
for con in [i for i in map(str.strip, f.readlines()) if len(i) > 0 and not i.startswith('#')]:
if con.startswith('file://'):
con = os.path.basename(con)
elif con.startswith('--only-binary'):
continue
elif con.startswith('-e') and '#egg=' in con: # version control URLs, take the egg= part at the end only
con_m = re.search(r'#egg=([^\s]+)', con)
if not con_m:
print('Malformed input. Cannot find name in {}'.format(con))
sys.exit(1)
con = con_m[1]
name_m = PYTHON_PACKAGE_RE.search(con)
if not name_m:
print('Malformed input. Cannot find name in {}'.format(con))
sys.exit(1)
constr_dict[name_m[0]] = con
# We need to constrain package dependencies as well. So all installed packages need to be checked.
# For example package A requires package B. We have only A in our requirements. But the newest version of B could
# broke at some time and in that case we add a constraint for B (on the server) but don't have to update the
# requirement file (in the ESP-IDF repo).
required_set |= set(i.key for i in pkg_resources.working_set)
not_satisfied = []
with open(args.requirements) as f:
for line in f:
line = line.strip()
# pkg_resources.require() cannot handle the full requirements file syntax so we need to make
# adjustments for options which we use.
if line.startswith('file://'):
line = os.path.basename(line)
if line.startswith('--only-binary'):
continue
if line.startswith('-e') and '#egg=' in line: # version control URLs, take the egg= part at the end only
line = re.search(r'#egg=([^\s]+)', line).group(1) # type: ignore
try:
pkg_resources.require(line)
except Exception:
not_satisfied.append(line)
for requirement in required_set:
# If there is a version-specific constraint for the requirement then use it. Otherwise, just use the
# requirement as is.
to_require = constr_dict.get(requirement, requirement)
try:
pkg_resources.require(to_require)
except pkg_resources.ResolutionError:
not_satisfied.append(to_require)
if len(not_satisfied) > 0:
print('The following Python requirements are not satisfied:')
for requirement in not_satisfied:
print(requirement)
if os.path.realpath(args.requirements) != os.path.realpath(default_requirements_path):
# we're using this script to check non-default requirements.txt, so tell the user to run pip
print('Please check the documentation for the feature you are using, or run "%s -m pip install -r %s"' % (sys.executable, args.requirements))
elif os.environ.get('IDF_PYTHON_ENV_PATH'):
print(os.linesep.join(not_satisfied))
if 'IDF_PYTHON_ENV_PATH' in os.environ:
# We are running inside a private virtual environment under IDF_TOOLS_PATH,
# ask the user to run install.bat again.
if sys.platform == 'win32':
install_script = 'install.bat'
else:
install_script = 'install.sh'
print('To install the missing packages, please run "%s"' % os.path.join(idf_path, install_script)) # type: ignore
install_script = 'install.bat' if sys.platform == 'win32' else 'install.sh'
print('To install the missing packages, please run "{}"'.format(install_script))
else:
print('Please follow the instructions found in the "Set up the tools" section of '
'ESP-IDF Getting Started Guide')
'ESP-IDF Getting Started Guide.')
print('Diagnostic information:')
idf_python_env_path = os.environ.get('IDF_PYTHON_ENV_PATH')
@@ -81,4 +92,4 @@ if __name__ == '__main__':
print(' PATH: {}'.format(os.getenv('PATH')))
sys.exit(1)
print('Python requirements from {} are satisfied.'.format(args.requirements))
print('Python requirements are satisfied.')