alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-09 12:35:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

style: format python files with isort and double-quote-string-fixer

Browse Source
This commit is contained in:
Fu Hanxi
2021-01-26 10:49:01 +08:00
parent dc8402ea61
commit 0146f258d7
276 changed files with 8241 additions and 8162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
tools/ci/deploy_docs.py
View File

@@ -22,9 +22,10 @@ import os
import os.path
import re
import stat
import sys
import subprocess
import sys
import tarfile
import packaging.version
@@ -34,86 +35,86 @@ def env(variable, default=None):
# import sanitize_version from the docs directory, shared with here
sys.path.append(os.path.join(env("IDF_PATH"), "docs"))
sys.path.append(os.path.join(env('IDF_PATH'), 'docs'))
from sanitize_version import sanitize_version # noqa
def main():
# if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
git_ver = env("GIT_VER") # output of git describe --always
ci_ver = env("CI_COMMIT_REF_NAME", git_ver) # branch or tag we're building for (used for 'release' & URL)
git_ver = env('GIT_VER') # output of git describe --always
ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL)
version = sanitize_version(ci_ver)
print("Git version: {}".format(git_ver))
print("CI Version: {}".format(ci_ver))
print("Deployment version: {}".format(version))
print('Git version: {}'.format(git_ver))
print('CI Version: {}'.format(ci_ver))
print('Deployment version: {}'.format(version))
if not version:
raise RuntimeError("A version is needed to deploy")
raise RuntimeError('A version is needed to deploy')
build_dir = env("DOCS_BUILD_DIR") # top-level local build dir, where docs have already been built
build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built
if not build_dir:
raise RuntimeError("Valid DOCS_BUILD_DIR is needed to deploy")
raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy')
url_base = env("DOCS_DEPLOY_URL_BASE") # base for HTTP URLs, used to print the URL to the log after deploying
url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying
docs_server = env("DOCS_DEPLOY_SERVER") # ssh server to deploy to
docs_user = env("DOCS_DEPLOY_SERVER_USER")
docs_path = env("DOCS_DEPLOY_PATH") # filesystem path on DOCS_SERVER
docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to
docs_user = env('DOCS_DEPLOY_SERVER_USER')
docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER
if not docs_server:
raise RuntimeError("Valid DOCS_DEPLOY_SERVER is needed to deploy")
raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy')
if not docs_user:
raise RuntimeError("Valid DOCS_DEPLOY_SERVER_USER is needed to deploy")
raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy')
docs_server = "{}@{}".format(docs_user, docs_server)
docs_server = '{}@{}'.format(docs_user, docs_server)
if not docs_path:
raise RuntimeError("Valid DOCS_DEPLOY_PATH is needed to deploy")
raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy')
print("DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}".format(docs_server, docs_path))
print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path))
tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
deploy(version, tarball_path, docs_path, docs_server)
print("Docs URLs:")
print('Docs URLs:')
doc_deploy_type = os.getenv('TYPE')
for vurl in version_urls:
language, _, target = vurl.split('/')
tag = '{}_{}'.format(language, target)
url = "{}/{}/index.html".format(url_base, vurl) # (index.html needed for the preview server)
url = re.sub(r"([^:])//", r"\1/", url) # get rid of any // that isn't in the https:// part
url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server)
url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part
print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
# note: it would be neater to use symlinks for stable, but because of the directory order
# (language first) it's kind of a pain to do on a remote server, so we just repeat the
# process but call the version 'stable' this time
if is_stable_version(version):
print("Deploying again as stable version...")
tarball_path, version_urls = build_doc_tarball("stable", git_ver, build_dir)
deploy("stable", tarball_path, docs_path, docs_server)
print('Deploying again as stable version...')
tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir)
deploy('stable', tarball_path, docs_path, docs_server)
def deploy(version, tarball_path, docs_path, docs_server):
def run_ssh(commands):
""" Log into docs_server and run a sequence of commands using ssh """
print("Running ssh: {}".format(commands))
subprocess.run(["ssh", "-o", "BatchMode=yes", docs_server, "-x", " && ".join(commands)], check=True)
print('Running ssh: {}'.format(commands))
subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True)
# copy the version tarball to the server
run_ssh(["mkdir -p {}".format(docs_path)])
print("Running scp {} to {}".format(tarball_path, "{}:{}".format(docs_server, docs_path)))
subprocess.run(["scp", "-B", tarball_path, "{}:{}".format(docs_server, docs_path)], check=True)
run_ssh(['mkdir -p {}'.format(docs_path)])
print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path)))
subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True)
tarball_name = os.path.basename(tarball_path)
run_ssh(["cd {}".format(docs_path),
"rm -rf ./*/{}".format(version), # remove any pre-existing docs matching this version
"tar -zxvf {}".format(tarball_name), # untar the archive with the new docs
"rm {}".format(tarball_name)])
run_ssh(['cd {}'.format(docs_path),
'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version
'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs
'rm {}'.format(tarball_name)])
# Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
# as the version will be invalid for a window of time. Better to do it atomically, but this is
@@ -124,21 +125,21 @@ def build_doc_tarball(version, git_ver, build_dir):
""" Make a tar.gz archive of the docs, in the directory structure used to deploy as
the given version """
version_paths = []
tarball_path = "{}/{}.tar.gz".format(build_dir, version)
tarball_path = '{}/{}.tar.gz'.format(build_dir, version)
# find all the 'html/' directories under build_dir
html_dirs = glob.glob("{}/**/html/".format(build_dir), recursive=True)
print("Found %d html directories" % len(html_dirs))
html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True)
print('Found %d html directories' % len(html_dirs))
pdfs = glob.glob("{}/**/latex/build/*.pdf".format(build_dir), recursive=True)
print("Found %d PDFs in latex directories" % len(pdfs))
pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True)
print('Found %d PDFs in latex directories' % len(pdfs))
# add symlink for stable and latest and adds them to PDF blob
symlinks = create_and_add_symlinks(version, git_ver, pdfs)
def not_sources_dir(ti):
""" Filter the _sources directories out of the tarballs """
if ti.name.endswith("/_sources"):
if ti.name.endswith('/_sources'):
return None
ti.mode |= stat.S_IWGRP # make everything group-writeable
@@ -149,7 +150,7 @@ def build_doc_tarball(version, git_ver, build_dir):
except OSError:
pass
with tarfile.open(tarball_path, "w:gz") as tarball:
with tarfile.open(tarball_path, 'w:gz') as tarball:
for html_dir in html_dirs:
# html_dir has the form '<ignored>/<language>/<target>/html/'
target_dirname = os.path.dirname(os.path.dirname(html_dir))
@@ -157,7 +158,7 @@ def build_doc_tarball(version, git_ver, build_dir):
language = os.path.basename(os.path.dirname(target_dirname))
# when deploying, we want the top-level directory layout 'language/version/target'
archive_path = "{}/{}/{}".format(language, version, target)
archive_path = '{}/{}/{}'.format(language, version, target)
print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
tarball.add(html_dir, archive_path, filter=not_sources_dir)
version_paths.append(archive_path)
@@ -171,7 +172,7 @@ def build_doc_tarball(version, git_ver, build_dir):
language = os.path.basename(os.path.dirname(target_dirname))
# when deploying, we want the layout 'language/version/target/pdf'
archive_path = "{}/{}/{}/{}".format(language, version, target, pdf_filename)
archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename)
print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
tarball.add(pdf_path, archive_path)
@@ -192,34 +193,34 @@ def create_and_add_symlinks(version, git_ver, pdfs):
symlinks.append(symlink_path)
pdfs.extend(symlinks)
print("Found %d PDFs in latex directories after adding symlink" % len(pdfs))
print('Found %d PDFs in latex directories after adding symlink' % len(pdfs))
return symlinks
def is_stable_version(version):
""" Heuristic for whether this is the latest stable release """
if not version.startswith("v"):
if not version.startswith('v'):
return False # branch name
if "-" in version:
if '-' in version:
return False # prerelease tag
git_out = subprocess.check_output(["git", "tag", "-l"]).decode("utf-8")
git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8')
versions = [v.strip() for v in git_out.split("\n")]
versions = [v for v in versions if re.match(r"^v[\d\.]+$", v)] # include vX.Y.Z only
versions = [v.strip() for v in git_out.split('\n')]
versions = [v for v in versions if re.match(r'^v[\d\.]+$', v)] # include vX.Y.Z only
versions = [packaging.version.parse(v) for v in versions]
max_version = max(versions)
if max_version.public != version[1:]:
print("Stable version is v{}. This version is {}.".format(max_version.public, version))
print('Stable version is v{}. This version is {}.'.format(max_version.public, version))
return False
else:
print("This version {} is the stable version".format(version))
print('This version {} is the stable version'.format(version))
return True
if __name__ == "__main__":
if __name__ == '__main__':
main()