docs: deploy docs to production server from CI

.gitlab-ci.yml

@@ -46,6 +46,8 @@ variables:
 # target test config file, used by assign test job
   CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
 
+  # Versioned esp-idf-doc env image to use for all document building jobs
+  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v3"
 
 
 # before each job, we need to check if this job is filtered by bot stage/job filter

docs/build_docs.py

@@ -9,6 +9,20 @@
 #
 # Specific custom docs functionality should be added in conf_common.py or in a Sphinx extension, not here.
 #
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 from __future__ import print_function
 import argparse
 import locale

docs/conf_common.py

@@ -21,6 +21,7 @@ import os
 import os.path
 import re
 import subprocess
+from sanitize_version import sanitize_version
 from idf_extensions.util import download_file_if_missing
 
 # build_docs on the CI server sometimes fails under Python3. This is a workaround:
@@ -107,17 +108,7 @@ version = subprocess.check_output(['git', 'describe']).strip().decode('utf-8')
 
 # The 'release' version is the same as version for non-CI builds, but for CI
 # builds on a branch then it's replaced with the branch name
-try:
+release = sanitize_version(version)
-    # default to using the Gitlab CI branch or tag if one is present
-    release = os.environ['CI_COMMIT_REF_NAME']
-
-    # emulate RTD's naming scheme for branches, master->latest, etc
-    release = release.replace('/', '-')
-    if release == "master":
-        release = "latest"
-except KeyError:
-    # otherwise, fall back to the full git version (no branch info)
-    release = version
 
 print('Version: {0}  Release: {1}'.format(version, release))
 

docs/sanitize_version.py

@@ -0,0 +1,43 @@
+# Tiny Python module to sanitize a Git version into something that can be used in a URL
+#
+# (this is used in multiple places: conf_common.py and in tools/ci/docs_deploy
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+
+def sanitize_version(original_version):
+    """ Given a version (probably output from 'git describe --always' or similar), return
+    a URL-safe sanitized version. (this is used as 'release' config variable when building
+    the docs.)
+
+    Will override the original version with the Gitlab CI CI_COMMIT_REF_NAME environment variable if
+    this is present.
+
+    Also follows the RTD-ism that master branch is named 'latest'
+
+    """
+
+    try:
+        version = os.environ['CI_COMMIT_REF_NAME']
+    except KeyError:
+        version = original_version
+
+    if version == "master":
+        return "latest"
+
+    version = version.replace('/', '-')
+
+    return version

tools/ci/config/build.yml

@@ -232,7 +232,7 @@ build_test_apps_esp32s2:
 
 .build_docs_template: &build_docs_template
   stage: build
-  image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:v2
+  image: $ESP_IDF_DOC_ENV_IMAGE
   tags:
     - build_docs
   artifacts:

tools/ci/config/deploy.yml

@@ -70,63 +70,32 @@ push_to_github:
     - git remote add github git@github.com:espressif/esp-idf.git
     - tools/ci/push_to_github.sh
 
-.upload_doc_archive: &upload_doc_archive |
+.deploy_docs_template:
-  cd docs/_build/$DOCLANG/$DOCTGT
+  extends: .before_script_lesser
-  mv html $DOCTGT
-  tar czf $DOCTGT.tar.gz $DOCTGT
-  # arrange for URL format language/version/target
-  ssh $DOCS_SERVER -x "mkdir -p $DOCS_PATH/$DOCLANG/$GIT_VER"
-  scp $DOCTGT.tar.gz $DOCS_SERVER:$DOCS_PATH/$DOCLANG/$GIT_VER"
-  ssh $DOCS_SERVER -x "cd $DOCS_PATH/$DOCLANG/$GIT_VER && tar xzf ${DOCTGT}.tar.gz && rm -f ../latest && ln -s . ../latest"
-  cd -
-
-deploy_docs:
   stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_IDF_DOC_ENV_IMAGE
   tags:
     - deploy
     - shiny
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD_DOCS
   dependencies:
     - build_docs_en_esp32
     - build_docs_en_esp32s2
     - build_docs_zh_CN_esp32
     - build_docs_zh_CN_esp32s2
-  extends: .before_script_lesser
+  variables:
+    DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
+    PYTHONUNBUFFERED: 1
   script:
     - mkdir -p ~/.ssh
     - chmod 700 ~/.ssh
-    - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
+    - echo -n $DOCS_DEPLOY_PRIVATEKEY > ~/.ssh/id_rsa_base64
     - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
     - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
+    - echo -e "Host $DOCS_DEPLOY_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_DEPLOY_SERVER_USER\n" >> ~/.ssh/config
     - export GIT_VER=$(git describe --always)
 
-    - DOCLANG=en; DOCTGT=esp32
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py
-    - *upload_doc_archive
 
-    - DOCLANG=en; DOCTGT=esp32s2
-    - *upload_doc_archive
-
-    - DOCLANG=zh_CN; DOCTGT=esp32
-    - *upload_doc_archive
-
-    - DOCLANG=zh_CN; DOCTGT=esp32s2
-    - *upload_doc_archive
-
-    # log link to doc URLs
-    - echo "[document $TYPE][en][esp32] $DOCS_URL_BASE/en/${GIT_VER}/esp32/index.html"
-    - echo "[document $TYPE][en][esp32s2] $DOCS_URL_BASE/en/${GIT_VER}/esp32s2/index.html"
-    - echo "[document $TYPE][zh_CN][esp32] $DOCS_URL_BASE/zh_CN/${GIT_VER}/esp32/index.html"
-    - echo "[document $TYPE][zh_CN][esp32s2] $DOCS_URL_BASE/zh_CN/${GIT_VER}/esp32s2/index.html"
 
 # deploys docs to CI_DOCKER_REGISTRY webserver, for internal review
 deploy_docs_preview:
@@ -139,26 +108,29 @@ deploy_docs_preview:
       - $BOT_LABEL_BUILD_DOCS
   variables:
     TYPE: "preview"
-    # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_PATH for preview server so we keep them
+    # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
-    DOCS_DEPLOY_KEY: "$DOCS_DEPLOY_KEY"
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY"
-    DOCS_SERVER: "$DOCS_SERVER"
+    DOCS_DEPLOY_SERVER: "$DOCS_SERVER"
-    DOCS_PATH: "$DOCS_PATH"
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER"
-    DOCS_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
+    DOCS_DEPLOY_PATH: "$DOCS_PATH"
+    DOCS_DEPLOY_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
 
 # deploy docs to production webserver
 deploy_docs_production:
   extends: .deploy_docs_template
   only:
     refs:
+      # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
       - master
       - /^release\/v/
       - /^v\d+\.\d+(\.\d+)?($|-)/
   variables:
     TYPE: "preview"
-    DOCS_DEPLOY_KEY: "WIP"
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
-    DOCS_SERVER: "WIP"
+    DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER"
-    DOCS_PATH: "WIP"
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
-    DOCS_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
+    DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
+    DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
 
 deploy_test_result:
   stage: deploy

tools/ci/config/post_deploy.yml

@@ -1,6 +1,6 @@
 .check_doc_links_template: &check_doc_links_template
   stage: post_deploy
-  image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:v2
+  image: $ESP_IDF_DOC_ENV_IMAGE
   tags: [ "build", "amd64", "internet" ]
   only:
     - master

tools/ci/deploy_docs.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+#
+# CI script to deploy docs to a webserver. Not useful outside of CI environment
+#
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import glob
+import os
+import os.path
+import re
+import stat
+import sys
+import subprocess
+import tarfile
+import packaging.version
+
+
+def env(variable, default=None):
+    """ Shortcut to return the expanded version of an environment variable """
+    return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
+
+
+# import sanitize_version from the docs directory, shared with here
+sys.path.append(os.path.join(env("IDF_PATH"), "docs"))
+from sanitize_version import sanitize_version  # noqa
+
+
+def main():
+    # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
+    git_ver = env("GIT_VER")  # output of git describe --always
+    ci_ver = env("CI_COMMIT_REF_NAME", git_ver)  # branch or tag we're building for (used for 'release' & URL)
+
+    version = sanitize_version(ci_ver)
+    print("Git version: {}".format(git_ver))
+    print("CI Version: {}".format(ci_ver))
+    print("Deployment version: {}".format(version))
+
+    if not version:
+        raise RuntimeError("A version is needed to deploy")
+
+    build_dir = env("DOCS_BUILD_DIR")  # top-level local build dir, where docs have already been built
+
+    if not build_dir:
+        raise RuntimeError("Valid DOCS_BUILD_DIR is needed to deploy")
+
+    url_base = env("DOCS_DEPLOY_URL_BASE")  # base for HTTP URLs, used to print the URL to the log after deploying
+
+    docs_server = env("DOCS_DEPLOY_SERVER")  # ssh server to deploy to
+    docs_user = env("DOCS_DEPLOY_SERVER_USER")
+    docs_path = env("DOCS_DEPLOY_PATH")  # filesystem path on DOCS_SERVER
+
+    if not docs_server:
+        raise RuntimeError("Valid DOCS_DEPLOY_SERVER is needed to deploy")
+
+    if not docs_user:
+        raise RuntimeError("Valid DOCS_DEPLOY_SERVER_USER is needed to deploy")
+
+    docs_server = "{}@{}".format(docs_user, docs_server)
+
+    if not docs_path:
+        raise RuntimeError("Valid DOCS_DEPLOY_PATH is needed to deploy")
+
+    print("DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}".format(docs_server, docs_path))
+
+    tarball_path, version_urls = build_doc_tarball(version, build_dir)
+
+    deploy(version, tarball_path, docs_path, docs_server)
+
+    print("Docs URLs:")
+    for vurl in version_urls:
+        url = "{}/{}/index.html".format(url_base, vurl)  # (index.html needed for the preview server)
+        url = re.sub(r"([^:])//", r"\1/", url)  # get rid of any // that isn't in the https:// part
+        print(url)
+
+    # note: it would be neater to use symlinks for stable, but because of the directory order
+    # (language first) it's kind of a pain to do on a remote server, so we just repeat the
+    # process but call the version 'stable' this time
+    if is_stable_version(version):
+        print("Deploying again as stable version...")
+        tarball_path, version_urls = build_doc_tarball("stable", build_dir)
+        deploy("stable", tarball_path, docs_path, docs_server)
+
+
+def deploy(version, tarball_path, docs_path, docs_server):
+
+    def run_ssh(commands):
+        """ Log into docs_server and run a sequence of commands using ssh """
+        print("Running ssh: {}".format(commands))
+        subprocess.run(["ssh", "-o", "BatchMode=yes", docs_server, "-x", " && ".join(commands)], check=True)
+
+    # copy the version tarball to the server
+    run_ssh(["mkdir -p {}".format(docs_path)])
+    print("Running scp {} to {}".format(tarball_path, "{}:{}".format(docs_server, docs_path)))
+    subprocess.run(["scp", "-B", tarball_path, "{}:{}".format(docs_server, docs_path)], check=True)
+
+    tarball_name = os.path.basename(tarball_path)
+
+    run_ssh(["cd {}".format(docs_path),
+             "rm -rf ./*/{}".format(version),   # remove any pre-existing docs matching this version
+             "tar -zxvf {}".format(tarball_name),  # untar the archive with the new docs
+             "rm {}".format(tarball_name)])
+
+    # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
+    # as the version will be invalid for a window of time. Better to do it atomically, but this is
+    # another thing made much more complex by the directory structure putting language before version...
+
+
+def build_doc_tarball(version, build_dir):
+    """ Make a tar.gz archive of the docs, in the directory structure used to deploy as
+        the given version """
+    version_paths = []
+    tarball_path = "{}/{}.tar.gz".format(build_dir, version)
+
+    # find all the 'html/' directories under build_dir
+    html_dirs = glob.glob("{}/**/html/".format(build_dir), recursive=True)
+    print("Found %d html directories" % len(html_dirs))
+
+    def not_sources_dir(ti):
+        """ Filter the _sources directories out of the tarballs """
+        if ti.name.endswith("/_sources"):
+            return None
+
+        ti.mode |= stat.S_IWGRP  # make everything group-writeable
+        return ti
+
+    try:
+        os.remove(tarball_path)
+    except OSError:
+        pass
+
+    with tarfile.open(tarball_path, "w:gz") as tarball:
+        for html_dir in html_dirs:
+            # html_dir has the form '<ignored>/<language>/<target>/html/'
+            target_dirname = os.path.dirname(os.path.dirname(html_dir))
+            target = os.path.basename(target_dirname)
+            language = os.path.basename(os.path.dirname(target_dirname))
+
+            # when deploying, we want the top-level directory layout 'language/version/target'
+            archive_path = "{}/{}/{}".format(language, version, target)
+            print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
+            tarball.add(html_dir, archive_path, filter=not_sources_dir)
+            version_paths.append(archive_path)
+
+    return (os.path.abspath(tarball_path), version_paths)
+
+
+def is_stable_version(version):
+    """ Heuristic for whether this is the latest stable release """
+    if not version.startswith("v"):
+        return False  # branch name
+    if "-" in version:
+        return False  # prerelease tag
+
+    git_out = subprocess.run(["git", "tag", "-l"], capture_output=True, check=True)
+
+    versions = [v.strip() for v in git_out.stdout.decode("utf-8").split("\n")]
+    versions = [v for v in versions if re.match(r"^v[\d\.]+$", v)]  # include vX.Y.Z only
+
+    versions = [packaging.version.parse(v) for v in versions]
+
+    max_version = max(versions)
+
+    if max_version.public != version[1:]:
+        print("Stable version is v{}. This version is {}.".format(max_version.public, version))
+        return False
+    else:
+        print("This version {} is the stable version".format(version))
+        return True
+
+
+if __name__ == "__main__":
+    main()

tools/ci/executable-list.txt

@@ -45,6 +45,7 @@ tools/ci/check_idf_version.sh
 tools/ci/check_public_headers.sh
 tools/ci/check_ut_cmake_make.sh
 tools/ci/checkout_project_ref.py
+tools/ci/deploy_docs.py
 tools/ci/envsubst.py
 tools/ci/fix_empty_prototypes.sh
 tools/ci/get-full-sources.sh