Add scripts for doc versioning (MDAnalysis#2749)

- Scripts move docs into a version subfolder - Write versions.json at root directory - Write sitemap_index.xml at root directory - Write html stubs for redirection - Change URL to CNAME one of docs.mdanalysis.org/
PicoCentauri · Mar 30, 2021 · 72725f1 · 72725f1
1 parent 661160f
commit 72725f1
Show file tree

Hide file tree

Showing 7 changed files with 124 additions and 139 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -18,6 +18,7 @@ env:
     - GIT_CI_USER=TravisCI
     - GIT_CI_EMAIL=TravisCI@mdanalysis.org
     - MDA_DOCDIR=${TRAVIS_BUILD_DIR}/package/doc/html/html
+    - MAINTAIN_DIR=${TRAVIS_BUILD_DIR}/maintainer
     # Set default python version to avoid repetition later
     - PYTHON_VERSION=3.6
     - BUILD_DOCS=false
@@ -54,7 +55,7 @@ matrix:
            BUILD_DOCS=true
            BUILD_CMD="cd ${TRAVIS_BUILD_DIR}/package && python setup.py build_ext --inplace"
            INSTALL_HOLE="false"
-           PIP_DEPENDENCIES="${PIP_DEPENDENCIES} sphinx==1.8.5 sphinx-sitemap sphinx_rtd_theme"
+           PIP_DEPENDENCIES="${PIP_DEPENDENCIES} sphinx==1.8.5 sphinx-sitemap sphinx_rtd_theme msmb_theme==1.2.0"
 
     - env: NAME="Lint"
            PYLINTRC="${TRAVIS_BUILD_DIR}/package/.pylintrc"
@@ -130,6 +131,14 @@ after_success:
       codecov; \
     fi
   # can't use test here since this leads to travis fails even though the build passes
-  - if [[ ${TRAVIS_PULL_REQUEST} == "false" ]] && [[ ${BUILD_DOCS} == "true" ]] && [[ ${TRAVIS_BRANCH} == ${GH_DOC_BRANCH} ]]; then
-            bash ${TRAVIS_BUILD_DIR}/maintainer/deploy_docs.sh;
+  # turn off blocking as it causes large writes to stdout to fail 
+  # (see https://github.com/travis-ci/travis-ci/issues/4704)
+  - |
+    if [[ ${TRAVIS_PULL_REQUEST} == "false" ]] && [[ ${BUILD_DOCS} == "true" ]] && [[ ${TRAVIS_BRANCH} == ${GH_DOC_BRANCH} ]]; then
+      python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
+      cd ${TRAVIS_BUILD_DIR}/package/MDAnalysis
+      export VERSION=$(python -c "import version; print(version.__version__)")
+      cd -
+      bash ${TRAVIS_BUILD_DIR}/maintainer/deploy_docs_via_travis.sh;
     fi
+
diff --git a/maintainer/deploy_docs_via_travis.sh b/maintainer/deploy_docs_via_travis.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+# Deploying docs from travis-ci.
+# See https://github.com/MDAnalysis/mdanalysis/issues/386
+# Script based on https://github.com/steveklabnik/automatically_update_github_pages_with_travis_example
+
+# Run this script from the top-level of the checked out git
+# repository. A github OAuth token must be available in the evironment
+# variable GH_TOKEN and is set up through the .travis.yml
+# env:global:secure parameter (encrypted with travis-ci's public key)/
+#
+# Additional environment variables set in .travis.yml
+#  GH_REPOSITORY     repo to full from and push to
+#  GH_DOC_BRANCH     branch from which the docs are built
+#  GIT_CI_USER       name of the user to push docs as
+#  GIT_CI_EMAIL      email of the user to push docs as
+#  MDA_DOCDIR        path to the docdir from top of repo
+#  MAINTAIN_DIR      path to maintainer/
+#  VERSION           version of MDAnalysis
+#
+# NOTE: If any of these environment variables are not set or 
+#       empty then the script will exit with and error (-o nounset).
+
+set -o errexit -o nounset
+
+function die () {
+    local msg="$1" err=${2:-1}
+    echo "ERROR: $msg [$err]"
+    exit $err
+}
+
+rev=$(git rev-parse --short HEAD)
+
+# the following tests should be superfluous because of -o nounset
+test -n "${GH_TOKEN}" || die "GH_TOKEN is empty: need OAuth GitHub token to continue" 100
+test -n "${GH_REPOSITORY}" || die "GH_REPOSITORY must be set in .travis.yml" 100
+test -n "${MDA_DOCDIR}" || die "MDA_DOCDIR must be set in .travis.yml" 100
+test -n "${MAINTAIN_DIR}" || die "MAINTAIN_DIR must be set in .travis.yml" 100
+test -n "${VERSION}" || die "VERSION must be set in .travis.yml" 100
+
+
+cd ${MDA_DOCDIR} || die "Failed to 'cd ${MDA_DOCDIR}'. Run from the top level of the repository"
+
+# move into $version subdirectory
+mkdir ../${VERSION} && mv * ../${VERSION}
+
+git init
+git config user.name "${GIT_CI_USER}"
+git config user.email "${GIT_CI_EMAIL}"
+
+mv ../${VERSION} $VERSION
+
+git remote add upstream "https://${GH_TOKEN}@${GH_REPOSITORY}"
+git fetch --depth 50 upstream ${GH_DOC_BRANCH} gh-pages
+git reset upstream/gh-pages
+
+# for dev, latest, home redirects
+mkdir dev latest
+export URL="https://docs.mdanalysis.org"
+python ${MAINTAIN_DIR}/update_json_stubs_sitemap.py
+touch .
+touch .nojekyll
+
+git add -A ${VERSION}/
+git add .nojekyll versions.json
+git add index.html dev latest
+git add *.xml
+
+# check for anything to commit
+# https://stackoverflow.com/questions/3878624/how-do-i-programmatically-determine-if-there-are-uncommited-changes
+git diff-index --quiet HEAD -- || git commit -m "rebuilt html docs for version ${VERSION} from branch ${GH_DOC_BRANCH} with sphinx at ${rev}"
+git push -q upstream HEAD:gh-pages
+
+
diff --git a/maintainer/update_json_stubs_sitemap.py b/maintainer/update_json_stubs_sitemap.py
@@ -13,10 +13,6 @@
 import os
 import shutil
 import xml.etree.ElementTree as ET
-import errno
-import glob
-import textwrap
-import shutil
 
 try:
     from urllib.request import Request, urlopen
@@ -26,16 +22,6 @@
 URL = os.environ['URL']
 VERSION = os.environ['VERSION']
 
-if "http" not in URL:
-    raise ValueError("URL should have the transfer protocol (HTTP/S). "
-                     f"Given: $URL={URL}")
-
-try:
-    int(VERSION[0])
-except ValueError:
-    raise ValueError("$VERSION should start with a number. "
-                     f"Given: $VERSION={VERSION}") from None
-
 
 def get_web_file(filename, callback, default):
     url = os.path.join(URL, filename)
@@ -54,30 +40,14 @@ def get_web_file(filename, callback, default):
         return callback(data)
 
 
-def write_redirect(file, version='', outfile=None):
-    if outfile is None:
-        outfile = file
-    url = os.path.join(URL, version, file)
-    REDIRECT = textwrap.dedent(f"""
-    <!DOCTYPE html>
-    <meta charset="utf-8">
-    <title>Redirecting to {url}</title>
-    <meta http-equiv="refresh" content="0; URL={url}">
-    <link rel="canonical" href="{url}">
-    """)
-    with open(outfile, 'w') as f:
-        f.write(REDIRECT)
-    print(f"Wrote redirect from {url} to {outfile}")
-
-
 # ========= WRITE JSON =========
 # Update $root/versions.json with links to the right version
 versions = get_web_file('versions.json', json.loads, [])
 existing = [item['version'] for item in versions]
 already_exists = VERSION in existing
-latest = 'dev' not in VERSION
 
 if not already_exists:
+    latest = 'dev' not in VERSION
     if latest:
         for ver in versions:
             ver['latest'] = False
@@ -89,112 +59,52 @@ def write_redirect(file, version='', outfile=None):
         'latest': latest
     })
 
+with open("versions.json", 'w') as f:
+    json.dump(versions, f, indent=2)
+
+# ========= WRITE HTML STUBS =========
+# Add HTML files to redirect:
+# index.html -> latest release
+# latest/index.html -> latest release
+# dev/index.html -> dev docs
+
+REDIRECT = """
+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>Redirecting to {url}</title>
+<meta http-equiv="refresh" content="0; URL={url}">
+<link rel="canonical" href="{url}">
+"""
+
 for ver in versions[::-1]:
     if ver['latest']:
-        latest_version = ver['version']
-        break
+        latest_url = ver['url']
 else:
     try:
-        latest_version = versions[-1]['version']
+        latest_url = versions[-1]['url']
     except IndexError:
-        latest_version = None
+        latest_url = None
 
 for ver in versions[::-1]:
-    if '-dev' in ver['version']:
-        dev_version = ver['version']
+    if 'dev' in ver['version']:
+        dev_url = ver['url']
         break
 else:
     try:
-        dev_version = versions[-1]['version']
+        dev_url = versions[-1]['url']
     except IndexError:
-        dev_version = None
+        dev_url = None
 
-versions.sort(key=lambda x: x["version"])
+if latest_url:
+    with open('index.html', 'w') as f:
+        f.write(REDIRECT.format(url=latest_url))
 
-# ========= WRITE HTML STUBS AND COPY DOCS =========
-# Add HTML files to redirect:
-# index.html -> stable/ docs
-# latest/index.html -> latest release (not dev docs)
-# stable/ : a copy of the release docs with the highest number (2.0.0 instead of 1.0.0)
-# dev/ : a copy of the develop docs with the highest number (2.0.0-dev instead of 1.0.1-dev)
-# sitemap.xml files are updated by replacing URL strings
-
-
-def redirect_sitemap(old_version, new_version):
-    """Replace paths in copied sitemap.xml with new directory path
-
-    Sitemaps can only contain URLs 'within' that directory structure.
-    For more, see https://www.sitemaps.org/faq.html#faq_sitemap_location
-    """
-    file = f"{new_version}/sitemap.xml"
-    old = f"{URL}/{old_version}/"
-    new = f"{URL}/{new_version}/"
-    try:
-        with open(file, "r") as f:
-            contents = f.read()
-    except OSError:
-        raise ValueError(f"{file} not found")
-    redirected = contents.replace(old, new)
-    with open(file, "w") as f:
-        f.write(redirected)
-    print(f"Redirected URLs in {file} from {old} to {new}")
-
-
-def add_or_update_version(version):
-    """Add or update the version path to versions.json"""
-    for ver in versions:
-        if ver["version"] == version:
-            ver["url"] = os.path.join(URL, version)
-            break
-    else:
-        versions.append({
-            "version": version,
-            "display": version,
-            "url": os.path.join(URL, version),
-            "latest": False
-        })
-
-
-def copy_version(old_version, new_version):
-    """Copy docs from one directory to another with all bells and whistles"""
-    shutil.copytree(old_version, new_version)
-    print(f"Copied {old_version} to {new_version}")
-    redirect_sitemap(old_version, new_version)
-    add_or_update_version(new_version)
-
-
-# Copy stable/ docs and write redirects from root level docs
-if latest:
-    copy_version(VERSION, "stable")
-    html_files = glob.glob(f'stable/**/*.html', recursive=True)
-    for file in html_files:
-        # below should be true because we only globbed stable/* paths
-        assert file.startswith("stable/")
-        outfile = file[7:]  # strip "stable/"
-        dirname = os.path.dirname(outfile)
-        if dirname and not os.path.exists(dirname):
-            try:
-                os.makedirs(dirname)
-            except OSError as exc:
-                if exc.errno != errno.EEXIST:
-                    raise
-
-        write_redirect(file, '', outfile)
-
-# Separate just in case we update versions.json or muck around manually
-# with docs
-if latest_version:
-    write_redirect('index.html', "stable")
-    write_redirect('index.html', latest_version, 'latest/index.html')
-
-# Copy dev/ docs
-if dev_version and dev_version == VERSION:
-    copy_version(VERSION, "dev")
-
-# update versions.json online
-with open("versions.json", 'w') as f:
-    json.dump(versions, f, indent=2)
+    with open('latest/index.html', 'w') as f:
+        f.write(REDIRECT.format(url=latest_url))
 
+if dev_url:
+    with open('dev/index.html', 'w') as f:
+        f.write(REDIRECT.format(url=dev_url))
 
 # ========= WRITE SUPER SITEMAP.XML =========
 # make one big sitemap.xml

diff --git a/package/doc/README b/package/doc/README
@@ -39,7 +39,7 @@ The documentation for the latest stable release can always be found at
 The docs for the latest development version are also on the internet
 at
 
-   https://docs.mdanalysis.org/dev
+   http://docs.mdanalysis.org/dev
 
 The manual includes all the doc strings with some additional text; it
 is a work in progress and suggestions to improve it are welcome. File

diff --git a/package/doc/sphinx/source/conf.py b/package/doc/sphinx/source/conf.py
@@ -15,7 +15,6 @@
 import os
 import platform
 import datetime
-import MDAnalysis as mda
 import msmb_theme  # for little versions pop-up
 # https://sphinx-rtd-theme.readthedocs.io/en/stable/
 import sphinx_rtd_theme

diff --git a/package/setup.py b/package/setup.py
@@ -159,7 +159,7 @@ class MDAExtension(Extension, object):
     #  care of calling it when needed.
     def __init__(self, name, sources, *args, **kwargs):
         self._mda_include_dirs = []
-        # don't abspath sources else packaging fails on Windows (issue #3129)
+        sources = [abspath(s) for s in sources]
         super(MDAExtension, self).__init__(name, sources, *args, **kwargs)
 
     @property
@@ -548,14 +548,8 @@ def long_description(readme):
     except (OSError, IOError):
         warnings.warn('Cannot write the list of authors.')
 
-    try:
-        # when building from repository for creating the distribution
-        LONG_DESCRIPTION = long_description("pypi-description.rst")
-    except OSError:
-        # when building from a tar file for installation
-        # (LONG_DESCRIPTION is not really needed)
-        LONG_DESCRIPTION = "MDAnalysis -- https://www.mdanalysis.org/"
-
+    with open(abspath('SUMMARY.txt')) as summary:
+        LONG_DESCRIPTION = summary.read()
     CLASSIFIERS = [
         'Development Status :: 6 - Mature',
         'Environment :: Console',

diff --git a/testsuite/README b/testsuite/README
@@ -53,8 +53,8 @@ For questions and discussions for code development, join the developer list http
 
 .. |devdocs| image:: https://img.shields.io/badge/docs-development-yellow.svg
    :alt: Documentation (development version)
-   :target: https://docs.mdanalysis.org/dev
-
+   :target: https://docs.mdanalysis.org/dev/
+	     
 .. |developergroup| image:: https://img.shields.io/badge/Google%20Group-Developers-lightgrey.svg
    :alt: Developer Google Group
    :target: https://groups.google.com/group/mdnalysis-devel