Add scripts for doc versioning (#2749)

- Scripts move docs into a version subfolder
- Write versions.json at root directory
- Write sitemap_index.xml at root directory
- Write html stubs for redirection
- Change URL to CNAME one of docs.mdanalysis.org/

.travis.yml

@@ -18,6 +18,7 @@ env:
     - GIT_CI_USER=TravisCI
     - GIT_CI_EMAIL=TravisCI@mdanalysis.org
     - MDA_DOCDIR=${TRAVIS_BUILD_DIR}/package/doc/html/html
+    - MAINTAIN_DIR=${TRAVIS_BUILD_DIR}/maintainer
     # Set default python version to avoid repetition later
     - PYTHON_VERSION=3.6
     - BUILD_DOCS=false
@@ -54,7 +55,7 @@ matrix:
            BUILD_DOCS=true
            BUILD_CMD="cd ${TRAVIS_BUILD_DIR}/package && python setup.py build_ext --inplace"
            INSTALL_HOLE="false"
-           PIP_DEPENDENCIES="${PIP_DEPENDENCIES} sphinx==1.8.5 sphinx-sitemap sphinx_rtd_theme"
+           PIP_DEPENDENCIES="${PIP_DEPENDENCIES} sphinx==1.8.5 sphinx-sitemap sphinx_rtd_theme msmb_theme==1.2.0"
 
     - env: NAME="Lint"
            PYLINTRC="${TRAVIS_BUILD_DIR}/package/.pylintrc"
@@ -130,6 +131,14 @@ after_success:
       codecov; \
     fi
   # can't use test here since this leads to travis fails even though the build passes
-  - if [[ ${TRAVIS_PULL_REQUEST} == "false" ]] && [[ ${BUILD_DOCS} == "true" ]] && [[ ${TRAVIS_BRANCH} == ${GH_DOC_BRANCH} ]]; then
-            bash ${TRAVIS_BUILD_DIR}/maintainer/deploy_docs.sh;
+  # turn off blocking as it causes large writes to stdout to fail 
+  # (see https://github.com/travis-ci/travis-ci/issues/4704)
+  - |
+    if [[ ${TRAVIS_PULL_REQUEST} == "false" ]] && [[ ${BUILD_DOCS} == "true" ]] && [[ ${TRAVIS_BRANCH} == ${GH_DOC_BRANCH} ]]; then
+      python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
+      cd ${TRAVIS_BUILD_DIR}/package/MDAnalysis
+      export VERSION=$(python -c "import version; print(version.__version__)")
+      cd -
+      bash ${TRAVIS_BUILD_DIR}/maintainer/deploy_docs_via_travis.sh;
     fi
+

README.rst

@@ -128,7 +128,7 @@ to find uncovered code.
 
 .. |devdocs| image:: https://img.shields.io/badge/docs-development-yellow.svg
    :alt: Documentation (development version)
-   :target: https://www.mdanalysis.org/mdanalysis/
+   :target: https://docs.mdanalysis.org/dev
 
 .. |numfocus| image:: https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A
    :alt: Powered by NumFOCUS

maintainer/deploy_docs.sh → maintainer/deploy_docs_via_travis.sh

@@ -14,6 +14,8 @@
 #  GIT_CI_USER       name of the user to push docs as
 #  GIT_CI_EMAIL      email of the user to push docs as
 #  MDA_DOCDIR        path to the docdir from top of repo
+#  MAINTAIN_DIR      path to maintainer/
+#  VERSION           version of MDAnalysis
 #
 # NOTE: If any of these environment variables are not set or 
 #       empty then the script will exit with and error (-o nounset).
@@ -32,24 +34,40 @@ rev=$(git rev-parse --short HEAD)
 test -n "${GH_TOKEN}" || die "GH_TOKEN is empty: need OAuth GitHub token to continue" 100
 test -n "${GH_REPOSITORY}" || die "GH_REPOSITORY must be set in .travis.yml" 100
 test -n "${MDA_DOCDIR}" || die "MDA_DOCDIR must be set in .travis.yml" 100
+test -n "${MAINTAIN_DIR}" || die "MAINTAIN_DIR must be set in .travis.yml" 100
+test -n "${VERSION}" || die "VERSION must be set in .travis.yml" 100
+
 
 cd ${MDA_DOCDIR} || die "Failed to 'cd ${MDA_DOCDIR}'. Run from the top level of the repository"
 
+# move into $version subdirectory
+mkdir ../${VERSION} && mv * ../${VERSION}
+
 git init
 git config user.name "${GIT_CI_USER}"
 git config user.email "${GIT_CI_EMAIL}"
 
+mv ../${VERSION} $VERSION
+
 git remote add upstream "https://${GH_TOKEN}@${GH_REPOSITORY}"
 git fetch --depth 50 upstream ${GH_DOC_BRANCH} gh-pages
 git reset upstream/gh-pages
 
+# for dev, latest, home redirects
+mkdir dev latest
+export URL="https://docs.mdanalysis.org"
+python ${MAINTAIN_DIR}/update_json_stubs_sitemap.py
 touch .
 touch .nojekyll
 
-git add -A .
+git add -A ${VERSION}/
+git add .nojekyll versions.json
+git add index.html dev latest
+git add *.xml
+
 # check for anything to commit
 # https://stackoverflow.com/questions/3878624/how-do-i-programmatically-determine-if-there-are-uncommited-changes
-git diff-index --quiet HEAD -- || git commit -m "rebuilt html docs from branch ${GH_DOC_BRANCH} with sphinx at ${rev}"
+git diff-index --quiet HEAD -- || git commit -m "rebuilt html docs for version ${VERSION} from branch ${GH_DOC_BRANCH} with sphinx at ${rev}"
 git push -q upstream HEAD:gh-pages
 
 

maintainer/update_json_stubs_sitemap.py

@@ -0,0 +1,140 @@
+# Adapted from mdtraj/devtools/travis-ci/update_versions_json.py,
+# by Robert T McGibbon, under the LGPLv2.1 license
+# Similarly by the terms of LGPL this is also in the public domain
+# Lily Wang, 2020
+#
+# This script is called by deploy_docs_via_travis.sh to:
+#  1. update versions.json
+#  2. Write some redirect stubs
+#  3. Write a sitemap.xml file for the root directory
+#
+
+import json
+import os
+import shutil
+import xml.etree.ElementTree as ET
+
+try:
+    from urllib.request import Request, urlopen
+except ImportError:
+    from urllib2 import Request, urlopen
+
+URL = os.environ['URL']
+VERSION = os.environ['VERSION']
+
+
+def get_web_file(filename, callback, default):
+    url = os.path.join(URL, filename)
+    try:
+        page = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+        data = urlopen(page).read().decode()
+    except Exception as e:
+        print(e)
+        try:
+            with open(filename, 'r') as f:
+                return callback(f)
+        except IOError as e:
+            print(e)
+            return default
+    else:
+        return callback(data)
+
+
+# ========= WRITE JSON =========
+# Update $root/versions.json with links to the right version
+versions = get_web_file('versions.json', json.loads, [])
+existing = [item['version'] for item in versions]
+already_exists = VERSION in existing
+
+if not already_exists:
+    latest = 'dev' not in VERSION
+    if latest:
+        for ver in versions:
+            ver['latest'] = False
+
+    versions.append({
+        'version': VERSION,
+        'display': VERSION,
+        'url': os.path.join(URL, VERSION),
+        'latest': latest
+    })
+
+with open("versions.json", 'w') as f:
+    json.dump(versions, f, indent=2)
+
+# ========= WRITE HTML STUBS =========
+# Add HTML files to redirect:
+# index.html -> latest release
+# latest/index.html -> latest release
+# dev/index.html -> dev docs
+
+REDIRECT = """
+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>Redirecting to {url}</title>
+<meta http-equiv="refresh" content="0; URL={url}">
+<link rel="canonical" href="{url}">
+"""
+
+for ver in versions[::-1]:
+    if ver['latest']:
+        latest_url = ver['url']
+else:
+    try:
+        latest_url = versions[-1]['url']
+    except IndexError:
+        latest_url = None
+
+for ver in versions[::-1]:
+    if 'dev' in ver['version']:
+        dev_url = ver['url']
+        break
+else:
+    try:
+        dev_url = versions[-1]['url']
+    except IndexError:
+        dev_url = None
+
+if latest_url:
+    with open('index.html', 'w') as f:
+        f.write(REDIRECT.format(url=latest_url))
+
+    with open('latest/index.html', 'w') as f:
+        f.write(REDIRECT.format(url=latest_url))
+
+if dev_url:
+    with open('dev/index.html', 'w') as f:
+        f.write(REDIRECT.format(url=dev_url))
+
+# ========= WRITE SUPER SITEMAP.XML =========
+# make one big sitemap.xml
+ET.register_namespace('xhtml', "http://www.w3.org/1999/xhtml")
+
+# so we could make 1 big sitemap as commented
+# below, but they must be max 50 MB / 50k URL.
+# Yes, this is 100+ releases, but who knows when
+# that'll happen and who'll look at this then?
+# bigroot = ET.Element("urlset")
+# bigroot.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")
+# for ver in versions:
+#     tree = get_web_file(ver['version']+'/sitemap.xml', ET.fromstring,
+#                         ET.fromstring(''))
+#     root = tree.getroot()
+#     bigroot.extend(root.getchildren())
+# ET.ElementTree(bigroot).write('sitemap.xml',
+#                               xml_declaration=True,
+#                               encoding='utf-8',
+#                               method="xml")
+
+# so instead we make a sitemap of sitemaps.
+bigroot = ET.Element("sitemapindex")
+bigroot.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")
+for ver in versions:
+    path = os.path.join(URL, '{}/sitemap.xml'.format(ver['version']))
+    sitemap = ET.SubElement(bigroot, 'sitemap')
+    ET.SubElement(sitemap, 'loc').text = path
+
+ET.ElementTree(bigroot).write('sitemap_index.xml',
+                              xml_declaration=True,
+                              encoding='utf-8',
+                              method="xml")

package/doc/README

@@ -34,12 +34,12 @@ Online html manual
 
 The documentation for the latest stable release can always be found at
 
-   https://docs.mdanalysis.org
+   https://www.mdanalysis.org/docs
 
 The docs for the latest development version are also on the internet
 at
 
-   http://devdocs.mdanalysis.org
+   http://docs.mdanalysis.org/dev
 
 The manual includes all the doc strings with some additional text; it
 is a work in progress and suggestions to improve it are welcome. File