diff --git a/.travis.yml b/.travis.yml index ba98443157a..084b8c95852 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ env: - GIT_CI_USER=TravisCI - GIT_CI_EMAIL=TravisCI@mdanalysis.org - MDA_DOCDIR=${TRAVIS_BUILD_DIR}/package/doc/html/html + - MAINTAIN_DIR=${TRAVIS_BUILD_DIR}/maintainer # Set default python version to avoid repetition later - PYTHON_VERSION=3.6 - BUILD_DOCS=false @@ -54,7 +55,7 @@ matrix: BUILD_DOCS=true BUILD_CMD="cd ${TRAVIS_BUILD_DIR}/package && python setup.py build_ext --inplace" INSTALL_HOLE="false" - PIP_DEPENDENCIES="${PIP_DEPENDENCIES} sphinx==1.8.5 sphinx-sitemap sphinx_rtd_theme" + PIP_DEPENDENCIES="${PIP_DEPENDENCIES} sphinx==1.8.5 sphinx-sitemap sphinx_rtd_theme msmb_theme==1.2.0" - env: NAME="Lint" PYLINTRC="${TRAVIS_BUILD_DIR}/package/.pylintrc" @@ -130,6 +131,14 @@ after_success: codecov; \ fi # can't use test here since this leads to travis fails even though the build passes - - if [[ ${TRAVIS_PULL_REQUEST} == "false" ]] && [[ ${BUILD_DOCS} == "true" ]] && [[ ${TRAVIS_BRANCH} == ${GH_DOC_BRANCH} ]]; then - bash ${TRAVIS_BUILD_DIR}/maintainer/deploy_docs.sh; + # turn off blocking as it causes large writes to stdout to fail + # (see https://github.com/travis-ci/travis-ci/issues/4704) + - | + if [[ ${TRAVIS_PULL_REQUEST} == "false" ]] && [[ ${BUILD_DOCS} == "true" ]] && [[ ${TRAVIS_BRANCH} == ${GH_DOC_BRANCH} ]]; then + python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' + cd ${TRAVIS_BUILD_DIR}/package/MDAnalysis + export VERSION=$(python -c "import version; print(version.__version__)") + cd - + bash ${TRAVIS_BUILD_DIR}/maintainer/deploy_docs_via_travis.sh; fi + diff --git a/maintainer/deploy_docs_via_travis.sh b/maintainer/deploy_docs_via_travis.sh new file mode 100644 index 00000000000..af503e56727 --- /dev/null +++ b/maintainer/deploy_docs_via_travis.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Deploying docs from travis-ci. +# See https://github.com/MDAnalysis/mdanalysis/issues/386 +# Script based on https://github.com/steveklabnik/automatically_update_github_pages_with_travis_example + +# Run this script from the top-level of the checked out git +# repository. A github OAuth token must be available in the evironment +# variable GH_TOKEN and is set up through the .travis.yml +# env:global:secure parameter (encrypted with travis-ci's public key)/ +# +# Additional environment variables set in .travis.yml +# GH_REPOSITORY repo to full from and push to +# GH_DOC_BRANCH branch from which the docs are built +# GIT_CI_USER name of the user to push docs as +# GIT_CI_EMAIL email of the user to push docs as +# MDA_DOCDIR path to the docdir from top of repo +# MAINTAIN_DIR path to maintainer/ +# VERSION version of MDAnalysis +# +# NOTE: If any of these environment variables are not set or +# empty then the script will exit with and error (-o nounset). + +set -o errexit -o nounset + +function die () { + local msg="$1" err=${2:-1} + echo "ERROR: $msg [$err]" + exit $err +} + +rev=$(git rev-parse --short HEAD) + +# the following tests should be superfluous because of -o nounset +test -n "${GH_TOKEN}" || die "GH_TOKEN is empty: need OAuth GitHub token to continue" 100 +test -n "${GH_REPOSITORY}" || die "GH_REPOSITORY must be set in .travis.yml" 100 +test -n "${MDA_DOCDIR}" || die "MDA_DOCDIR must be set in .travis.yml" 100 +test -n "${MAINTAIN_DIR}" || die "MAINTAIN_DIR must be set in .travis.yml" 100 +test -n "${VERSION}" || die "VERSION must be set in .travis.yml" 100 + + +cd ${MDA_DOCDIR} || die "Failed to 'cd ${MDA_DOCDIR}'. Run from the top level of the repository" + +# move into $version subdirectory +mkdir ../${VERSION} && mv * ../${VERSION} + +git init +git config user.name "${GIT_CI_USER}" +git config user.email "${GIT_CI_EMAIL}" + +mv ../${VERSION} $VERSION + +git remote add upstream "https://${GH_TOKEN}@${GH_REPOSITORY}" +git fetch --depth 50 upstream ${GH_DOC_BRANCH} gh-pages +git reset upstream/gh-pages + +# for dev, latest, home redirects +mkdir dev latest +export URL="https://docs.mdanalysis.org" +python ${MAINTAIN_DIR}/update_json_stubs_sitemap.py +touch . +touch .nojekyll + +git add -A ${VERSION}/ +git add .nojekyll versions.json +git add index.html dev latest +git add *.xml + +# check for anything to commit +# https://stackoverflow.com/questions/3878624/how-do-i-programmatically-determine-if-there-are-uncommited-changes +git diff-index --quiet HEAD -- || git commit -m "rebuilt html docs for version ${VERSION} from branch ${GH_DOC_BRANCH} with sphinx at ${rev}" +git push -q upstream HEAD:gh-pages + + diff --git a/maintainer/update_json_stubs_sitemap.py b/maintainer/update_json_stubs_sitemap.py index f6b1436abc4..a699a0729a5 100644 --- a/maintainer/update_json_stubs_sitemap.py +++ b/maintainer/update_json_stubs_sitemap.py @@ -13,10 +13,6 @@ import os import shutil import xml.etree.ElementTree as ET -import errno -import glob -import textwrap -import shutil try: from urllib.request import Request, urlopen @@ -26,16 +22,6 @@ URL = os.environ['URL'] VERSION = os.environ['VERSION'] -if "http" not in URL: - raise ValueError("URL should have the transfer protocol (HTTP/S). " - f"Given: $URL={URL}") - -try: - int(VERSION[0]) -except ValueError: - raise ValueError("$VERSION should start with a number. " - f"Given: $VERSION={VERSION}") from None - def get_web_file(filename, callback, default): url = os.path.join(URL, filename) @@ -54,30 +40,14 @@ def get_web_file(filename, callback, default): return callback(data) -def write_redirect(file, version='', outfile=None): - if outfile is None: - outfile = file - url = os.path.join(URL, version, file) - REDIRECT = textwrap.dedent(f""" - - - Redirecting to {url} - - - """) - with open(outfile, 'w') as f: - f.write(REDIRECT) - print(f"Wrote redirect from {url} to {outfile}") - - # ========= WRITE JSON ========= # Update $root/versions.json with links to the right version versions = get_web_file('versions.json', json.loads, []) existing = [item['version'] for item in versions] already_exists = VERSION in existing -latest = 'dev' not in VERSION if not already_exists: + latest = 'dev' not in VERSION if latest: for ver in versions: ver['latest'] = False @@ -89,112 +59,52 @@ def write_redirect(file, version='', outfile=None): 'latest': latest }) +with open("versions.json", 'w') as f: + json.dump(versions, f, indent=2) + +# ========= WRITE HTML STUBS ========= +# Add HTML files to redirect: +# index.html -> latest release +# latest/index.html -> latest release +# dev/index.html -> dev docs + +REDIRECT = """ + + +Redirecting to {url} + + +""" + for ver in versions[::-1]: if ver['latest']: - latest_version = ver['version'] - break + latest_url = ver['url'] else: try: - latest_version = versions[-1]['version'] + latest_url = versions[-1]['url'] except IndexError: - latest_version = None + latest_url = None for ver in versions[::-1]: - if '-dev' in ver['version']: - dev_version = ver['version'] + if 'dev' in ver['version']: + dev_url = ver['url'] break else: try: - dev_version = versions[-1]['version'] + dev_url = versions[-1]['url'] except IndexError: - dev_version = None + dev_url = None -versions.sort(key=lambda x: x["version"]) +if latest_url: + with open('index.html', 'w') as f: + f.write(REDIRECT.format(url=latest_url)) -# ========= WRITE HTML STUBS AND COPY DOCS ========= -# Add HTML files to redirect: -# index.html -> stable/ docs -# latest/index.html -> latest release (not dev docs) -# stable/ : a copy of the release docs with the highest number (2.0.0 instead of 1.0.0) -# dev/ : a copy of the develop docs with the highest number (2.0.0-dev instead of 1.0.1-dev) -# sitemap.xml files are updated by replacing URL strings - - -def redirect_sitemap(old_version, new_version): - """Replace paths in copied sitemap.xml with new directory path - - Sitemaps can only contain URLs 'within' that directory structure. - For more, see https://www.sitemaps.org/faq.html#faq_sitemap_location - """ - file = f"{new_version}/sitemap.xml" - old = f"{URL}/{old_version}/" - new = f"{URL}/{new_version}/" - try: - with open(file, "r") as f: - contents = f.read() - except OSError: - raise ValueError(f"{file} not found") - redirected = contents.replace(old, new) - with open(file, "w") as f: - f.write(redirected) - print(f"Redirected URLs in {file} from {old} to {new}") - - -def add_or_update_version(version): - """Add or update the version path to versions.json""" - for ver in versions: - if ver["version"] == version: - ver["url"] = os.path.join(URL, version) - break - else: - versions.append({ - "version": version, - "display": version, - "url": os.path.join(URL, version), - "latest": False - }) - - -def copy_version(old_version, new_version): - """Copy docs from one directory to another with all bells and whistles""" - shutil.copytree(old_version, new_version) - print(f"Copied {old_version} to {new_version}") - redirect_sitemap(old_version, new_version) - add_or_update_version(new_version) - - -# Copy stable/ docs and write redirects from root level docs -if latest: - copy_version(VERSION, "stable") - html_files = glob.glob(f'stable/**/*.html', recursive=True) - for file in html_files: - # below should be true because we only globbed stable/* paths - assert file.startswith("stable/") - outfile = file[7:] # strip "stable/" - dirname = os.path.dirname(outfile) - if dirname and not os.path.exists(dirname): - try: - os.makedirs(dirname) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - - write_redirect(file, '', outfile) - -# Separate just in case we update versions.json or muck around manually -# with docs -if latest_version: - write_redirect('index.html', "stable") - write_redirect('index.html', latest_version, 'latest/index.html') - -# Copy dev/ docs -if dev_version and dev_version == VERSION: - copy_version(VERSION, "dev") - -# update versions.json online -with open("versions.json", 'w') as f: - json.dump(versions, f, indent=2) + with open('latest/index.html', 'w') as f: + f.write(REDIRECT.format(url=latest_url)) +if dev_url: + with open('dev/index.html', 'w') as f: + f.write(REDIRECT.format(url=dev_url)) # ========= WRITE SUPER SITEMAP.XML ========= # make one big sitemap.xml diff --git a/package/doc/README b/package/doc/README index 432b0db8292..60b66f94ae4 100644 --- a/package/doc/README +++ b/package/doc/README @@ -39,7 +39,7 @@ The documentation for the latest stable release can always be found at The docs for the latest development version are also on the internet at - https://docs.mdanalysis.org/dev + http://docs.mdanalysis.org/dev The manual includes all the doc strings with some additional text; it is a work in progress and suggestions to improve it are welcome. File diff --git a/package/doc/sphinx/source/conf.py b/package/doc/sphinx/source/conf.py index efa62c1ce47..141af51dd6f 100644 --- a/package/doc/sphinx/source/conf.py +++ b/package/doc/sphinx/source/conf.py @@ -15,7 +15,6 @@ import os import platform import datetime -import MDAnalysis as mda import msmb_theme # for little versions pop-up # https://sphinx-rtd-theme.readthedocs.io/en/stable/ import sphinx_rtd_theme diff --git a/package/setup.py b/package/setup.py index 10da7e71371..74f29bd449a 100755 --- a/package/setup.py +++ b/package/setup.py @@ -159,7 +159,7 @@ class MDAExtension(Extension, object): # care of calling it when needed. def __init__(self, name, sources, *args, **kwargs): self._mda_include_dirs = [] - # don't abspath sources else packaging fails on Windows (issue #3129) + sources = [abspath(s) for s in sources] super(MDAExtension, self).__init__(name, sources, *args, **kwargs) @property @@ -548,14 +548,8 @@ def long_description(readme): except (OSError, IOError): warnings.warn('Cannot write the list of authors.') - try: - # when building from repository for creating the distribution - LONG_DESCRIPTION = long_description("pypi-description.rst") - except OSError: - # when building from a tar file for installation - # (LONG_DESCRIPTION is not really needed) - LONG_DESCRIPTION = "MDAnalysis -- https://www.mdanalysis.org/" - + with open(abspath('SUMMARY.txt')) as summary: + LONG_DESCRIPTION = summary.read() CLASSIFIERS = [ 'Development Status :: 6 - Mature', 'Environment :: Console', diff --git a/testsuite/README b/testsuite/README index a9379c68b2a..4c62bd799df 100644 --- a/testsuite/README +++ b/testsuite/README @@ -53,8 +53,8 @@ For questions and discussions for code development, join the developer list http .. |devdocs| image:: https://img.shields.io/badge/docs-development-yellow.svg :alt: Documentation (development version) - :target: https://docs.mdanalysis.org/dev - + :target: https://docs.mdanalysis.org/dev/ + .. |developergroup| image:: https://img.shields.io/badge/Google%20Group-Developers-lightgrey.svg :alt: Developer Google Group :target: https://groups.google.com/group/mdnalysis-devel