Update Travis CI build and run/lint/verify examples (#508)

* Add CLI to run/lint/verify notebook examples * Remove unused imports and fix broken URLs * Fix dependencies * Refactor Travis CI setup into separate scripts, based on Bokeh's CI setup.
holoviz · Oct 23, 2017 · 95ff99f · 95ff99f
1 parent d7311e5
commit 95ff99f
Show file tree

Hide file tree

Showing 10 changed files with 363 additions and 62 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -4,55 +4,65 @@
 # https://github.com/travis-ci/travis-ci/issues/4729).
 language: generic
 
-os:
-  - linux
-  - osx
-
-# get a more recent version of osx than the default image
-osx_image: xcode8.3
-
 sudo: false
 
-env:
-  - PYTHON_VERSION="2.7"
-  - PYTHON_VERSION="3.5"
-  - PYTHON_VERSION="3.6"
-
-install:
-  # Install conda
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh;
-    else
-      wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
-    fi  
-  - bash miniconda.sh -b -p $HOME/miniconda
+before_install:
   - export PATH="$HOME/miniconda/bin:$PATH"
-  - conda config --set always_yes yes --set changeps1 no
-  - conda update conda
-
-  # Install dependencies
-  - conda create -n test-environment python=$PYTHON_VERSION
-  - source activate test-environment
-  - conda install dask numba numpy pandas pillow toolz xarray datashape colorcet rasterio scikit-image param
-  - conda install -c gbrener -c conda-forge xarray # remove this line when xarray >= 0.9.7
-  # Optional dependencies, for testing only
-  - conda install -c conda-forge pytest pytest-benchmark
-  - conda install matplotlib
-  # Dependencies for lint checking only
-  - conda install jupyter flake8
-
-  - python setup.py develop --no-deps
-  - conda env export
-
 
-script:
-    # TODO: why different tests for python 2 vs 3?
-    - if [[ $PYTHON_VERSION == '2.7' ]]; then py.test datashader/tests --doctest-modules --doctest-ignore-import-errors --verbose; else py.test datashader/tests --verbose; fi
-    - flake8 --ignore E,W datashader examples
-    # lint checking of example notebooks (ugly)
-    - for NB in examples/*.ipynb; do sed -e 's/"[ ]*%/"#%/' "$NB" > "${NB}~"; done && jupyter nbconvert examples/*.ipynb~ --to script --output-dir examples/lint && flake8 --ignore=E,W examples/lint/*.py
-    # benchmarks
-    - py.test datashader/benchmarks
+jobs:
+  include:
+    - install: scripts/ci/install_examples
+      script: scripts/ci/test_examples
+      os: linux
+      env:
+      - GROUP=examples
+      - PYTHON_VERSION=3.6
+    - install: scripts/ci/install_unit
+      script: scripts/ci/test_unit
+      os: linux
+      env:
+      - GROUP=unit
+      - PYTHON_VERSION=2.7
+    - install: scripts/ci/install_unit
+      script: scripts/ci/test_unit
+      os: linux
+      env:
+      - GROUP=unit
+      - PYTHON_VERSION=3.5
+    - install: scripts/ci/install_unit
+      script: scripts/ci/test_unit
+      os: linux
+      env:
+      - GROUP=unit
+      - PYTHON_VERSION=3.6
+    - install: scripts/ci/install_examples
+      script: scripts/ci/test_examples
+      os: osx
+      osx_image: xcode8.3
+      env:
+      - GROUP=examples
+      - PYTHON_VERSION=3.6
+    - install: scripts/ci/install_unit
+      script: scripts/ci/test_unit
+      os: osx
+      osx_image: xcode8.3
+      env:
+      - GROUP=unit
+      - PYTHON_VERSION=2.7
+    - install: scripts/ci/install_unit
+      script: scripts/ci/test_unit
+      os: osx
+      osx_image: xcode8.3
+      env:
+      - GROUP=unit
+      - PYTHON_VERSION=3.5
+    - install: scripts/ci/install_unit
+      script: scripts/ci/test_unit
+      os: osx
+      osx_image: xcode8.3
+      env:
+      - GROUP=unit
+      - PYTHON_VERSION=3.6
 
 notifications:
   email: false
diff --git a/examples/edge_bundling.ipynb b/examples/edge_bundling.ipynb
@@ -25,13 +25,11 @@
     "import pandas as pd\n",
     "import holoviews as hv\n",
     "import fastparquet as fp\n",
-    "import datashader as ds\n",
     "\n",
     "from colorcet import fire\n",
     "from datashader.bundling import directly_connect_edges, hammer_bundle\n",
-    "from datashader.utils import export_image\n",
     "\n",
-    "from holoviews.operation.datashader import aggregate, shade, datashade, dynspread\n",
+    "from holoviews.operation.datashader import datashade, dynspread\n",
     "from holoviews.operation import decimate\n",
     "\n",
     "from dask.distributed import Client\n",
@@ -214,7 +212,7 @@
    "source": [
     "## Using graphs from NetworkX\n",
     "\n",
-    "The above examples constructed networks by hand.  A convenient way to get access to a large number of  [graph types](https://networkx.readthedocs.io/en/stable/reference/generators.html) is the separate [NetworkX](https://networkx.readthedocs.io) package.  Here, we will select several standard graph structures, lay them each out in the same fixed circular shape using NetworkX, and then show how they will appear without bundling, with moderate levels of bundling, and with high amounts of bundling."
+    "The above examples constructed networks by hand.  A convenient way to get access to a large number of [graph types](https://networkx.github.io/documentation/stable/reference/generators.html) is the separate [NetworkX](https://networkx.readthedocs.io) package.  Here, we will select several standard graph structures, lay them each out in the same fixed circular shape using NetworkX, and then show how they will appear without bundling, with moderate levels of bundling, and with high amounts of bundling."
    ]
   },
   {
@@ -393,7 +391,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "If you click around and hover, you should see interesting groups of nodes, and can then set up further interactive tools using [HoloViews' stream support](http://holoviews.org/Tutorials/Streams.html) to reveal aspects relevant to your research interests or questions.\n",
+    "If you click around and hover, you should see interesting groups of nodes, and can then set up further interactive tools using [HoloViews' stream support](http://holoviews.org/user_guide/Responding_to_Events.html) to reveal aspects relevant to your research interests or questions.\n",
     "\n",
     "As you can see, datashader lets you work with very large graph datasets, though there are a number of decisions to make by trial and error, you do have to be careful when doing computationally expensive operations like edge bundling, and interactive information will only be available for a limited subset of the data at any one time due to data-size limitations of current web browsers."
    ]

diff --git a/examples/environment.yml b/examples/environment.yml
@@ -6,19 +6,31 @@ channels:
 
 dependencies:
  - attrs
+ - beautifulsoup4
  - bokeh
  - cartopy
  - colorcet
+ - conda-forge::graphviz
+ - conda-forge::pytest
+ - conda-forge::pytest-benchmark
+ - conda-forge::python-graphviz
  - dask
  - datashader
  - dill
  - distributed
+ - fastparquet
+ - flake8
+ - gbrener::xarray>=0.9.6 # For xarray.open_rasterio()
  - geoviews
  - ioam::holoviews>=1.8.3
+ - ipython
  - iris
  - jupyter
+ - jupyter_dashboards
  - krb5
  - matplotlib
+ - nbconvert
+ - nbformat
  - networkx>=2.0
  - numba
  - numpy
@@ -27,25 +39,18 @@ dependencies:
  - paramnb
  - pyproj
  - pytables
- - conda-forge::pytest
- - conda-forge::pytest-benchmark
+ - python-snappy
  - python=3.6
  - rasterio
  - requests
  - scikit-image
  - scipy
  - shapely
+ - snappy
  - statsmodels
+ - streamz==0.1.0
  - tblib
- - gbrener::xarray>=0.9.6 # For xarray.open_rasterio()
  - yaml
- - fastparquet
- - snappy
- - python-snappy
- - jupyter_dashboards
- - streamz==0.1.0
- - conda-forge::graphviz
- - conda-forge::python-graphviz
  - pip:
    - cachey
    - webargs