Add Travis Caching and separate scripts for CI (#866)

* add caching * remove cached miniconda * swap coveralls for codecov * adding new ci helper * caching tardis env * add refdata to caching * fetch reference data * ref data avail * remove the fetch reference data caching - to be done in next PR * update ref_data_home * update comments from unoebauer * update refdata path * add single quotes * remove git lfs from separate install * update clone refdata * remove space after equal * update travis yaml
tardis-sn · Aug 30, 2018 · 0e8dafb · 0e8dafb
1 parent 8c367dc
commit 0e8dafb
Show file tree

Hide file tree

Showing 5 changed files with 80 additions and 41 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -8,8 +8,11 @@ python:
 virtualenv:
   system_site_packages: true
 
-cache:
-  apt: true
+addons:
+  apt:
+    packages:
+    - clang
+    - gdb
 
 sudo: true
 
@@ -20,61 +23,52 @@ env:
         - ASTROPY_USE_SYSTEM_PYTEST=1
         - SETUP_CMD='test'
         - TEST_MODE='normal'
-        - TARDIS_REF_DATA_URL='https://github.com/tardis-sn/tardis-refdata.git'
+        - REF_DATA_HOME=$HOME/tardis-refdata
+        - REF_DATA_GITHUBURL='https://github.com/tardis-sn/tardis-refdata.git'
         - MINICONDA_URL='http://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh'
         - SAVE_COVERAGE=false
+        - GIT_LFS_SKIP_SMUDGE=1
 
 matrix:
     include:
         - python: 2.7
           env:
             - COMPILER=gcc
-            - SETUP_CMD='test --coverage --args="--tardis-refdata=$HOME/tardis-refdata/"'
+            - SETUP_CMD='test --coverage --args="--tardis-refdata=$REF_DATA_HOME"'
             - TEST_MODE='spectrum'
             - SAVE_COVERAGE=true
 
         - python: 2.7
           env:
             - COMPILER=clang
-            - SETUP_CMD='test --args="--tardis-refdata=$HOME/tardis-refdata/"'
+            - SETUP_CMD='test --args="--tardis-refdata=$REF_DATA_HOME"'
             - TEST_MODE='spectrum'
 
 #trouble with osx building due to segfault at cython (https://github.com/cython/cython/issues/2199)
         - os: osx
           language: generic
           env:
             - COMPILER=clang
-            - SETUP_CMD='test --args="--tardis-refdata=$HOME/tardis-refdata/"'
+            - SETUP_CMD='test --args="--tardis-refdata=$REF_DATA_HOME"'
             - TEST_MODE='spectrum'
             - MINICONDA_URL='http://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh'
 
-
-        - python: 2.7
-          env: SETUP_CMD='test'
-
-addons:
-  apt:
-    packages:
-    - clang
-    - gdb
-
-
+cache:
+  apt: true
+  directories:
+    - $HOME/miniconda
+    #- $REF_DATA_HOME
 
 
 before_install:
     # We do this to make sure we get the dependencies so pip works below
-    - export PYTHONIOENCODING=UTF8
-    - wget $MINICONDA_URL -O miniconda.sh
-    - chmod +x miniconda.sh
-    - bash miniconda.sh -b -p $HOME/miniconda
-    - export PATH=$HOME/miniconda/bin:$PATH
-    - hash -r
-    - conda update --yes conda
-    - if [[ $TEST_MODE == 'spectrum' ]]; then conda install -c conda-forge git-lfs=2.2.1 -y; fi
+    - source ci-helpers/install_miniconda.sh
+    #- if [[ $TEST_MODE == 'spectrum' ]]; then conda install -c conda-forge git-lfs=2.2.1 -y; fi
     - if [[ $TEST_MODE == 'spectrum' ]]; then git lfs install --skip-smudge; fi
-    - if [[ $TEST_MODE == 'spectrum' ]]; then git clone $TARDIS_REF_DATA_URL $HOME/tardis-refdata; fi
-    - if [[ $TEST_MODE == 'spectrum' ]]; then cd $HOME/tardis-refdata; fi
-    # Use the following to get the ref-data from the master; 
+    - echo git clone $REF_DATA_GITHUBURL $REF_DATA_HOME
+    - if [[ $TEST_MODE == 'spectrum' ]]; then git clone $REF_DATA_GITHUBURL $REF_DATA_HOME; fi
+    - if [[ $TEST_MODE == 'spectrum' ]]; then cd $REF_DATA_HOME; fi
+# Use the following to get the ref-data from the master;
     - if [[ $TEST_MODE == 'spectrum' ]]; then git fetch origin; fi
     - if [[ $TEST_MODE == 'spectrum' ]]; then git checkout origin/master; fi
     # Use the following to get the ref-data from a specific pull request; 
@@ -86,19 +80,11 @@ before_install:
     - if [[ $TEST_MODE == 'spectrum' ]]; then echo MD5 `md5sum unit_test_data.h5`; fi
     - if [[ $TEST_MODE == 'spectrum' ]]; then cd $TRAVIS_BUILD_DIR; fi
 
+#- source ci-helpers/fetch_reference_data.sh
+
 
 install:
-   - cd $TRAVIS_BUILD_DIR
-   - conda env create -f tardis_env27.yml
-   - source activate tardis
-   - conda install -y -c conda-forge codecov
-   #trouble with building due to segfault at cython (https://github.com/cython/cython/issues/2199)
-   #remove if we can get normal cython through conda
-   - git clone https://github.com/cython/cython
-   - cd cython
-   - git checkout c485b1b77264c3c75d090a3c526de24966830d42
-   - CFLAGS="$CFLAGS -D CYTHON_CLINE_IN_TRACEBACK=0" python setup.py install
-   - cd ..
+   - source ci-helpers/install_tardis_env.sh
 
 script:
     - echo CC=$COMPILER python setup.py $SETUP_CMD

diff --git a/ci-helpers/fetch_reference_data.sh b/ci-helpers/fetch_reference_data.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+git lfs install
+if test -e $REF_DATA_HOME; then
+    echo "Ref data available"
+else
+    git clone https://github.com/tardis-sn/tardis-refdata.git $REF_DATA_HOME
+cd $REF_DATA_HOME
+# Use the following to get the ref-data from the master;
+git fetch origin
+git checkout origin/master
+# Use the following to get the ref-data from a specific pull request
+#git fetch origin pull/11/head:thomson-ref
+git lfs pull --include="atom_data/kurucz_cd23_chianti_H_He.h5" origin/master
+git lfs pull --include="atom_data/chianti_He.h5" origin/master
+git lfs pull --include="plasma_reference/" origin/master
+git lfs pull --include="unit_test_data.h5" origin/master
+cd $TRAVIS_BUILD_DIR
diff --git a/ci-helpers/install_miniconda.sh b/ci-helpers/install_miniconda.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+export PYTHONIOENCODING=UTF8
+if test -e $HOME/miniconda/bin; then
+    echo "miniconda already installed.";
+    export PATH=$HOME/miniconda/bin:$PATH
+    hash -r
+    #conda update --yes conda
+
+else
+    wget $MINICONDA_URL -O miniconda.sh
+    chmod +x miniconda.sh
+    rm -r $HOME/miniconda
+    bash miniconda.sh -b -p $HOME/miniconda
+
+    export PATH=$HOME/miniconda/bin:$PATH
+    hash -r
+    conda update --yes conda
+fi
diff --git a/ci-helpers/install_tardis_env.sh b/ci-helpers/install_tardis_env.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+cd $TRAVIS_BUILD_DIR
+if test -e $HOME/miniconda/envs/tardis; then
+    echo "TARDIS env already installed.";
+    # Also check for tardis_env27.yml change
+else
+   conda env create -f tardis_env27.yml
+   #trouble with building due to segfault at cython (https://github.com/cython/cython/issues/2199)
+   #remove if we can get normal cython through conda
+   git clone https://github.com/cython/cython
+   cd cython
+   git checkout c485b1b77264c3c75d090a3c526de24966830d42
+   CFLAGS="$CFLAGS -D CYTHON_CLINE_IN_TRACEBACK=0" python setup.py install
+   cd ..
+fi
+
+source activate tardis
diff --git a/tardis_env27.yml b/tardis_env27.yml
@@ -34,12 +34,13 @@ dependencies:
 - requests=2.9.1
 - docopt=0.6.2
 - pytest-cov=2.2.1
-
+- codecov
+#testing requirements
+- git-lfs=2.4
 - pip:
   - sphinx_bootstrap_theme
   - sphinxcontrib-bibtex
   - sphinxcontrib-tikz
-  - coveralls
   - pytest-html==1.10.0
   - dokuwiki
   - dot2tex