Skip to content

fitgrid

fitgrid #423

Workflow file for this run

# Continous integration and deployment (CID) workflow
#
# T. Urbach 12/2020
#
# - based on spudtr-cid.yml
# - canonical package semantic version strings are M.N.P and M.N.P.devX
# - job matrix runs on linux-64, osx-64 for Python 3.7, 3.8
# - each linux, osx py3X job builds, installs, pytests, and deploys its own conda py3X packages
# - one py3X job deploys codecov and docs
# - cron runs daily to conda build, install, pytest the latest commit on default branch
#
# CI:
#
# - conda build, install, and pytest py3[78] x [linux-64, osx-64] tarballs
# - conda convert py3[78] linux-64 -> win-64 (untested)
# - build sphinx docs
#
# Deployment:
#
# Development: package version M.N.P.devX on dev branch triggers
# - anaconda upload py3X platform tarballs to channel label /pre-release
# - twine python build/sdist to test.pypi.org
#
# Stable release: version M.N.P on release tagged vM.N.P triggers
# - anaconda upload py3X platform tarballs to channel label /main
# - codecov
# - sphinx docs to gh-pages
# - twine python build/sdist to pypi.org
#
# Notes:
#
# * Conda env wrangling. To run conda activate in bash shells, the conda
# shell.bash hook must be sourced. The usual "conda init bash"
# installs the hook in .bashrc which isn't consulted (or exits
# early) for the Actions non-interactive shells. One option is to
# re-install the hook each time it is needed in a run step like so:
#
# - eval "$(conda shell.bash hook)"
# - conda activate myenv
#
# The approach here is to install the hook in .profile_bash one time
# in the conda setup and run bash as a non-interactive login shell.
#
# * Github releases. Github published releases generate a release and a
# push event. Pushed stable release tags are ignored to prevent
# double workflow runs.
#
# * Deprecated Python pacakge uploads to test.pypi.org and pypi.org in
# version 0.5.1.dev5.
name: fitgrid
on:
push:
branches: ['*']
tags-ignore:
- v[0-9].[0-9].[0-9] # prevent double runs for vM.N.P releases
pull_request:
release:
types: [published]
schedule:
# min, hour, day, month, day of week. digit = when, / = step by, * = all
# midnight Sunday
- cron: '0 0 * * 0'
env:
PACKAGE_NAME: fitgrid
DEPLOY_PY_VER: 3.8 # only this job deploys codecov, docs, sdist
DEPLOY_OS: ubuntu-latest
defaults:
run:
# login shell to source the conda hook in .bash_profile
shell:
bash -l {0}
jobs:
fitgrid-cid:
runs-on: ${{ matrix.os }} # ubuntu-latest
strategy:
matrix:
py_ver: [3.7, 3.8, 3.9]
os: [ubuntu-latest, macos-10.15] # Intel macs
env:
PY_VER: ${{ matrix.py_ver }}
CONDA_BLD_PATH: /tmp/ci_conda_bld
outputs:
# tarballs are py3X job-specific
conda-tarball: ${{ steps.conda-bld.outputs.conda-tarball }}
steps:
# ------------------------------------------------------------
# 1. Install
- uses: actions/checkout@v2 # fetch the current commit, verified action
- name: conda setup
run: |
# short SHA gets baked into the conda package filename in conda/meta.yaml
echo "GIT_ABBREV_COMMIT=_g${GITHUB_SHA:0:8}" >> $GITHUB_ENV
if [[ ${{ runner.os }} == Linux ]]; then \
miniconda_url='https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh'; \
fi
if [[ ${{ runner.os }} == macOS ]]; then \
miniconda_url='https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh'; \
fi
wget $miniconda_url -O $HOME/miniconda.sh
bash ~/miniconda.sh -b -p $HOME/miniconda
hash -r
$HOME/miniconda/bin/conda shell.bash hook >> ~/.bash_profile # instead of conda init bash
source ~/.bash_profile
hash -r
conda config --set always_yes yes --set changeps1 no
conda config --set bld_path $CONDA_BLD_PATH
# switch to mamba/boa and conda-forge
conda config --add channels conda-forge
# revised mamba channel priority semantics, strict in 0.14 is flexible in 0.15
conda config --set channel_priority flexible # should be default but enforce
conda install -n base -q conda-build conda-verify anaconda-client mamba boa
echo "# ------------------------------------------------------------"
conda info -a
echo "# ------------------------------------------------------------"
mamba info -a
# ------------------------------------------------------------
# 2. CI
- name: conda mambabuild --python=${{ env.PY_VER }} conda
id: conda-bld
env:
OS: ${{ runner.os }} # Linux, macOS
run: |
# conda-forge added in setup
conda mambabuild --python=$PY_VER -c ejolly conda
# filename is last line when conda build is chatty
tarball=$(conda build --python=$PY_VER conda --output | tail -1)
# atrocious hack, only linux job converts to win-64 for upload
if [[ $OS == "Linux" ]]; then \
conda convert -p win-64 -o $CONDA_BLD_PATH $tarball; \
fi
echo "conda build tarball" $tarball
echo "::set-output name=conda-tarball::$tarball"
mamba search fitgrid -c $CONDA_BLD_PATH
# create an env w/ local package
- name: conda create env_${{ env.PY_VER }} python=${{ env.PY_VER }} ${{ env.PACKAGE_NAME}}
run: |
mamba create -n env_$PY_VER python=$PY_VER $PACKAGE_NAME "blas=*=mkl*" -c $CONDA_BLD_PATH -c ejolly
- name: black, pytest, codecov
run: |
conda activate env_$PY_VER
mamba install -q black pytest pytest-cov codecov
black --check --verbose -S -l 79 .
conda list
lscpu
python -c 'import numpy; numpy.show_config()'
pytest --cov=$PACKAGE_NAME && codecov # test as installed by mamba
- name: build sphinx docs
if: ${{ matrix.py_ver == env.DEPLOY_PY_VER && matrix.os == env.DEPLOY_OS }}
run: |
conda activate env_$PY_VER # required to run doc demos
# conda install -q sphinx sphinx_rtd_theme jupyter nbsphinx "nbconvert!=5.4" -c defaults -c conda-forge
# conda install -q pandoc -c conda-forge
mamba install -q sphinx sphinx_rtd_theme sphinx-gallery pandoc # -c defaults -c conda-forge
make -C docs html && touch docs/build/html/.nojekyll
- name: build python package sdist
if: ${{ matrix.py_ver == env.DEPLOY_PY_VER }}
run: python setup.py sdist
# ------------------------------------------------------------
# 3. Deploy
#
# DEPLOY_TYPE selects conda, python package, and docs uploads
#
# for regex matching
# . canonical version strings are M.N.P.devX and M.N.P
# . tarballs are $CONDA_BLD_PATH/linux-64/PACKAGE_NAME-.+-py3[678]_g[a-z0-9]{8}.tar.bz2
# . pkg_version is entire M.N.P.devX version string
# . pkg_mnp is M.N.P only
- name: set DEPLOY_TYPE=<pre-release|main|no_deploy>
env:
OS: ${{ runner.os }} # Linux, macOS
run: |
# lookup the conda tarball package version and M.N.P string (if any)
tarball=${{ steps.conda-bld.outputs.conda-tarball }}
pkg_version=`echo $tarball | sed -n "s/.*${PACKAGE_NAME}-\(.*\)-.*/\1/p"`
pkg_mnp=`echo $pkg_version | sed -n "s/\([0-9]*\.[0-9]*\.[0-9]\).*/\1/p"`
# pre-release if package version is M.N.P.devX on dev branch
if [[ \
$GITHUB_REF =~ ^refs/heads/dev$ && \
$pkg_version =~ ^([0-9]+\.){2}[0-9]+(\.dev[0-9]+){0,1}$ \
]]; then \
deploy_type="pre-release"; \
fi
# main if package version is M.N.P on release tag vM.N.P
if [[ \
$GITHUB_REF =~ ^refs/tags/v([0-9]+\.){2}[0-9]+$ && \
$pkg_version == $pkg_mnp \
]]; then \
deploy_type="main"; \
fi
# else, default
deploy_type=${deploy_type:-no_deploy}
# set top-level env for uploads
echo "PKG_VERSION=${pkg_version}" >> $GITHUB_ENV
echo "PKG_MNP=${pkg_mnp}" >> $GITHUB_ENV
echo "DEPLOY_TYPE=${deploy_type}" >> $GITHUB_ENV
# logging
echo $PACKAGE_NAME $deploy_type $tarball $pkg_version $pkg_mnp
- name: >-
show ${{ env.PACKAGE_NAME }}
${{ env.PKG_VERSION }}
env_${{ env.PY_VER }}
deploy_type=${{ env.DEPLOY_TYPE }}
run: |
conda activate env_$PY_VER
printenv | sort
conda list --explicit
# linux and osx jobs built their own tarballs; linux-64 conda converts to win-64
- name: deploy conda package python ${{ matrix.py_ver}} ${{ env.DEPLOY_TYPE }}
if: ${{ env.DEPLOY_TYPE == 'pre-release' || env.DEPLOY_TYPE == 'main' }}
env:
ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
run: >-
anaconda
-t "$ANACONDA_TOKEN"
upload $CONDA_BLD_PATH/**/${PACKAGE_NAME}*.tar.bz2
-l $DEPLOY_TYPE
--skip-existing
- name: deploy sphinx docs to gh-pages
# switch docs deployment repo/branch on deploy type
# if: ${{ matrix.py_ver == env.DEPLOY_PY_VER && matrix.os == env.DEPLOY_OS }} # testing only
if: >-
${{
matrix.py_ver == env.DEPLOY_PY_VER
&& matrix.os == env.DEPLOY_OS
&& (env.DEPLOY_TYPE == 'main' || env.DEPLOY_TYPE == 'pre-release')
}}
env:
# machine user account PAT
GH_USER_NAME: robo-kutaslab
GH_USER_EMAIL: robo.kutaslab@gmail.com
GH_PAGES_TOKEN: ${{ secrets.GH_PAGES_TOKEN }}
# vM.N.P tagged release docs are pushed here
GH_PAGES_BRANCH: gh-pages
# actions on dev branch push docs to this separate repo
GH_PAGES_DEV_REPOSITORY: kutaslab/fitgrid-dev-docs
GH_PAGES_DEV_BRANCH: gh-pages-dev
run: |
# set docs version repo/branch
if [[ ${{ env.DEPLOY_TYPE }} == main ]]; then \
DOCS_REPO=$GITHUB_REPOSITORY; \
DOCS_BRANCH=$GH_PAGES_BRANCH; \
fi
if [[ ${{ env.DEPLOY_TYPE }} == pre-release ]]; then \
DOCS_REPO=$GH_PAGES_DEV_REPOSITORY; \
DOCS_BRANCH=$GH_PAGES_DEV_BRANCH; \
fi
echo "deploy type ${{ env.DEPLOY_TYPE }}: sphinx docs on $DOCS_REPO/$DOCS_BRANCH"
git config --global user.name $GH_USER_NAME
git config --global user.email $GH_USER_EMAIL
git config --global init.defaultBranch main
# wherever sphinx wrote the html, repo specific
cd $GITHUB_WORKSPACE/docs/build/html
git init
git remote add origin "https://$GH_PAGES_TOKEN@github.com/${DOCS_REPO}"
git checkout --orphan $DOCS_BRANCH
git add -A
git commit -a -m "deploy type $DEPLOY_TYPE python $DEPLOY_PY_VER"
git push -u origin $DOCS_BRANCH --force