fitgrid #423
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Continous integration and deployment (CID) workflow | |
| # | |
| # T. Urbach 12/2020 | |
| # | |
| # - based on spudtr-cid.yml | |
| # - canonical package semantic version strings are M.N.P and M.N.P.devX | |
| # - job matrix runs on linux-64, osx-64 for Python 3.7, 3.8 | |
| # - each linux, osx py3X job builds, installs, pytests, and deploys its own conda py3X packages | |
| # - one py3X job deploys codecov and docs | |
| # - cron runs daily to conda build, install, pytest the latest commit on default branch | |
| # | |
| # CI: | |
| # | |
| # - conda build, install, and pytest py3[78] x [linux-64, osx-64] tarballs | |
| # - conda convert py3[78] linux-64 -> win-64 (untested) | |
| # - build sphinx docs | |
| # | |
| # Deployment: | |
| # | |
| # Development: package version M.N.P.devX on dev branch triggers | |
| # - anaconda upload py3X platform tarballs to channel label /pre-release | |
| # - twine python build/sdist to test.pypi.org | |
| # | |
| # Stable release: version M.N.P on release tagged vM.N.P triggers | |
| # - anaconda upload py3X platform tarballs to channel label /main | |
| # - codecov | |
| # - sphinx docs to gh-pages | |
| # - twine python build/sdist to pypi.org | |
| # | |
| # Notes: | |
| # | |
| # * Conda env wrangling. To run conda activate in bash shells, the conda | |
| # shell.bash hook must be sourced. The usual "conda init bash" | |
| # installs the hook in .bashrc which isn't consulted (or exits | |
| # early) for the Actions non-interactive shells. One option is to | |
| # re-install the hook each time it is needed in a run step like so: | |
| # | |
| # - eval "$(conda shell.bash hook)" | |
| # - conda activate myenv | |
| # | |
| # The approach here is to install the hook in .profile_bash one time | |
| # in the conda setup and run bash as a non-interactive login shell. | |
| # | |
| # * Github releases. Github published releases generate a release and a | |
| # push event. Pushed stable release tags are ignored to prevent | |
| # double workflow runs. | |
| # | |
| # * Deprecated Python pacakge uploads to test.pypi.org and pypi.org in | |
| # version 0.5.1.dev5. | |
| name: fitgrid | |
| on: | |
| push: | |
| branches: ['*'] | |
| tags-ignore: | |
| - v[0-9].[0-9].[0-9] # prevent double runs for vM.N.P releases | |
| pull_request: | |
| release: | |
| types: [published] | |
| schedule: | |
| # min, hour, day, month, day of week. digit = when, / = step by, * = all | |
| # midnight Sunday | |
| - cron: '0 0 * * 0' | |
| env: | |
| PACKAGE_NAME: fitgrid | |
| DEPLOY_PY_VER: 3.8 # only this job deploys codecov, docs, sdist | |
| DEPLOY_OS: ubuntu-latest | |
| defaults: | |
| run: | |
| # login shell to source the conda hook in .bash_profile | |
| shell: | |
| bash -l {0} | |
| jobs: | |
| fitgrid-cid: | |
| runs-on: ${{ matrix.os }} # ubuntu-latest | |
| strategy: | |
| matrix: | |
| py_ver: [3.7, 3.8, 3.9] | |
| os: [ubuntu-latest, macos-10.15] # Intel macs | |
| env: | |
| PY_VER: ${{ matrix.py_ver }} | |
| CONDA_BLD_PATH: /tmp/ci_conda_bld | |
| outputs: | |
| # tarballs are py3X job-specific | |
| conda-tarball: ${{ steps.conda-bld.outputs.conda-tarball }} | |
| steps: | |
| # ------------------------------------------------------------ | |
| # 1. Install | |
| - uses: actions/checkout@v2 # fetch the current commit, verified action | |
| - name: conda setup | |
| run: | | |
| # short SHA gets baked into the conda package filename in conda/meta.yaml | |
| echo "GIT_ABBREV_COMMIT=_g${GITHUB_SHA:0:8}" >> $GITHUB_ENV | |
| if [[ ${{ runner.os }} == Linux ]]; then \ | |
| miniconda_url='https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh'; \ | |
| fi | |
| if [[ ${{ runner.os }} == macOS ]]; then \ | |
| miniconda_url='https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh'; \ | |
| fi | |
| wget $miniconda_url -O $HOME/miniconda.sh | |
| bash ~/miniconda.sh -b -p $HOME/miniconda | |
| hash -r | |
| $HOME/miniconda/bin/conda shell.bash hook >> ~/.bash_profile # instead of conda init bash | |
| source ~/.bash_profile | |
| hash -r | |
| conda config --set always_yes yes --set changeps1 no | |
| conda config --set bld_path $CONDA_BLD_PATH | |
| # switch to mamba/boa and conda-forge | |
| conda config --add channels conda-forge | |
| # revised mamba channel priority semantics, strict in 0.14 is flexible in 0.15 | |
| conda config --set channel_priority flexible # should be default but enforce | |
| conda install -n base -q conda-build conda-verify anaconda-client mamba boa | |
| echo "# ------------------------------------------------------------" | |
| conda info -a | |
| echo "# ------------------------------------------------------------" | |
| mamba info -a | |
| # ------------------------------------------------------------ | |
| # 2. CI | |
| - name: conda mambabuild --python=${{ env.PY_VER }} conda | |
| id: conda-bld | |
| env: | |
| OS: ${{ runner.os }} # Linux, macOS | |
| run: | | |
| # conda-forge added in setup | |
| conda mambabuild --python=$PY_VER -c ejolly conda | |
| # filename is last line when conda build is chatty | |
| tarball=$(conda build --python=$PY_VER conda --output | tail -1) | |
| # atrocious hack, only linux job converts to win-64 for upload | |
| if [[ $OS == "Linux" ]]; then \ | |
| conda convert -p win-64 -o $CONDA_BLD_PATH $tarball; \ | |
| fi | |
| echo "conda build tarball" $tarball | |
| echo "::set-output name=conda-tarball::$tarball" | |
| mamba search fitgrid -c $CONDA_BLD_PATH | |
| # create an env w/ local package | |
| - name: conda create env_${{ env.PY_VER }} python=${{ env.PY_VER }} ${{ env.PACKAGE_NAME}} | |
| run: | | |
| mamba create -n env_$PY_VER python=$PY_VER $PACKAGE_NAME "blas=*=mkl*" -c $CONDA_BLD_PATH -c ejolly | |
| - name: black, pytest, codecov | |
| run: | | |
| conda activate env_$PY_VER | |
| mamba install -q black pytest pytest-cov codecov | |
| black --check --verbose -S -l 79 . | |
| conda list | |
| lscpu | |
| python -c 'import numpy; numpy.show_config()' | |
| pytest --cov=$PACKAGE_NAME && codecov # test as installed by mamba | |
| - name: build sphinx docs | |
| if: ${{ matrix.py_ver == env.DEPLOY_PY_VER && matrix.os == env.DEPLOY_OS }} | |
| run: | | |
| conda activate env_$PY_VER # required to run doc demos | |
| # conda install -q sphinx sphinx_rtd_theme jupyter nbsphinx "nbconvert!=5.4" -c defaults -c conda-forge | |
| # conda install -q pandoc -c conda-forge | |
| mamba install -q sphinx sphinx_rtd_theme sphinx-gallery pandoc # -c defaults -c conda-forge | |
| make -C docs html && touch docs/build/html/.nojekyll | |
| - name: build python package sdist | |
| if: ${{ matrix.py_ver == env.DEPLOY_PY_VER }} | |
| run: python setup.py sdist | |
| # ------------------------------------------------------------ | |
| # 3. Deploy | |
| # | |
| # DEPLOY_TYPE selects conda, python package, and docs uploads | |
| # | |
| # for regex matching | |
| # . canonical version strings are M.N.P.devX and M.N.P | |
| # . tarballs are $CONDA_BLD_PATH/linux-64/PACKAGE_NAME-.+-py3[678]_g[a-z0-9]{8}.tar.bz2 | |
| # . pkg_version is entire M.N.P.devX version string | |
| # . pkg_mnp is M.N.P only | |
| - name: set DEPLOY_TYPE=<pre-release|main|no_deploy> | |
| env: | |
| OS: ${{ runner.os }} # Linux, macOS | |
| run: | | |
| # lookup the conda tarball package version and M.N.P string (if any) | |
| tarball=${{ steps.conda-bld.outputs.conda-tarball }} | |
| pkg_version=`echo $tarball | sed -n "s/.*${PACKAGE_NAME}-\(.*\)-.*/\1/p"` | |
| pkg_mnp=`echo $pkg_version | sed -n "s/\([0-9]*\.[0-9]*\.[0-9]\).*/\1/p"` | |
| # pre-release if package version is M.N.P.devX on dev branch | |
| if [[ \ | |
| $GITHUB_REF =~ ^refs/heads/dev$ && \ | |
| $pkg_version =~ ^([0-9]+\.){2}[0-9]+(\.dev[0-9]+){0,1}$ \ | |
| ]]; then \ | |
| deploy_type="pre-release"; \ | |
| fi | |
| # main if package version is M.N.P on release tag vM.N.P | |
| if [[ \ | |
| $GITHUB_REF =~ ^refs/tags/v([0-9]+\.){2}[0-9]+$ && \ | |
| $pkg_version == $pkg_mnp \ | |
| ]]; then \ | |
| deploy_type="main"; \ | |
| fi | |
| # else, default | |
| deploy_type=${deploy_type:-no_deploy} | |
| # set top-level env for uploads | |
| echo "PKG_VERSION=${pkg_version}" >> $GITHUB_ENV | |
| echo "PKG_MNP=${pkg_mnp}" >> $GITHUB_ENV | |
| echo "DEPLOY_TYPE=${deploy_type}" >> $GITHUB_ENV | |
| # logging | |
| echo $PACKAGE_NAME $deploy_type $tarball $pkg_version $pkg_mnp | |
| - name: >- | |
| show ${{ env.PACKAGE_NAME }} | |
| ${{ env.PKG_VERSION }} | |
| env_${{ env.PY_VER }} | |
| deploy_type=${{ env.DEPLOY_TYPE }} | |
| run: | | |
| conda activate env_$PY_VER | |
| printenv | sort | |
| conda list --explicit | |
| # linux and osx jobs built their own tarballs; linux-64 conda converts to win-64 | |
| - name: deploy conda package python ${{ matrix.py_ver}} ${{ env.DEPLOY_TYPE }} | |
| if: ${{ env.DEPLOY_TYPE == 'pre-release' || env.DEPLOY_TYPE == 'main' }} | |
| env: | |
| ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} | |
| run: >- | |
| anaconda | |
| -t "$ANACONDA_TOKEN" | |
| upload $CONDA_BLD_PATH/**/${PACKAGE_NAME}*.tar.bz2 | |
| -l $DEPLOY_TYPE | |
| --skip-existing | |
| - name: deploy sphinx docs to gh-pages | |
| # switch docs deployment repo/branch on deploy type | |
| # if: ${{ matrix.py_ver == env.DEPLOY_PY_VER && matrix.os == env.DEPLOY_OS }} # testing only | |
| if: >- | |
| ${{ | |
| matrix.py_ver == env.DEPLOY_PY_VER | |
| && matrix.os == env.DEPLOY_OS | |
| && (env.DEPLOY_TYPE == 'main' || env.DEPLOY_TYPE == 'pre-release') | |
| }} | |
| env: | |
| # machine user account PAT | |
| GH_USER_NAME: robo-kutaslab | |
| GH_USER_EMAIL: robo.kutaslab@gmail.com | |
| GH_PAGES_TOKEN: ${{ secrets.GH_PAGES_TOKEN }} | |
| # vM.N.P tagged release docs are pushed here | |
| GH_PAGES_BRANCH: gh-pages | |
| # actions on dev branch push docs to this separate repo | |
| GH_PAGES_DEV_REPOSITORY: kutaslab/fitgrid-dev-docs | |
| GH_PAGES_DEV_BRANCH: gh-pages-dev | |
| run: | | |
| # set docs version repo/branch | |
| if [[ ${{ env.DEPLOY_TYPE }} == main ]]; then \ | |
| DOCS_REPO=$GITHUB_REPOSITORY; \ | |
| DOCS_BRANCH=$GH_PAGES_BRANCH; \ | |
| fi | |
| if [[ ${{ env.DEPLOY_TYPE }} == pre-release ]]; then \ | |
| DOCS_REPO=$GH_PAGES_DEV_REPOSITORY; \ | |
| DOCS_BRANCH=$GH_PAGES_DEV_BRANCH; \ | |
| fi | |
| echo "deploy type ${{ env.DEPLOY_TYPE }}: sphinx docs on $DOCS_REPO/$DOCS_BRANCH" | |
| git config --global user.name $GH_USER_NAME | |
| git config --global user.email $GH_USER_EMAIL | |
| git config --global init.defaultBranch main | |
| # wherever sphinx wrote the html, repo specific | |
| cd $GITHUB_WORKSPACE/docs/build/html | |
| git init | |
| git remote add origin "https://$GH_PAGES_TOKEN@github.com/${DOCS_REPO}" | |
| git checkout --orphan $DOCS_BRANCH | |
| git add -A | |
| git commit -a -m "deploy type $DEPLOY_TYPE python $DEPLOY_PY_VER" | |
| git push -u origin $DOCS_BRANCH --force | |