Run Workflows Weekly #14
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Workflows Weekly | |
| # The purpose of this CI file is to stress test our workflows at scale. | |
| # The *_weekly.yml workflows scatter over a much larger number of examples. | |
| # Note that toil-cwl-runner is slightly slower than cwltool, but | |
| # there are deadlocking problems with `cwltool --parallel` | |
| on: | |
| # push: # Do NOT run weekly workflows on every push! | |
| # pull_request: (either on origin or on upstream pull request) | |
| schedule: | |
| # Instead, run it every Tuesday at midnight. | |
| - cron: '0 0 * * 2' | |
| workflow_dispatch: | |
| defaults: | |
| run: | |
| shell: bash -l {0} # Invoke bash in login mode, NOT interactive mode. | |
| # This will cause bash to look for the startup file ~/.bash_profile, NOT ~/.bashrc | |
| # This is important since conda init writes to ~/.bashrc | |
| permissions: | |
| actions: read | |
| contents: read | |
| pull-requests: read | |
| jobs: | |
| run_weekly_toil: | |
| # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency | |
| # This will prevent DOS attacks from people blasting the CI with rapid fire commits. | |
| concurrency: | |
| group: ${{ github.workflow }}-weekly-toil-${{ github.ref }} | |
| cancel-in-progress: true | |
| runs-on: [self-hosted, linux] | |
| steps: | |
| - name: Checkout workflow-inference-compiler | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ github.repository_owner }}/workflow-inference-compiler | |
| ref: master | |
| path: workflow-inference-compiler | |
| - name: Checkout biobb_adapters | |
| if: always() | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: jfennick/biobb_adapters | |
| ref: master | |
| path: biobb_adapters | |
| - name: Checkout mm-workflows | |
| if: always() | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ github.repository_owner }}/mm-workflows | |
| ref: main | |
| path: mm-workflows | |
| - name: Remove old global config | |
| if: always() | |
| run: rm -rf "/home/$(whoami)/wic/" && rm -rf "/home/$(whoami)/.toil/" | |
| # For self-hosted runners, make sure we use new global config settings | |
| - name: Remove old mamba environment | |
| if: always() | |
| run: rm -rf "/home/$(whoami)/actions-runner/_work/workflow-inference-compiler/workflow-inference-compiler/3/envs/wic_github_actions/" | |
| # For self-hosted runners, make sure we install into a new mamba environment | |
| # Toil is currently incompatible with pypy | |
| # - name: Append pypy to conda environment files | |
| # if: runner.os != 'Windows' | |
| # run: cd workflow-inference-compiler/install/ && echo " - pypy" >> system_deps.yml | |
| - name: Setup mamba (linux, macos) | |
| if: always() | |
| uses: conda-incubator/setup-miniconda@v2.2.0 | |
| with: | |
| miniforge-variant: Mambaforge # NOT Mambaforge-pypy3 | |
| # Toil is (currently) incompatible with pypy. Mambaforge-pypy3 | |
| # installs pypy in the base environment (only). Although we are using | |
| # another environment, better to avoid the problem altogether by | |
| # not using Mambaforge-pypy3 | |
| miniforge-version: latest | |
| environment-file: workflow-inference-compiler/install/system_deps.yml | |
| activate-environment: wic_github_actions | |
| use-mamba: true | |
| channels: conda-forge | |
| python-version: "3.9.*" # pypy is not yet compatible with 3.10 and 3.11 | |
| - name: Docker pull | |
| if: always() | |
| run: cd mm-workflows/ && ./dockerPull.sh | |
| # For self-hosted runners, make sure the docker cache is up-to-date. | |
| - name: Install Workflow Inference Compiler | |
| if: always() | |
| run: cd workflow-inference-compiler/ && pip install ".[all]" | |
| - name: Generate WIC Validation Jsonschema | |
| if: always() | |
| run: cd workflow-inference-compiler/ && wic --generate_schemas_only | |
| - name: Install Molecular Modeling Workflows | |
| if: always() | |
| # Also run mm-workflows command to generate | |
| # mm-workflows/autogenerated/schemas/config_schemas.json | |
| run: cd mm-workflows/ && pip install ".[all]" && mm-workflows | |
| - name: PyTest Run Workflows (On Push) | |
| if: always() | |
| # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 | |
| run: cd workflow-inference-compiler/ && pytest -k test_run_workflows_on_push --workers 8 --cwl_runner toil-cwl-runner # --cov | |
| - name: PyTest Run Inlined Workflows | |
| if: always() | |
| # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 | |
| run: cd workflow-inference-compiler/ && pytest -k test_run_inlined_workflows_on_push --workers 8 --cwl_runner toil-cwl-runner # --cov | |
| - name: PyTest Run Workflows (Weekly) | |
| if: always() | |
| # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 | |
| run: cd workflow-inference-compiler/ && pytest -k test_run_workflows_weekly --workers 8 --cwl_runner toil-cwl-runner # --cov | |
| - name: PyTest Run Inlined Workflows (Weekly) | |
| if: always() | |
| # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 | |
| run: cd workflow-inference-compiler/ && pytest -k test_run_inlined_workflows_weekly --workers 8 --cwl_runner toil-cwl-runner # --cov |