From bbe80c163312b70ce2d845beb72ba0bbc0fdda5d Mon Sep 17 00:00:00 2001 From: Jake Fennick Date: Fri, 9 Feb 2024 07:32:39 -1000 Subject: [PATCH] add image-workflows to cross-repo CI (#165) * add image-workflows to cross-repo CI * workflow_dispatch limited to 10 input parameters --------- Co-authored-by: Jake Fennick --- .github/my_actions/branch_dispatch/action.yml | 8 +-- .../my_actions/branch_dispatch/dist/index.js | 8 +-- .github/my_actions/branch_dispatch/index.js | 8 +-- .../branch_dispatch_pull_request.yml | 32 ++++++------ .github/workflows/branch_dispatch_push.yml | 24 ++++----- .../branch_dispatch_repository_dispatch.yml | 24 ++++----- .github/workflows/fuzzy_compile_weekly.yml | 27 ++++++++-- .github/workflows/lint_and_test.yml | 41 ++++++++++++---- .github/workflows/lint_and_test_macos.yml | 27 ++++++++-- .github/workflows/run_workflows.yml | 47 ++++++++++++++---- .github/workflows/run_workflows_weekly.yml | 33 +++++++++++-- README.md | 1 + install/install_image-workflows.sh | 11 +++++ src/wic/cli.py | 2 +- src/wic/plugins.py | 49 +++++++++++++++++-- 15 files changed, 253 insertions(+), 89 deletions(-) create mode 100755 install/install_image-workflows.sh diff --git a/.github/my_actions/branch_dispatch/action.yml b/.github/my_actions/branch_dispatch/action.yml index b4848c8e..3ee37e4e 100644 --- a/.github/my_actions/branch_dispatch/action.yml +++ b/.github/my_actions/branch_dispatch/action.yml @@ -46,12 +46,12 @@ inputs: description: 'The name of the mm-workflows branch / ref' type: string required: true - biobb_adapters_owner: - description: 'The name of the biobb_adapters owner' + image_workflows_owner: + description: 'The name of the image-workflows owner' type: string required: true - biobb_adapters_ref: - description: 'The name of the biobb_adapters branch / ref' + image_workflows_ref: + description: 'The name of the image-workflows branch / ref' type: string required: true access_token: diff --git a/.github/my_actions/branch_dispatch/dist/index.js b/.github/my_actions/branch_dispatch/dist/index.js index bc704924..6e08f58a 100644 --- a/.github/my_actions/branch_dispatch/dist/index.js +++ b/.github/my_actions/branch_dispatch/dist/index.js @@ -30,8 +30,8 @@ try { const commit_message = core.getInput('commit_message'); const mm_workflows_owner = core.getInput('mm_workflows_owner'); const mm_workflows_ref = core.getInput('mm_workflows_ref'); - const biobb_adapters_owner = core.getInput('biobb_adapters_owner'); - const biobb_adapters_ref = core.getInput('biobb_adapters_ref'); + const image_workflows_owner = core.getInput('image_workflows_owner'); + const image_workflows_ref = core.getInput('image_workflows_ref'); const access_token = core.getInput('access_token'); if (!access_token) { @@ -63,8 +63,8 @@ try { "wic_ref": wic_ref, "mm-workflows_owner": mm_workflows_owner, "mm-workflows_ref": mm_workflows_ref, - "biobb_adapters_owner": biobb_adapters_owner, - "biobb_adapters_ref": biobb_adapters_ref, + "image-workflows_owner": image_workflows_owner, + "image-workflows_ref": image_workflows_ref, }, }), headers: { diff --git a/.github/my_actions/branch_dispatch/index.js b/.github/my_actions/branch_dispatch/index.js index 50b97bc1..20085194 100644 --- a/.github/my_actions/branch_dispatch/index.js +++ b/.github/my_actions/branch_dispatch/index.js @@ -20,8 +20,8 @@ try { const commit_message = core.getInput('commit_message'); const mm_workflows_owner = core.getInput('mm_workflows_owner'); const mm_workflows_ref = core.getInput('mm_workflows_ref'); - const biobb_adapters_owner = core.getInput('biobb_adapters_owner'); - const biobb_adapters_ref = core.getInput('biobb_adapters_ref'); + const image_workflows_owner = core.getInput('image_workflows_owner'); + const image_workflows_ref = core.getInput('image_workflows_ref'); const access_token = core.getInput('access_token'); if (!access_token) { @@ -53,8 +53,8 @@ try { "wic_ref": wic_ref, "mm-workflows_owner": mm_workflows_owner, "mm-workflows_ref": mm_workflows_ref, - "biobb_adapters_owner": biobb_adapters_owner, - "biobb_adapters_ref": biobb_adapters_ref, + "image-workflows_owner": image_workflows_owner, + "image-workflows_ref": image_workflows_ref, }, }), headers: { diff --git a/.github/workflows/branch_dispatch_pull_request.yml b/.github/workflows/branch_dispatch_pull_request.yml index 111dce79..6fa26dd8 100644 --- a/.github/workflows/branch_dispatch_pull_request.yml +++ b/.github/workflows/branch_dispatch_pull_request.yml @@ -74,17 +74,6 @@ jobs: default_branch: master access_token: ${{ steps.generate_token.outputs.token }} - - name: Check existence of biobb_adapters - uses: ./.github/my_actions/check_existence/ # Must start with ./ - id: ce_biobb_adapters - with: - repository: biobb_adapters - sender_repo_owner: ${{ github.event.pull_request.head.repo.owner.login }} - sender_repo_ref: ${{ github.event.pull_request.head.ref }} - default_owner: jfennick # NOTE: NOT PolusAI (does not exist) - default_branch: master - access_token: ${{ steps.generate_token.outputs.token }} - - name: Check existence of dispatch ref in wic uses: ./.github/my_actions/check_existence/ # Must start with ./ id: ce_wic_dispatch @@ -96,7 +85,6 @@ jobs: default_branch: master access_token: ${{ steps.generate_token.outputs.token }} - # For other repositories, the entire step below should be copied and edited to make new steps. - name: Check existence of mm-workflows uses: ./.github/my_actions/check_existence/ # Must start with ./ id: ce_mm-workflows @@ -108,6 +96,18 @@ jobs: default_branch: main access_token: ${{ steps.generate_token.outputs.token }} + # For other repositories, the entire step below should be copied and edited to make new steps. + - name: Check existence of image-workflows + uses: ./.github/my_actions/check_existence/ # Must start with ./ + id: ce_image-workflows + with: + repository: image-workflows + sender_repo_owner: ${{ github.event.pull_request.head.repo.owner.login }} + sender_repo_ref: ${{ github.event.pull_request.head.ref }} + default_owner: PolusAI + default_branch: main + access_token: ${{ steps.generate_token.outputs.token }} + - name: Branch dispatch lint_and_test.yml uses: ./.github/my_actions/branch_dispatch/ # Must start with ./ id: bd_lint_and_test @@ -123,8 +123,8 @@ jobs: commit_message: ${{ env.commit_message }} mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }} mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }} - biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }} - biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }} + image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }} + image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }} access_token: ${{ steps.generate_token.outputs.token }} - name: Branch dispatch run_workflows.yml @@ -142,6 +142,6 @@ jobs: commit_message: ${{ env.commit_message }} mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }} mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }} - biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }} - biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }} + image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }} + image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }} access_token: ${{ steps.generate_token.outputs.token }} diff --git a/.github/workflows/branch_dispatch_push.yml b/.github/workflows/branch_dispatch_push.yml index 9d6a4860..96693c0f 100644 --- a/.github/workflows/branch_dispatch_push.yml +++ b/.github/workflows/branch_dispatch_push.yml @@ -38,23 +38,23 @@ jobs: default_branch: master access_token: ${{ steps.generate_token.outputs.token }} - - name: Check existence of biobb_adapters + - name: Check existence of mm-workflows uses: ./.github/my_actions/check_existence/ # Must start with ./ - id: ce_biobb_adapters + id: ce_mm-workflows with: - repository: biobb_adapters + repository: mm-workflows sender_repo_owner: ${{ github.repository_owner }} sender_repo_ref: ${{ github.ref_name }} - default_owner: jfennick # NOTE: NOT PolusAI (does not exist) - default_branch: master + default_owner: PolusAI + default_branch: main access_token: ${{ steps.generate_token.outputs.token }} # For other repositories, the entire step below should be copied and edited to make new steps. - - name: Check existence of mm-workflows + - name: Check existence of image-workflows uses: ./.github/my_actions/check_existence/ # Must start with ./ - id: ce_mm-workflows + id: ce_image-workflows with: - repository: mm-workflows + repository: image-workflows sender_repo_owner: ${{ github.repository_owner }} sender_repo_ref: ${{ github.ref_name }} default_owner: PolusAI @@ -76,8 +76,8 @@ jobs: commit_message: ${{ github.event.head_commit.message }} mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }} mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }} - biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }} - biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }} + image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }} + image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }} access_token: ${{ steps.generate_token.outputs.token }} - name: Branch dispatch run_workflows.yml @@ -95,6 +95,6 @@ jobs: commit_message: ${{ github.event.head_commit.message }} mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }} mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }} - biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }} - biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }} + image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }} + image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }} access_token: ${{ steps.generate_token.outputs.token }} diff --git a/.github/workflows/branch_dispatch_repository_dispatch.yml b/.github/workflows/branch_dispatch_repository_dispatch.yml index 0a84bbc7..1b92190d 100644 --- a/.github/workflows/branch_dispatch_repository_dispatch.yml +++ b/.github/workflows/branch_dispatch_repository_dispatch.yml @@ -70,23 +70,23 @@ jobs: default_branch: master access_token: ${{ steps.generate_token.outputs.token }} - - name: Check existence of biobb_adapters + - name: Check existence of mm-workflows uses: ./.github/my_actions/check_existence/ # Must start with ./ - id: ce_biobb_adapters + id: ce_mm-workflows with: - repository: biobb_adapters + repository: mm-workflows sender_repo_owner: ${{ github.event.client_payload.owner }} sender_repo_ref: ${{ github.event.client_payload.ref_name }} - default_owner: jfennick # NOTE: NOT PolusAI (does not exist) - default_branch: master + default_owner: PolusAI + default_branch: main access_token: ${{ steps.generate_token.outputs.token }} # For other repositories, the entire step below should be copied and edited to make new steps. - - name: Check existence of mm-workflows + - name: Check existence of image-workflows uses: ./.github/my_actions/check_existence/ # Must start with ./ - id: ce_mm-workflows + id: ce_image-workflows with: - repository: mm-workflows + repository: image-workflows sender_repo_owner: ${{ github.event.client_payload.owner }} sender_repo_ref: ${{ github.event.client_payload.ref_name }} default_owner: PolusAI @@ -108,8 +108,8 @@ jobs: commit_message: ${{ github.event.client_payload.commit_message }} mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }} mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }} - biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }} - biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }} + image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }} + image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }} access_token: ${{ steps.generate_token.outputs.token }} - name: Branch dispatch run_workflows.yml @@ -127,6 +127,6 @@ jobs: commit_message: ${{ github.event.client_payload.commit_message }} mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }} mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }} - biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }} - biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }} + image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }} + image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }} access_token: ${{ steps.generate_token.outputs.token }} diff --git a/.github/workflows/fuzzy_compile_weekly.yml b/.github/workflows/fuzzy_compile_weekly.yml index ea4774fd..de23e5ed 100644 --- a/.github/workflows/fuzzy_compile_weekly.yml +++ b/.github/workflows/fuzzy_compile_weekly.yml @@ -59,6 +59,14 @@ jobs: ref: main path: mm-workflows + - name: Checkout image-workflows + if: always() + uses: actions/checkout@v3 + with: + repository: ${{ github.repository_owner }}/image-workflows + ref: main + path: image-workflows + - name: Setup mamba (linux, macos) if: runner.os != 'Windows' uses: conda-incubator/setup-miniconda@v3.0.1 @@ -74,10 +82,6 @@ jobs: if: always() run: cd workflow-inference-compiler/ && pip install ".[all]" - - name: Generate WIC Validation Jsonschema - if: always() - run: cd workflow-inference-compiler/ && wic --generate_schemas_only - - name: Install Molecular Modeling Workflows if: always() # Also run mm-workflows command to generate @@ -87,6 +91,21 @@ jobs: # (Many of the packages conflict with pypy.) run: cd mm-workflows/ && pip install ".[test]" && mm-workflows + - name: Generate WIC Python API Workflows (*.py -> *.yml) + if: always() + run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' + + - name: Generate WIC Validation Jsonschema + if: always() + run: cd workflow-inference-compiler/ && wic --generate_schemas_only + + # Please read docs/validation.md#Property-Based-Testing + # This is essentially an integration test for all of the + # WIC Python API workflows as well as the WIC Python API itself. + - name: Validate WIC Python API Workflows (*.py -> *.yml) + if: always() + run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' + # Since a randomly chosen subschema is used every time, repeat 10X for more coverage. - name: PyTest Fuzzy Compile diff --git a/.github/workflows/lint_and_test.yml b/.github/workflows/lint_and_test.yml index 267a4c39..484199e2 100644 --- a/.github/workflows/lint_and_test.yml +++ b/.github/workflows/lint_and_test.yml @@ -38,12 +38,12 @@ on: description: The branch name within the mm-workflows repository required: true type: string - biobb_adapters_owner: - description: The account name of the biobb_adapters repository + image-workflows_owner: + description: The account name of the image-workflows repository required: true type: string - biobb_adapters_ref: - description: The branch name within the biobb_adapters repository + image-workflows_ref: + description: The branch name within the image-workflows repository required: true type: string @@ -102,8 +102,10 @@ jobs: if: always() uses: actions/checkout@v3 with: - repository: ${{ inputs.biobb_adapters_owner }}/biobb_adapters - ref: ${{ inputs.biobb_adapters_ref }} + # NOTE: temporarily hardcode jfennick & master because we can only + # have up to 10 input parameters for workflow_dispatch... + repository: jfennick/biobb_adapters + ref: master path: biobb_adapters - name: Checkout mm-workflows @@ -114,6 +116,14 @@ jobs: ref: ${{ inputs.mm-workflows_ref }} path: mm-workflows + - name: Checkout image-workflows + if: always() + uses: actions/checkout@v3 + with: + repository: ${{ inputs.image-workflows_owner }}/image-workflows + ref: ${{ inputs.image-workflows_ref }} + path: image-workflows + # NOTE: pypy actually decreases performance for lint_and_test.yml (by a factor of ~2) # - name: Append pypy to conda environment files # if: runner.os != 'Windows' @@ -154,10 +164,6 @@ jobs: if: always() run: cd workflow-inference-compiler/ && pip install ".[all]" - - name: Generate WIC Validation Jsonschema - if: always() - run: cd workflow-inference-compiler/ && wic --generate_schemas_only - - name: Install Molecular Modeling Workflows if: always() # Also run mm-workflows command to generate @@ -167,6 +173,21 @@ jobs: # (Many of the packages conflict with pypy.) run: cd mm-workflows/ && pip install ".[test]" && mm-workflows + - name: Generate WIC Python API Workflows (*.py -> *.yml) + if: always() + run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' + + - name: Generate WIC Validation Jsonschema + if: always() + run: cd workflow-inference-compiler/ && wic --generate_schemas_only + + # Please read docs/validation.md#Property-Based-Testing + # This is essentially an integration test for all of the + # WIC Python API workflows as well as the WIC Python API itself. + - name: Validate WIC Python API Workflows (*.py -> *.yml) + if: always() + run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' + - name: Build Documentation if: always() run: cd workflow-inference-compiler/docs && make html diff --git a/.github/workflows/lint_and_test_macos.yml b/.github/workflows/lint_and_test_macos.yml index c944b46f..7506dc70 100644 --- a/.github/workflows/lint_and_test_macos.yml +++ b/.github/workflows/lint_and_test_macos.yml @@ -59,6 +59,14 @@ jobs: ref: main path: mm-workflows + - name: Checkout image-workflows + if: always() + uses: actions/checkout@v3 + with: + repository: ${{ github.repository_owner }}/image-workflows + ref: main + path: image-workflows + - name: Setup mamba (linux, macos) if: runner.os != 'Windows' uses: conda-incubator/setup-miniconda@v2.2.0 @@ -80,10 +88,6 @@ jobs: if: always() run: cd workflow-inference-compiler/ && pip install ".[all]" - - name: Generate WIC Validation Jsonschema - if: always() - run: cd workflow-inference-compiler/ && wic --generate_schemas_only - - name: Install Molecular Modeling Workflows if: always() # Also run mm-workflows command to generate @@ -93,6 +97,21 @@ jobs: # (Many of the packages conflict with pypy.) run: cd mm-workflows/ && pip install ".[test]" && mm-workflows + - name: Generate WIC Python API Workflows (*.py -> *.yml) + if: always() + run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' + + - name: Generate WIC Validation Jsonschema + if: always() + run: cd workflow-inference-compiler/ && wic --generate_schemas_only + + # Please read docs/validation.md#Property-Based-Testing + # This is essentially an integration test for all of the + # WIC Python API workflows as well as the WIC Python API itself. + - name: Validate WIC Python API Workflows (*.py -> *.yml) + if: always() + run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' + - name: Build Documentation if: always() run: cd workflow-inference-compiler/docs && make html diff --git a/.github/workflows/run_workflows.yml b/.github/workflows/run_workflows.yml index 66a007b0..7071524d 100644 --- a/.github/workflows/run_workflows.yml +++ b/.github/workflows/run_workflows.yml @@ -38,12 +38,12 @@ on: description: The branch name within the mm-workflows repository required: true type: string - biobb_adapters_owner: - description: The account name of the biobb_adapters repository + image-workflows_owner: + description: The account name of the image-workflows repository required: true type: string - biobb_adapters_ref: - description: The branch name within the biobb_adapters repository + image-workflows_ref: + description: The branch name within the image-workflows repository required: true type: string @@ -79,8 +79,10 @@ jobs: if: always() uses: actions/checkout@v3 with: - repository: ${{ inputs.biobb_adapters_owner }}/biobb_adapters - ref: ${{ inputs.biobb_adapters_ref }} + # NOTE: temporarily hardcode jfennick & master because we can only + # have up to 10 input parameters for workflow_dispatch... + repository: jfennick/biobb_adapters + ref: master path: biobb_adapters - name: Checkout mm-workflows @@ -91,6 +93,14 @@ jobs: ref: ${{ inputs.mm-workflows_ref }} path: mm-workflows + - name: Checkout image-workflows + if: always() + uses: actions/checkout@v3 + with: + repository: ${{ inputs.image-workflows_owner }}/image-workflows + ref: ${{ inputs.image-workflows_ref }} + path: image-workflows + - name: Remove old global config if: always() run: rm -rf "/home/$(whoami)/wic/" && rm -rf "/home/$(whoami)/.toil/" @@ -132,10 +142,6 @@ jobs: if: always() run: cd workflow-inference-compiler/ && pip install ".[all]" - - name: Generate WIC Validation Jsonschema - if: always() - run: cd workflow-inference-compiler/ && wic --generate_schemas_only - - name: Install Molecular Modeling Workflows if: always() # Also run mm-workflows command to generate @@ -145,6 +151,27 @@ jobs: # (Many of the packages conflict with pypy.) run: cd mm-workflows/ && pip install ".[test]" && mm-workflows + - name: Generate WIC Python API Workflows (*.py -> *.yml) + if: always() + run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' + + - name: Generate WIC Validation Jsonschema + if: always() + run: cd workflow-inference-compiler/ && wic --generate_schemas_only + + # Please read docs/validation.md#Property-Based-Testing + # This is essentially an integration test for all of the + # WIC Python API workflows as well as the WIC Python API itself. + - name: Validate WIC Python API Workflows (*.py -> *.yml) + if: always() + run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' + + # Please read docker_remove_entrypoints.sh + # Note that this requires --no_force_docker_pull (so that we don't immediately un-remove them). + - name: docker remove entrypoints + if: always() + run: cd image-workflows/ && cwl_adapters/docker_remove_entrypoints.sh + - name: PyTest Run Workflows if: always() # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 diff --git a/.github/workflows/run_workflows_weekly.yml b/.github/workflows/run_workflows_weekly.yml index bb1a78a1..e70211ac 100644 --- a/.github/workflows/run_workflows_weekly.yml +++ b/.github/workflows/run_workflows_weekly.yml @@ -57,6 +57,14 @@ jobs: ref: main path: mm-workflows + - name: Checkout image-workflows + if: always() + uses: actions/checkout@v3 + with: + repository: ${{ github.repository_owner }}/image-workflows + ref: main + path: image-workflows + - name: Remove old global config if: always() run: rm -rf "/home/$(whoami)/wic/" && rm -rf "/home/$(whoami)/.toil/" @@ -97,10 +105,6 @@ jobs: if: always() run: cd workflow-inference-compiler/ && pip install ".[all]" - - name: Generate WIC Validation Jsonschema - if: always() - run: cd workflow-inference-compiler/ && wic --generate_schemas_only - - name: Install Molecular Modeling Workflows if: always() # Also run mm-workflows command to generate @@ -110,6 +114,27 @@ jobs: # (Many of the packages conflict with pypy.) run: cd mm-workflows/ && pip install ".[test]" && mm-workflows + - name: Generate WIC Python API Workflows (*.py -> *.yml) + if: always() + run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' + + - name: Generate WIC Validation Jsonschema + if: always() + run: cd workflow-inference-compiler/ && wic --generate_schemas_only + + # Please read docs/validation.md#Property-Based-Testing + # This is essentially an integration test for all of the + # WIC Python API workflows as well as the WIC Python API itself. + - name: Validate WIC Python API Workflows (*.py -> *.yml) + if: always() + run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()' + + # Please read docker_remove_entrypoints.sh + # Note that this requires --no_force_docker_pull (so that we don't immediately un-remove them). + - name: docker remove entrypoints + if: always() + run: cd image-workflows/ && cwl_adapters/docker_remove_entrypoints.sh + - name: PyTest Run Workflows (On Push) if: always() # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 diff --git a/README.md b/README.md index a02051ba..1766b5a8 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ pre-commit install # Required for developers cd install ./install_biobb_adapters.sh ./install_mm-workflows.sh +./install_image-workflows.sh cd .. ``` ``` diff --git a/install/install_image-workflows.sh b/install/install_image-workflows.sh new file mode 100755 index 00000000..27aefa37 --- /dev/null +++ b/install/install_image-workflows.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e +cd ../.. + +if [ -d "image-workflows" ] +then + echo "Directory image-workflows already exists." + echo "Please install image-workflows manually." + exit 1 +else + git clone https://github.com/PolusAI/image-workflows.git +fi diff --git a/src/wic/cli.py b/src/wic/cli.py index 72c220dd..2e4121cf 100644 --- a/src/wic/cli.py +++ b/src/wic/cli.py @@ -38,7 +38,7 @@ help='Do not check whether there are too many running docker processes before running workflows.') parser.add_argument('--user_space_docker_cmd', default='docker', help='Specify which command to use to run OCI containers.') -parser.add_argument('--no_force_docker_pull', default=False, action="store_true", +parser.add_argument('--no_force_docker_pull', default=True, action="store_true", help='''Do not force docker pull before running workflows. \n(i.e. allow overwriting images in your local docker cache)''') diff --git a/src/wic/plugins.py b/src/wic/plugins.py index c609ea69..92fbcb3c 100644 --- a/src/wic/plugins.py +++ b/src/wic/plugins.py @@ -9,6 +9,7 @@ import yaml from . import utils +from .python_cwl_adapter import import_python_file from .wic_types import Cwl, StepId, Tool, Tools @@ -112,7 +113,7 @@ def get_tools_cwl(homedir: str, validate_plugins: bool = False, skip_schemas: bo return tools_cwl -def get_yml_paths(homedir: str) -> Dict[str, Dict[str, Path]]: +def get_workflow_paths(homedir: str, extension: str) -> Dict[str, Dict[str, Path]]: """Uses glob() to recursively find all of the yml workflow definition files within any subdirectory of each yml_dir in yml_dirs_file. NOTE: This function assumes all yml files found are workflow definition files, @@ -121,7 +122,7 @@ def get_yml_paths(homedir: str) -> Dict[str, Dict[str, Path]]: Args: homedir (str): The users home directory - yml_dirs_file (Path): The subdirectories in which to search for yml files + extension (str): The filename extension (either 'yml' or 'py') Returns: Dict[str, Dict[str, Path]]: A dict containing the filepath stem and filepath of each yml file @@ -135,11 +136,11 @@ def get_yml_paths(homedir: str) -> Dict[str, Dict[str, Path]]: # "PurePath.relative_to() requires self to be the subpath of the argument, but os.path.relpath() does not." # See https://docs.python.org/3/library/pathlib.html#id4 and # See https://stackoverflow.com/questions/67452690/pathlib-path-relative-to-vs-os-path-relpath - pattern_yml = str(Path(yml_dir) / '**/*.yml') + pattern_yml = str(Path(yml_dir) / f'**/*.{extension}') yml_paths_sorted = sorted(glob.glob(pattern_yml, recursive=True), key=len, reverse=True) Path('autogenerated/schemas/workflows/').mkdir(parents=True, exist_ok=True) if len(yml_paths_sorted) == 0: - print(f'Warning! No yml files found in {yml_dir}.\nCheck {yml_dirs_file.absolute()}') + print(f'Warning! No {extension} files found in {yml_dir}.\nCheck {yml_dirs_file.absolute()}') print('This almost certainly means you are not in the correct directory.') yml_paths = {} for yml_path_str in yml_paths_sorted: @@ -154,3 +155,43 @@ def get_yml_paths(homedir: str) -> Dict[str, Dict[str, Path]]: yml_paths_all[yml_namespace] = {**ns_dict, **yml_paths} return yml_paths_all + + +def get_yml_paths(homedir: str) -> Dict[str, Dict[str, Path]]: + return get_workflow_paths(homedir, 'yml') + + +def get_py_paths(homedir: str) -> Dict[str, Dict[str, Path]]: + return get_workflow_paths(homedir, 'py') + + +def blindly_execute_python_workflows() -> None: + """This function imports (read: blindly executes) all python files in yml_dirs.txt + The python files are assumed to build a wic.api.pythonapi.Workflow object and + call the .compile() method, which has the desired side effect of writing a yml file to disk. + The python files should NOT call the .run() method! + (from any code path that is automatically executed on import) + """ + # I hope u like Remote Code Execution vulnerabilities! + # See https://en.wikipedia.org/wiki/Arithmetical_hierarchy + paths = get_py_paths(str(Path().home())) + paths_tuples = [(path_str, path) + for namespace, paths_dict in paths.items() + for path_str, path in paths_dict.items()] + for path_str, path in paths_tuples: + # NOTE: Use anything (unique?) for the python_module_name. + try: + module = import_python_file(path_str, path) + except Exception as e: + print(f'Could not import python file {path}') + print(e) + # TODO: Determine how to handle import failures + # NOTE: We could require all python API files to define a function, say + # def workflow(...) -> Workflow + # and then we could programmatically call it here: + # retval: Workflow = module.workflow(**args) + # which would allow us to programmatically call Workflow methods: + # retval.compile() # hopefully retval is actually a Workflow object! + # But since this is python (i.e. not Haskell) that in no way eliminates + # the above security considerations. + # So for now let's keep it simple and assume .compile() has been called.