add image-workflows to cross-repo CI (#165)

* add image-workflows to cross-repo CI * workflow_dispatch limited to 10 input parameters --------- Co-authored-by: Jake Fennick <jake.fennick@axleinfo.com>
PolusAI · Feb 9, 2024 · bbe80c1 · bbe80c1
1 parent 63e58da
commit bbe80c1
Show file tree

Hide file tree

Showing 15 changed files with 253 additions and 89 deletions.
diff --git a/.github/my_actions/branch_dispatch/action.yml b/.github/my_actions/branch_dispatch/action.yml
@@ -46,12 +46,12 @@ inputs:
     description: 'The name of the mm-workflows branch / ref'
     type: string
     required: true
-  biobb_adapters_owner:
-    description: 'The name of the biobb_adapters owner'
+  image_workflows_owner:
+    description: 'The name of the image-workflows owner'
     type: string
     required: true
-  biobb_adapters_ref:
-    description: 'The name of the biobb_adapters branch / ref'
+  image_workflows_ref:
+    description: 'The name of the image-workflows branch / ref'
     type: string
     required: true
   access_token:

diff --git a/.github/my_actions/branch_dispatch/dist/index.js b/.github/my_actions/branch_dispatch/dist/index.js
@@ -30,8 +30,8 @@ try {
   const commit_message = core.getInput('commit_message');
   const mm_workflows_owner = core.getInput('mm_workflows_owner');
   const mm_workflows_ref = core.getInput('mm_workflows_ref');
-  const biobb_adapters_owner = core.getInput('biobb_adapters_owner');
-  const biobb_adapters_ref = core.getInput('biobb_adapters_ref');
+  const image_workflows_owner = core.getInput('image_workflows_owner');
+  const image_workflows_ref = core.getInput('image_workflows_ref');
   const access_token = core.getInput('access_token');
 
   if (!access_token) {
@@ -63,8 +63,8 @@ try {
         "wic_ref": wic_ref,
         "mm-workflows_owner": mm_workflows_owner,
         "mm-workflows_ref": mm_workflows_ref,
-        "biobb_adapters_owner": biobb_adapters_owner,
-        "biobb_adapters_ref": biobb_adapters_ref,
+        "image-workflows_owner": image_workflows_owner,
+        "image-workflows_ref": image_workflows_ref,
       },
     }),
     headers: {

diff --git a/.github/my_actions/branch_dispatch/index.js b/.github/my_actions/branch_dispatch/index.js
@@ -20,8 +20,8 @@ try {
   const commit_message = core.getInput('commit_message');
   const mm_workflows_owner = core.getInput('mm_workflows_owner');
   const mm_workflows_ref = core.getInput('mm_workflows_ref');
-  const biobb_adapters_owner = core.getInput('biobb_adapters_owner');
-  const biobb_adapters_ref = core.getInput('biobb_adapters_ref');
+  const image_workflows_owner = core.getInput('image_workflows_owner');
+  const image_workflows_ref = core.getInput('image_workflows_ref');
   const access_token = core.getInput('access_token');
 
   if (!access_token) {
@@ -53,8 +53,8 @@ try {
         "wic_ref": wic_ref,
         "mm-workflows_owner": mm_workflows_owner,
         "mm-workflows_ref": mm_workflows_ref,
-        "biobb_adapters_owner": biobb_adapters_owner,
-        "biobb_adapters_ref": biobb_adapters_ref,
+        "image-workflows_owner": image_workflows_owner,
+        "image-workflows_ref": image_workflows_ref,
       },
     }),
     headers: {

diff --git a/.github/workflows/branch_dispatch_pull_request.yml b/.github/workflows/branch_dispatch_pull_request.yml
@@ -74,17 +74,6 @@ jobs:
           default_branch: master
           access_token: ${{ steps.generate_token.outputs.token }}
 
-      - name: Check existence of biobb_adapters
-        uses: ./.github/my_actions/check_existence/ # Must start with ./
-        id: ce_biobb_adapters
-        with:
-          repository: biobb_adapters
-          sender_repo_owner: ${{ github.event.pull_request.head.repo.owner.login }}
-          sender_repo_ref: ${{ github.event.pull_request.head.ref }}
-          default_owner: jfennick  # NOTE: NOT PolusAI (does not exist)
-          default_branch: master
-          access_token: ${{ steps.generate_token.outputs.token }}
-
       - name: Check existence of dispatch ref in wic
         uses: ./.github/my_actions/check_existence/ # Must start with ./
         id: ce_wic_dispatch
@@ -96,7 +85,6 @@ jobs:
           default_branch: master
           access_token: ${{ steps.generate_token.outputs.token }}
 
-      # For other repositories, the entire step below should be copied and edited to make new steps.
       - name: Check existence of mm-workflows
         uses: ./.github/my_actions/check_existence/ # Must start with ./
         id: ce_mm-workflows
@@ -108,6 +96,18 @@ jobs:
           default_branch: main
           access_token: ${{ steps.generate_token.outputs.token }}
 
+      # For other repositories, the entire step below should be copied and edited to make new steps.
+      - name: Check existence of image-workflows
+        uses: ./.github/my_actions/check_existence/ # Must start with ./
+        id: ce_image-workflows
+        with:
+          repository: image-workflows
+          sender_repo_owner: ${{ github.event.pull_request.head.repo.owner.login }}
+          sender_repo_ref: ${{ github.event.pull_request.head.ref }}
+          default_owner: PolusAI
+          default_branch: main
+          access_token: ${{ steps.generate_token.outputs.token }}
+
       - name: Branch dispatch lint_and_test.yml
         uses: ./.github/my_actions/branch_dispatch/ # Must start with ./
         id: bd_lint_and_test
@@ -123,8 +123,8 @@ jobs:
           commit_message: ${{ env.commit_message }}
           mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }}
           mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }}
-          biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }}
-          biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }}
+          image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }}
+          image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }}
           access_token: ${{ steps.generate_token.outputs.token }}
 
       - name: Branch dispatch run_workflows.yml
@@ -142,6 +142,6 @@ jobs:
           commit_message: ${{ env.commit_message }}
           mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }}
           mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }}
-          biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }}
-          biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }}
+          image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }}
+          image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }}
           access_token: ${{ steps.generate_token.outputs.token }}
diff --git a/.github/workflows/branch_dispatch_push.yml b/.github/workflows/branch_dispatch_push.yml
@@ -38,23 +38,23 @@ jobs:
           default_branch: master
           access_token: ${{ steps.generate_token.outputs.token }}
 
-      - name: Check existence of biobb_adapters
+      - name: Check existence of mm-workflows
         uses: ./.github/my_actions/check_existence/ # Must start with ./
-        id: ce_biobb_adapters
+        id: ce_mm-workflows
         with:
-          repository: biobb_adapters
+          repository: mm-workflows
           sender_repo_owner: ${{ github.repository_owner }}
           sender_repo_ref: ${{ github.ref_name }}
-          default_owner: jfennick  # NOTE: NOT PolusAI (does not exist)
-          default_branch: master
+          default_owner: PolusAI
+          default_branch: main
           access_token: ${{ steps.generate_token.outputs.token }}
 
       # For other repositories, the entire step below should be copied and edited to make new steps.
-      - name: Check existence of mm-workflows
+      - name: Check existence of image-workflows
         uses: ./.github/my_actions/check_existence/ # Must start with ./
-        id: ce_mm-workflows
+        id: ce_image-workflows
         with:
-          repository: mm-workflows
+          repository: image-workflows
           sender_repo_owner: ${{ github.repository_owner }}
           sender_repo_ref: ${{ github.ref_name }}
           default_owner: PolusAI
@@ -76,8 +76,8 @@ jobs:
           commit_message: ${{ github.event.head_commit.message }}
           mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }}
           mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }}
-          biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }}
-          biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }}
+          image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }}
+          image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }}
           access_token: ${{ steps.generate_token.outputs.token }}
 
       - name: Branch dispatch run_workflows.yml
@@ -95,6 +95,6 @@ jobs:
           commit_message: ${{ github.event.head_commit.message }}
           mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }}
           mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }}
-          biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }}
-          biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }}
+          image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }}
+          image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }}
           access_token: ${{ steps.generate_token.outputs.token }}
diff --git a/.github/workflows/branch_dispatch_repository_dispatch.yml b/.github/workflows/branch_dispatch_repository_dispatch.yml
@@ -70,23 +70,23 @@ jobs:
           default_branch: master
           access_token: ${{ steps.generate_token.outputs.token }}
 
-      - name: Check existence of biobb_adapters
+      - name: Check existence of mm-workflows
         uses: ./.github/my_actions/check_existence/ # Must start with ./
-        id: ce_biobb_adapters
+        id: ce_mm-workflows
         with:
-          repository: biobb_adapters
+          repository: mm-workflows
           sender_repo_owner: ${{ github.event.client_payload.owner }}
           sender_repo_ref: ${{ github.event.client_payload.ref_name }}
-          default_owner: jfennick  # NOTE: NOT PolusAI (does not exist)
-          default_branch: master
+          default_owner: PolusAI
+          default_branch: main
           access_token: ${{ steps.generate_token.outputs.token }}
 
       # For other repositories, the entire step below should be copied and edited to make new steps.
-      - name: Check existence of mm-workflows
+      - name: Check existence of image-workflows
         uses: ./.github/my_actions/check_existence/ # Must start with ./
-        id: ce_mm-workflows
+        id: ce_image-workflows
         with:
-          repository: mm-workflows
+          repository: image-workflows
           sender_repo_owner: ${{ github.event.client_payload.owner }}
           sender_repo_ref: ${{ github.event.client_payload.ref_name }}
           default_owner: PolusAI
@@ -108,8 +108,8 @@ jobs:
           commit_message: ${{ github.event.client_payload.commit_message }}
           mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }}
           mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }}
-          biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }}
-          biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }}
+          image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }}
+          image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }}
           access_token: ${{ steps.generate_token.outputs.token }}
 
       - name: Branch dispatch run_workflows.yml
@@ -127,6 +127,6 @@ jobs:
           commit_message: ${{ github.event.client_payload.commit_message }}
           mm_workflows_owner: ${{ steps.ce_mm-workflows.outputs.owner }}
           mm_workflows_ref: ${{ steps.ce_mm-workflows.outputs.ref }}
-          biobb_adapters_owner: ${{ steps.ce_biobb_adapters.outputs.owner }}
-          biobb_adapters_ref: ${{ steps.ce_biobb_adapters.outputs.ref }}
+          image_workflows_owner: ${{ steps.ce_image-workflows.outputs.owner }}
+          image_workflows_ref: ${{ steps.ce_image-workflows.outputs.ref }}
           access_token: ${{ steps.generate_token.outputs.token }}
diff --git a/.github/workflows/fuzzy_compile_weekly.yml b/.github/workflows/fuzzy_compile_weekly.yml
@@ -59,6 +59,14 @@ jobs:
           ref: main
           path: mm-workflows
 
+      - name: Checkout image-workflows
+        if: always()
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ github.repository_owner }}/image-workflows
+          ref: main
+          path: image-workflows
+
       - name: Setup mamba (linux, macos)
         if: runner.os != 'Windows'
         uses: conda-incubator/setup-miniconda@v3.0.1
@@ -74,10 +82,6 @@ jobs:
         if: always()
         run: cd workflow-inference-compiler/ && pip install ".[all]"
 
-      - name: Generate WIC Validation Jsonschema
-        if: always()
-        run: cd workflow-inference-compiler/ && wic --generate_schemas_only
-
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate
@@ -87,6 +91,21 @@ jobs:
         # (Many of the packages conflict with pypy.)
         run: cd mm-workflows/ && pip install ".[test]" && mm-workflows
 
+      - name: Generate WIC Python API Workflows (*.py -> *.yml)
+        if: always()
+        run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()'
+
+      - name: Generate WIC Validation Jsonschema
+        if: always()
+        run: cd workflow-inference-compiler/ && wic --generate_schemas_only
+
+        # Please read docs/validation.md#Property-Based-Testing
+        # This is essentially an integration test for all of the
+        # WIC Python API workflows as well as the WIC Python API itself.
+      - name: Validate WIC Python API Workflows (*.py -> *.yml)
+        if: always()
+        run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()'
+
 # Since a randomly chosen subschema is used every time, repeat 10X for more coverage.
 
       - name: PyTest Fuzzy Compile

diff --git a/.github/workflows/lint_and_test.yml b/.github/workflows/lint_and_test.yml
@@ -38,12 +38,12 @@ on:
         description: The branch name within the mm-workflows repository
         required: true
         type: string
-      biobb_adapters_owner:
-        description: The account name of the biobb_adapters repository
+      image-workflows_owner:
+        description: The account name of the image-workflows repository
         required: true
         type: string
-      biobb_adapters_ref:
-        description: The branch name within the biobb_adapters repository
+      image-workflows_ref:
+        description: The branch name within the image-workflows repository
         required: true
         type: string
 
@@ -102,8 +102,10 @@ jobs:
         if: always()
         uses: actions/checkout@v3
         with:
-          repository: ${{ inputs.biobb_adapters_owner }}/biobb_adapters
-          ref: ${{ inputs.biobb_adapters_ref }}
+          # NOTE: temporarily hardcode jfennick & master because we can only
+          # have up to 10 input parameters for workflow_dispatch...
+          repository: jfennick/biobb_adapters
+          ref: master
           path: biobb_adapters
 
       - name: Checkout mm-workflows
@@ -114,6 +116,14 @@ jobs:
           ref: ${{ inputs.mm-workflows_ref }}
           path: mm-workflows
 
+      - name: Checkout image-workflows
+        if: always()
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ inputs.image-workflows_owner }}/image-workflows
+          ref: ${{ inputs.image-workflows_ref }}
+          path: image-workflows
+
       # NOTE: pypy actually decreases performance for lint_and_test.yml (by a factor of ~2)
       # - name: Append pypy to conda environment files
       #   if: runner.os != 'Windows'
@@ -154,10 +164,6 @@ jobs:
         if: always()
         run: cd workflow-inference-compiler/ && pip install ".[all]"
 
-      - name: Generate WIC Validation Jsonschema
-        if: always()
-        run: cd workflow-inference-compiler/ && wic --generate_schemas_only
-
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate
@@ -167,6 +173,21 @@ jobs:
         # (Many of the packages conflict with pypy.)
         run: cd mm-workflows/ && pip install ".[test]" && mm-workflows
 
+      - name: Generate WIC Python API Workflows (*.py -> *.yml)
+        if: always()
+        run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()'
+
+      - name: Generate WIC Validation Jsonschema
+        if: always()
+        run: cd workflow-inference-compiler/ && wic --generate_schemas_only
+
+        # Please read docs/validation.md#Property-Based-Testing
+        # This is essentially an integration test for all of the
+        # WIC Python API workflows as well as the WIC Python API itself.
+      - name: Validate WIC Python API Workflows (*.py -> *.yml)
+        if: always()
+        run: cd workflow-inference-compiler/ && python -c 'import wic; import wic.plugins; wic.plugins.blindly_execute_python_workflows()'
+
       - name: Build Documentation
         if: always()
         run: cd workflow-inference-compiler/docs && make html