Release 015

.github/workflows/fuzzy_compile_weekly.yml

@@ -67,11 +67,11 @@ jobs:
           ref: main
           path: image-workflows
 
-      - name: Setup mamba (linux, macos)
+      - name: Setup miniforge (linux, macos)
         if: runner.os != 'Windows'
         uses: conda-incubator/setup-miniconda@v3.0.1
         with:
-          miniforge-variant: Mambaforge-pypy3
+          miniforge-variant: Miniforge-pypy3
           miniforge-version: latest
           environment-file: workflow-inference-compiler/install/system_deps.yml
           activate-environment: wic

.github/workflows/lint_and_test.yml

@@ -133,22 +133,22 @@ jobs:
       #   if: runner.os == 'Windows'
       #   run: cd workflow-inference-compiler/install/ && echo "  - pypy" >> system_deps_windows.yml
 
-      - name: Setup mamba (linux, macos)
+      - name: Setup miniforge (linux, macos)
         if: runner.os != 'Windows'
         uses: conda-incubator/setup-miniconda@v3.0.1
         with:
-          miniforge-variant: Mambaforge-pypy3
+          miniforge-variant: Miniforge-pypy3
           miniforge-version: latest
           environment-file: workflow-inference-compiler/install/system_deps.yml
           activate-environment: wic
           channels: conda-forge
           python-version: "3.9.*" # pypy is not yet compatible with 3.10 and 3.11
 
-      - name: Setup mamba (windows)
+      - name: Setup miniforge (windows)
         if: runner.os == 'Windows'
         uses: conda-incubator/setup-miniconda@v3.0.1
         with:
-          miniforge-variant: Mambaforge-pypy3
+          miniforge-variant: Miniforge-pypy3
           miniforge-version: latest
           environment-file: workflow-inference-compiler/install/system_deps_windows.yml
           activate-environment: wic

.github/workflows/lint_and_test_macos.yml

@@ -67,11 +67,11 @@ jobs:
           ref: main
           path: image-workflows
 
-      - name: Setup mamba (linux, macos)
+      - name: Setup miniforge (linux, macos)
         if: runner.os != 'Windows'
         uses: conda-incubator/setup-miniconda@v2.2.0
         with:
-          miniforge-variant: Mambaforge-pypy3
+          miniforge-variant: Miniforge-pypy3
           miniforge-version: latest
           environment-file: workflow-inference-compiler/install/system_deps.yml
           activate-environment: wic

.github/workflows/run_workflows.yml

@@ -122,11 +122,11 @@ jobs:
       # we will get "EnvironmentNameNotFound: Could not find conda environment: wic_github_actions"
       # and (again, due to "set -e") the workflow step will fail.
 
-    - name: Setup mamba (linux, macos)
+    - name: Setup miniforge (linux, macos)
       if: always()
       uses: conda-incubator/setup-miniconda@v3.0.1
       with:
-        miniforge-variant: Mambaforge-pypy3
+        miniforge-variant: Miniforge-pypy3
         miniforge-version: latest
         environment-file: workflow-inference-compiler/install/system_deps.yml
         activate-environment: wic_github_actions
@@ -181,6 +181,21 @@ jobs:
       # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
       run: cd workflow-inference-compiler/ && pytest tests/test_rest_core.py -k test_rest_core --cwl_runner cwltool
 
+    - name: PyTest Run REST WFB Tests
+      if: always()
+      # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
+      run: cd workflow-inference-compiler/ && pytest tests/test_rest_wfb.py -k test_rest_wfb --cwl_runner cwltool
+
+    - name: PyTest Run ICT to CLT conversion Tests
+      if: always()
+      # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
+      run: cd workflow-inference-compiler/ && pytest tests/test_ict_to_clt_conversion.py -k test_ict_to_clt
+
+    - name: PyTest Run update wfb payload Tests
+      if: always()
+      # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
+      run: cd workflow-inference-compiler/ && pytest tests/test_fix_payload.py -k test_fix
+
     # NOTE: The steps below are for repository_dispatch only. For all other steps, please insert above
     # this comment.
 

.github/workflows/run_workflows_weekly.yml

@@ -80,15 +80,15 @@ jobs:
     #   if: runner.os != 'Windows'
     #   run: cd workflow-inference-compiler/install/ && echo "  - pypy" >> system_deps.yml
 
-    - name: Setup mamba (linux, macos)
+    - name: Setup miniforge (linux, macos)
       if: always()
       uses: conda-incubator/setup-miniconda@v3.0.1
       with:
-        miniforge-variant: Mambaforge  # NOT Mambaforge-pypy3
-        # Toil is (currently) incompatible with pypy. Mambaforge-pypy3
+        miniforge-variant: Miniforge  # NOT Miniforge-pypy3
+        # Toil is (currently) incompatible with pypy. Miniforge-pypy3
         # installs pypy in the base environment (only). Although we are using
         # another environment, better to avoid the problem altogether by
-        # not using Mambaforge-pypy3
+        # not using Miniforge-pypy3
         miniforge-version: latest
         environment-file: workflow-inference-compiler/install/system_deps.yml
         activate-environment: wic_github_actions

docker/Dockerfile_amazon

@@ -5,12 +5,12 @@ FROM amazonlinux
 # Install conda / mamba
 RUN yum update -y && yum install -y wget
 
-RUN CONDA="Mambaforge-Linux-x86_64.sh" && \
+RUN CONDA="Miniforge3-Linux-x86_64.sh" && \
     wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \
     chmod +x $CONDA && \
-    ./$CONDA -b -p /mambaforge && \
+    ./$CONDA -b -p /miniforge && \
     rm -f $CONDA
-ENV PATH /mambaforge/bin:$PATH
+ENV PATH /miniforge/bin:$PATH
 
 # Install wic
 RUN yum install -y git

docker/Dockerfile_amazon_pypy

@@ -5,12 +5,12 @@ FROM amazonlinux
 # Install conda / mamba
 RUN yum update -y && yum install -y wget
 
-RUN CONDA="Mambaforge-pypy3-Linux-x86_64.sh" && \
+RUN CONDA="Miniforge-pypy3-Linux-x86_64.sh" && \
     wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \
     chmod +x $CONDA && \
-    ./$CONDA -b -p /mambaforge-pypy3 && \
+    ./$CONDA -b -p /Miniforge-pypy3 && \
     rm -f $CONDA
-ENV PATH /mambaforge-pypy3/bin:$PATH
+ENV PATH /Miniforge-pypy3/bin:$PATH
 
 # Install wic
 RUN yum install -y git

docker/Dockerfile_debian

@@ -5,12 +5,12 @@ FROM debian:stable-slim
 # Install conda / mamba
 RUN apt-get update -y && apt-get install -y wget
 
-RUN CONDA="Mambaforge-Linux-x86_64.sh" && \
+RUN CONDA="Miniforge3-Linux-x86_64.sh" && \
     wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \
     chmod +x $CONDA && \
-    ./$CONDA -b -p /mambaforge && \
+    ./$CONDA -b -p /miniforge && \
     rm -f $CONDA
-ENV PATH /mambaforge/bin:$PATH
+ENV PATH /miniforge/bin:$PATH
 
 # Install wic
 RUN apt-get install -y git

docker/Dockerfile_debian_pypy

@@ -5,12 +5,12 @@ FROM debian:stable-slim
 # Install conda / mamba
 RUN apt-get update -y && apt-get install -y wget
 
-RUN CONDA="Mambaforge-pypy3-Linux-x86_64.sh" && \
+RUN CONDA="Miniforge-pypy3-Linux-x86_64.sh" && \
     wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \
     chmod +x $CONDA && \
-    ./$CONDA -b -p /mambaforge-pypy3 && \
+    ./$CONDA -b -p /Miniforge-pypy3 && \
     rm -f $CONDA
-ENV PATH /mambaforge-pypy3/bin:$PATH
+ENV PATH /Miniforge-pypy3/bin:$PATH
 
 # Install wic
 RUN apt-get install -y git

docker/Dockerfile_fedora

@@ -5,12 +5,12 @@ FROM fedora
 # Install conda / mamba
 RUN yum update -y && yum install -y wget
 
-RUN CONDA="Mambaforge-Linux-x86_64.sh" && \
+RUN CONDA="Miniforge3-Linux-x86_64.sh" && \
     wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \
     chmod +x $CONDA && \
-    ./$CONDA -b -p /mambaforge && \
+    ./$CONDA -b -p /miniforge && \
     rm -f $CONDA
-ENV PATH /mambaforge/bin:$PATH
+ENV PATH /miniforge/bin:$PATH
 
 # Install wic
 RUN yum install -y git

docker/Dockerfile_fedora_pypy

@@ -5,12 +5,12 @@ FROM fedora
 # Install conda / mamba
 RUN yum update -y && yum install -y wget
 
-RUN CONDA="Mambaforge-pypy3-Linux-x86_64.sh" && \
+RUN CONDA="Miniforge-pypy3-Linux-x86_64.sh" && \
     wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \
     chmod +x $CONDA && \
-    ./$CONDA -b -p /mambaforge-pypy3 && \
+    ./$CONDA -b -p /Miniforge-pypy3 && \
     rm -f $CONDA
-ENV PATH /mambaforge-pypy3/bin:$PATH
+ENV PATH /Miniforge-pypy3/bin:$PATH
 
 # Install wic
 RUN yum install -y git

docker/Dockerfile_redhat

@@ -6,12 +6,12 @@ FROM redhat/ubi9-minimal
 # Install conda / mamba
 RUN microdnf update -y && microdnf install -y wget
 
-RUN CONDA="Mambaforge-Linux-x86_64.sh" && \
+RUN CONDA="Miniforge3-Linux-x86_64.sh" && \
     wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \
     chmod +x $CONDA && \
-    ./$CONDA -b -p /mambaforge && \
+    ./$CONDA -b -p /miniforge && \
     rm -f $CONDA
-ENV PATH /mambaforge/bin:$PATH
+ENV PATH /miniforge/bin:$PATH
 
 # Install wic
 RUN microdnf install -y git

docker/Dockerfile_redhat_pypy

@@ -6,12 +6,12 @@ FROM redhat/ubi9-minimal
 # Install conda / mamba
 RUN microdnf update -y && microdnf install -y wget
 
-RUN CONDA="Mambaforge-pypy3-Linux-x86_64.sh" && \
+RUN CONDA="Miniforge-pypy3-Linux-x86_64.sh" && \
     wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \
     chmod +x $CONDA && \
-    ./$CONDA -b -p /mambaforge-pypy3 && \
+    ./$CONDA -b -p /Miniforge-pypy3 && \
     rm -f $CONDA
-ENV PATH /mambaforge-pypy3/bin:$PATH
+ENV PATH /Miniforge-pypy3/bin:$PATH
 
 # Install wic
 RUN microdnf install -y git

docker/Dockerfile_ubuntu

@@ -5,12 +5,12 @@ FROM ubuntu
 # Install conda / mamba
 RUN apt-get update -y && apt-get install -y wget
 
-RUN CONDA="Mambaforge-Linux-x86_64.sh" && \
+RUN CONDA="Miniforge3-Linux-x86_64.sh" && \
     wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \
     chmod +x $CONDA && \
-    ./$CONDA -b -p /mambaforge && \
+    ./$CONDA -b -p /miniforge && \
     rm -f $CONDA
-ENV PATH /mambaforge/bin:$PATH
+ENV PATH /miniforge/bin:$PATH
 
 # Install wic
 RUN apt-get install -y git

docker/Dockerfile_ubuntu_pypy

@@ -5,12 +5,12 @@ FROM ubuntu
 # Install conda / mamba
 RUN apt-get update -y && apt-get install -y wget
 
-RUN CONDA="Mambaforge-pypy3-Linux-x86_64.sh" && \
+RUN CONDA="Miniforge-pypy3-Linux-x86_64.sh" && \
     wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \
     chmod +x $CONDA && \
-    ./$CONDA -b -p /mambaforge-pypy3 && \
+    ./$CONDA -b -p /Miniforge-pypy3 && \
     rm -f $CONDA
-ENV PATH /mambaforge-pypy3/bin:$PATH
+ENV PATH /Miniforge-pypy3/bin:$PATH
 
 # Install wic
 RUN apt-get install -y git

examples/scripts/Dockerfile_check_linear_fit

@@ -1,4 +1,4 @@
-FROM condaforge/mambaforge-pypy3
+FROM condaforge/Miniforge-pypy3
 
 RUN mamba install -c conda-forge numpy scipy
 

examples/scripts/Dockerfile_scatter_plot

@@ -1,4 +1,4 @@
-FROM condaforge/mambaforge-pypy3
+FROM condaforge/Miniforge-pypy3
 
 RUN mamba install -c conda-forge matplotlib
 

install/install_conda.bat

@@ -1,4 +1,4 @@
-set CONDA="Mambaforge-pypy3-Windows-x86_64.exe"
+set CONDA="Miniforge-pypy3-Windows-x86_64.exe"
 curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/%CONDA%
 %CONDA%
 del %CONDA%

install/install_conda.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
-CONDA="Mambaforge-pypy3-$(uname)-$(uname -m).sh"
+CONDA="Miniforge-pypy3-$(uname)-$(uname -m).sh"
 curl -L -O  https://github.com/conda-forge/miniforge/releases/latest/download/"$CONDA"
 chmod +x "$CONDA"
 ./"$CONDA" -b
-~/mambaforge-pypy3/bin/mamba init
+~/Miniforge-pypy3/bin/mamba init
 rm -f "$CONDA"

install/install_pypy.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 # NOTE: mamba is a drop-in replacement for conda, just much faster.
 # (i.e. You can replace mamba with conda below.)
-# See https://github.com/conda-forge/miniforge#mambaforge-pypy3
+# See https://github.com/conda-forge/miniforge#Miniforge-pypy3
 CONDA=conda
 if [ "$(which mamba)" ]; then
     CONDA=mamba

install/install_system_deps.bat

@@ -1,6 +1,6 @@
 :: NOTE: mamba is a drop-in replacement for conda, just much faster.
 :: (i.e. You can replace mamba with conda below.)
-:: See https://github.com/conda-forge/miniforge#mambaforge-pypy3
+:: See https://github.com/conda-forge/miniforge#Miniforge-pypy3
 set CONDA=conda
 where /q mamba
 if not ERRORLEVEL 1 (set CONDA=mamba)

install/install_system_deps.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 # NOTE: mamba is a drop-in replacement for conda, just much faster.
 # (i.e. You can replace mamba with conda below.)
-# See https://github.com/conda-forge/miniforge#mambaforge-pypy3
+# See https://github.com/conda-forge/miniforge#Miniforge-pypy3
 CONDA=conda
 if [ "$(which mamba)" ]; then
     CONDA=mamba

pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
     "typeguard",
     "pydantic>=2.6",
     "pydantic-settings",
+    "pydantic[email]",
     "docker",
     # FYI also need uidmap to run podman rootless
     "podman",
@@ -104,11 +105,6 @@ runners = [
     "toil[cwl]",
     "cwl-utils",
 ]
-runners-src = [
-    "toil[cwl] @ git+https://github.com/sameeul/toil.git",
-    "cwltool @ git+https://github.com/sameeul/cwltool.git",
-    "cwl-utils @ git+https://github.com/sameeul/cwl-utils.git",
-]
 # See docs/requirements.txt
 doc = [
     "sphinx",
@@ -118,7 +114,6 @@ doc = [
 plots = ["matplotlib"]
 cyto = ["ipycytoscape"] # only for DAG visualization
 all_except_runner_src = ["sophios[test,doc,plots,cyto,mypy-types]"]
-all = ["sophios[test,doc,plots,cyto,runners-src,mypy-types]"]
 
 [project.scripts]
 sophios = "sophios.main:main"
@@ -141,7 +136,7 @@ namespaces = true
 
 [tool.setuptools.package-data]
 "*" = ["*.txt"]
-"sophios" = ["*.json"]
+"sophios" = ["**/*.json"]
 
 [tool.aliases]
 test = "pytest --workers 8"