Merge pull request #303 from sameeul/v021

V021

.github/workflows/fuzzy_compile_weekly.yml

@@ -72,7 +72,7 @@ jobs:
         uses: conda-incubator/setup-miniconda@v3.0.1
         with:
           miniforge-variant: Miniforge-pypy3
-          miniforge-version: latest
+          miniforge-version: 24.7.1-0
           environment-file: workflow-inference-compiler/install/system_deps.yml
           activate-environment: wic
           channels: conda-forge

.github/workflows/lint_and_test.yml

@@ -138,7 +138,7 @@ jobs:
         uses: conda-incubator/setup-miniconda@v3.0.1
         with:
           miniforge-variant: Miniforge-pypy3
-          miniforge-version: latest
+          miniforge-version: 24.7.1-0
           environment-file: workflow-inference-compiler/install/system_deps.yml
           activate-environment: wic
           channels: conda-forge
@@ -149,7 +149,7 @@ jobs:
         uses: conda-incubator/setup-miniconda@v3.0.1
         with:
           miniforge-variant: Miniforge-pypy3
-          miniforge-version: latest
+          miniforge-version: 24.7.1-0
           environment-file: workflow-inference-compiler/install/system_deps_windows.yml
           activate-environment: wic
           channels: conda-forge
@@ -218,13 +218,13 @@ jobs:
         if: always()
         run: cd workflow-inference-compiler/ && pytest -k test_cwl_embedding_independence # --cov --cov-config=.coveragerc_serial
         # NOTE: This test MUST be run in serial! See is_isomorphic_with_timeout()
-        timeout-minutes: 5 # backup timeout for windows
+        timeout-minutes: 20 # backup timeout for windows
 
       - name: PyTest Inline Subworkflows
         if: always()
         run: cd workflow-inference-compiler/ && pytest -k test_inline_subworkflows # --cov --cov-config=.coveragerc_serial
         # NOTE: This test MUST be run in serial! See is_isomorphic_with_timeout()
-        timeout-minutes: 5 # backup timeout for windows
+        timeout-minutes: 20 # backup timeout for windows
 
       - name: PyTest Scattering Scaling
         if: runner.os == 'Linux'

.github/workflows/lint_and_test_macos.yml

@@ -72,7 +72,7 @@ jobs:
         uses: conda-incubator/setup-miniconda@v2.2.0
         with:
           miniforge-variant: Miniforge-pypy3
-          miniforge-version: latest
+          miniforge-version: 24.7.1-0
           environment-file: workflow-inference-compiler/install/system_deps.yml
           activate-environment: wic
           use-mamba: true
@@ -142,10 +142,10 @@ jobs:
         if: always()
         run: cd workflow-inference-compiler/ && pytest -k test_cwl_embedding_independence # --cov --cov-config=.coveragerc_serial
         # NOTE: This test MUST be run in serial! See is_isomorphic_with_timeout()
-        timeout-minutes: 5 # backup timeout for windows
+        timeout-minutes: 20 # backup timeout for windows
 
       - name: PyTest Inline Subworkflows
         if: always()
         run: cd workflow-inference-compiler/ && pytest -k test_inline_subworkflows # --cov --cov-config=.coveragerc_serial
         # NOTE: This test MUST be run in serial! See is_isomorphic_with_timeout()
-        timeout-minutes: 5 # backup timeout for windows
+        timeout-minutes: 20 # backup timeout for windows

.github/workflows/publish_rest_docker.yml

@@ -0,0 +1,40 @@
+name: Publish REST API Docker
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+      - v[0-9]+.[0-9]+.[0-9]+-dev[0-9]+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx 🐳
+        uses: docker/setup-buildx-action@v1
+
+      - name: Get Tag
+        run: echo "tag=${{ github.ref_name }}" >> $GITHUB_ENV
+
+      - name: Print Tag
+        run: echo "Publishing with tag ${{ env.tag }}"
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Publish Sophios Container 🐳
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/Dockerfile_ubuntu_REST
+          push: true
+          tags: polusai/sophios-rest-api:${{ env.tag }}
+

.github/workflows/run_workflows.yml

@@ -127,7 +127,7 @@ jobs:
       uses: conda-incubator/setup-miniconda@v3.0.1
       with:
         miniforge-variant: Miniforge-pypy3
-        miniforge-version: latest
+        miniforge-version: 24.7.1-0
         environment-file: workflow-inference-compiler/install/system_deps.yml
         activate-environment: wic_github_actions
         channels: conda-forge

.github/workflows/run_workflows_weekly.yml

@@ -89,7 +89,7 @@ jobs:
         # installs pypy in the base environment (only). Although we are using
         # another environment, better to avoid the problem altogether by
         # not using Miniforge-pypy3
-        miniforge-version: latest
+        miniforge-version: 24.7.1-0
         environment-file: workflow-inference-compiler/install/system_deps.yml
         activate-environment: wic_github_actions
         use-mamba: true

docker/Dockerfile_ubuntu_REST

@@ -0,0 +1,19 @@
+FROM ubuntu:jammy
+
+RUN apt update && \
+    apt install software-properties-common -y && \
+    apt install curl -y && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python3 get-pip.py && \
+    apt autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY . /sophios
+WORKDIR /sophios
+
+RUN pip3 install /sophios --no-cache-dir
+
+# Then run the sophios REST API through port 3000
+EXPOSE 3000
+
+CMD ["uvicorn", "sophios.api.http.restapi:app", "--host", "0.0.0.0", "--port", "3000"]

docker/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '0.2.1'
+services:
+  fastapi-app:
+    image: polusai/sophios-rest-api:0.2.1
+    ports:
+      - "3000:3000"
+    environment:
+      - PATH=$PATH
+    command: ["uvicorn", "sophios.api.http.restapi:app", "--host", "0.0.0.0", "--port", "3000"]

pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
     # This 'graphviz' is equivalent to `conda install python-graphviz` or
     # `sudo apt install python3-graphviz` ONLY.
     "graphviz",
-    "jsonschema<4.18", # temporarily downgrade due to severe performance regression
+    "jsonschema",
     "pyyaml",
     "requests",
     "mergedeep",
@@ -42,7 +42,8 @@ dependencies = [
     "toil[cwl]",
     "fastapi",
     "python-jose",
-    "uvicorn"
+    "uvicorn",
+    "referencing"
 ]
 
 [project.readme]

src/sophios/api/http/restapi.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 import argparse
 import copy
+import uuid
 import yaml
 
 
@@ -13,7 +14,7 @@
 from sophios.utils_graphs import get_graph_reps
 from sophios.utils_yaml import wic_loader
 from sophios import utils_cwl
-from sophios.post_compile import cwl_inline_runtag
+from sophios.post_compile import cwl_inline_runtag, remove_entrypoints
 from sophios.cli import get_args
 from sophios.wic_types import CompilerInfo, Json, Tool, Tools, StepId, YamlTree, Cwl, NodeData
 from sophios.api.utils import converter
@@ -94,14 +95,15 @@ async def compile_wf(request: Request) -> Json:
     print('---------- Compile Workflow! ---------')
     # ========= PROCESS REQUEST OBJECT ==========
     req: Json = await request.json()
+    suppliedargs = ['--cwl_inline_runtag', '--generate_cwl_workflow']
     # clean up and convert the incoming object
     # schema preserving
     req = converter.update_payload_missing_inputs_outputs(req)
     wfb_payload = converter.raw_wfb_to_lean_wfb(req)
     # schema non-preserving
-    workflow_temp = converter.wfb_to_wic(wfb_payload)
-    wkflw_name = "generic_workflow"
-    args = get_args(wkflw_name, ['--inline_cwl_runtag', '--generate_cwl_workflow'])
+    workflow_temp = converter.wfb_to_wic(wfb_payload, req["plugins"])
+    wkflw_name = "workflow_"
+    args = get_args(wkflw_name, suppliedargs)
 
     # Build canonical workflow object
     workflow_can = utils_cwl.desugar_into_canonical_normal_form(workflow_temp)
@@ -128,29 +130,48 @@ async def compile_wf(request: Request) -> Json:
     yaml_tree: YamlTree = YamlTree(StepId(wkflw_name, plugin_ns), workflow_can)
 
     # ========= COMPILE WORKFLOW ================
-    args.ignore_dir_path = True
+    if req.get('run_local_env') == 'true':
+        args.ignore_dir_path = False
+    else:
+        args.ignore_dir_path = True
     compiler_info: CompilerInfo = compiler.compile_workflow(yaml_tree, args, [], [graph], {}, {}, {}, {},
                                                             tools_cwl, True, relative_run_path=True, testing=False)
 
     rose_tree = compiler_info.rose
     input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
-    cwl_inline_runtag(args, rose_tree)
+    rose_tree = cwl_inline_runtag(args, rose_tree)
+    rose_tree = remove_entrypoints(args, rose_tree)
     # ======== OUTPUT PROCESSING ================
     # ========= PROCESS COMPILED OBJECT =========
     sub_node_data: NodeData = rose_tree.data
     yaml_stem = sub_node_data.name
     cwl_tree = sub_node_data.compiled_cwl
     yaml_inputs = sub_node_data.workflow_inputs_file
-    cwl_tree_no_dd = remove_dot_dollar(cwl_tree)
-    yaml_inputs_no_dd = remove_dot_dollar(yaml_inputs)
 
     # Convert the compiled yaml file to json for labshare Compute.
-    cwl_tree_run = copy.deepcopy(cwl_tree_no_dd)
-
+    cwl_tree_run = copy.deepcopy(cwl_tree)
+    cwl_tree_run['steps_dict'] = {}
+    for step in cwl_tree_run['steps']:
+        node_name = step['id']
+        step.pop('id', None)
+        step = {node_name: step}
+        step_copy = copy.deepcopy(step)
+        cwl_tree_run['steps_dict'].update(step_copy)
+
+    cwl_tree_run.pop('steps', None)
+    cwl_tree_run['steps'] = cwl_tree_run.pop('steps_dict', None)
+
+    # currently there is a compiler bug where the output variables are duplicated
+    # this is a workaround to remove the duplicates till the compiler is fixed
+    for step in cwl_tree_run['steps']:
+
+        out_vars = cwl_tree_run['steps'][step]['out']
+        out_vars_unique = list(set(out_vars))
+        cwl_tree_run['steps'][step]['out'] = out_vars_unique
     compute_workflow: Json = {}
     compute_workflow = {
         "name": yaml_stem,
-        "cwlJobInputs": yaml_inputs_no_dd,
+        "cwlJobInputs": yaml_inputs,
         **cwl_tree_run
     }
     compute_workflow["retval"] = str(0)

src/sophios/api/pythonapi.py

@@ -2,7 +2,6 @@
 """CLT utilities."""
 import logging
 from pathlib import Path
-import subprocess as sub
 from typing import Any, ClassVar, Optional, TypeVar, Union
 
 import cwl_utils.parser as cu_parser
@@ -757,8 +756,8 @@ def run(self) -> None:
         rose_tree: RoseTree = compiler_info.rose
 
         post_compile.cwl_docker_extract(args, self.process_name)
-        post_compile.remove_entrypoints(args, rose_tree)
-
+        rose_tree = post_compile.remove_entrypoints(args, rose_tree)
+        post_compile.find_and_create_output_dirs(rose_tree)
         # Do NOT capture stdout and/or stderr and pipe warnings and errors into a black hole.
         retval = run_local_module.run_local(args, rose_tree, args.cachedir, args.cwl_runner, True)