add required files for rest api (core) (#247)

* add required files for rest api (core)

* add a single node/step workflow test rest api

* add orjson as dep and pytest fix gha

---------

Co-authored-by: Vasu Jaganath <vasu.jaganath@axleinfo.com>

.github/workflows/run_workflows.yml

@@ -168,13 +168,18 @@ jobs:
 
     - name: cwl-docker-extract (i.e. recursively docker pull)
       if: always()
-      run: cd workflow-inference-compiler/ && pytest -k test_cwl_docker_extract
+      run: cd workflow-inference-compiler/ && pytest tests/test_examples.py -k test_cwl_docker_extract
       # For self-hosted runners, make sure the docker cache is up-to-date.
 
     - name: PyTest Run Workflows
       if: always()
       # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
-      run: cd workflow-inference-compiler/ && pytest -k test_run_workflows_on_push --workers 8 --cwl_runner cwltool # --cov
+      run: cd workflow-inference-compiler/ && pytest tests/test_examples.py -k test_run_workflows_on_push --workers 8 --cwl_runner cwltool # --cov
+
+    - name: PyTest Run REST Core Tests
+      if: always()
+      # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
+      run: cd workflow-inference-compiler/ && pytest tests/test_rest_core.py -k test_rest_core --cwl_runner cwltool
 
     # NOTE: The steps below are for repository_dispatch only. For all other steps, please insert above
     # this comment.

install/system_deps.yml

@@ -35,8 +35,10 @@ dependencies:
 # Similarly, toil[cwl] depends on ruamel.yaml.clib for performance.
 # Install it with conda/mamba here.
   - ruamel.yaml.clib
-# Simiarly, cryptography needs to build binary wheels
+# Similarly, cryptography needs to build binary wheels
   - cryptography
 # Needs binary PyQt5 dependencies.
   - kubernetes-helm
   - zstandard
+# Needed for orjson wheels
+  - orjson

pyproject.toml

@@ -33,11 +33,15 @@ dependencies = [
     # See https://github.com/common-workflow-language/cwl-utils/releases/
     "typeguard",
     "pydantic>=2.6",
+    "pydantic-settings",
     "docker",
     # FYI also need uidmap to run podman rootless
     "podman",
     # We are using the official release for these packages for now
     "toil[cwl]",
+    "fastapi",
+    "python-jose",
+    "uvicorn"
 ]
 
 [project.readme]

src/sophios/api/http/restapi.py

@@ -0,0 +1,179 @@
+from pathlib import Path
+import argparse
+import yaml
+
+import uvicorn
+from fastapi import FastAPI, Request, status
+from fastapi.middleware.cors import CORSMiddleware
+
+from sophios import __version__, compiler
+from sophios import run_local, input_output
+from sophios.utils_graphs import get_graph_reps
+from sophios.utils_yaml import wic_loader
+from sophios import utils_cwl
+from sophios.cli import get_args
+from sophios.wic_types import CompilerInfo, Json, Tool, Tools, StepId, YamlTree, Cwl
+# from sophios.api.utils import converter
+# from .auth.auth import authenticate
+
+
+# helper functions
+
+
+def remove_dot_dollar(tree: Cwl) -> Cwl:
+    """Removes . and $ from dictionary keys, e.g. $namespaces and $schemas. Otherwise, you will get
+    {'error': {'statusCode': 500, 'message': 'Internal Server Error'}}
+    This is due to MongoDB:
+    See https://www.mongodb.com/docs/manual/reference/limits/#Restrictions-on-Field-Names
+    Args:
+        tree (Cwl): A Cwl document
+    Returns:
+        Cwl: A Cwl document with . and $ removed from $namespaces and $schemas
+    """
+    tree_str = str(yaml.dump(tree, sort_keys=False, line_break='\n', indent=2))
+    tree_str_no_dd = tree_str.replace('$namespaces', 'namespaces').replace(
+        '$schemas', 'schemas').replace('.wic', '_wic')
+    tree_no_dd: Cwl = yaml.load(tree_str_no_dd, Loader=wic_loader())  # This effectively copies tree
+    return tree_no_dd
+
+
+def get_yaml_tree(req: Json) -> Json:
+    """
+    Get the Sophios yaml tree from incoming JSON
+    Args:
+        req (JSON): A raw JSON content of incoming JSON object
+    Returns:
+        Cwl: A Cwl document with . and $ removed from $namespaces and $schemas
+    """
+    wkflw_name = "generic_workflow"
+    # args = converter.get_args(wkflw_name)
+    # yaml_tree_json: Json = converter.wfb_to_wic(req)
+    yaml_tree_json: Json = {}
+    return yaml_tree_json
+
+
+def run_workflow(compiler_info: CompilerInfo, args: argparse.Namespace) -> int:
+    """
+    Get the Sophios yaml tree from incoming JSON
+    Args:
+        req (JSON): A raw JSON content of incoming JSON object
+    Returns:
+        Cwl: A Cwl document with . and $ removed from $namespaces and $schemas
+    """
+    # ========= WRITE OUT =======================
+    input_output.write_to_disk(compiler_info.rose, Path('autogenerated/'), relative_run_path=True)
+    # ======== TEST RUN =========================
+    retval = run_local.run_local(args, compiler_info.rose, args.cachedir, 'cwltool', False)
+    return retval
+
+
+app = FastAPI()
+
+origins = ["*"]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/", status_code=status.HTTP_200_OK)
+# @authenticate
+async def root(request: Request) -> Json:
+    """The api has 1 route: compile
+
+    Returns:
+        Dict[str, str]: {"message": "The api has 1 route: compile"}
+    """
+    return {"message": "The api has 1 route: compile"}
+
+
+@app.post("/compile")
+# @authenticate
+async def compile_wf(request: Request) -> Json:
+    """The compile route compiles the json object from http request object built elsewhere
+
+    Args:
+        request (Request): request object built elsewhere
+
+    Returns:
+        compute_workflow (JSON): workflow json object ready to submit to compute
+    """
+    print('---------- Compile Workflow! ---------')
+    # ========= PROCESS REQUEST OBJECT ==========
+    req: Json = await request.json()
+    wkflw_name = "generic_workflow"
+    args = get_args(wkflw_name)
+
+    workflow_temp = {}
+    if req["links"] != []:
+        for node in req["nodes"]:
+            workflow_temp["id"] = node["id"]
+            workflow_temp["step"] = node["cwlScript"]  # Assume dict form
+    else:  # A single node workflow
+        node = req["nodes"][0]
+        workflow_temp = node["cwlScript"]
+
+    workflow_can = utils_cwl.desugar_into_canonical_normal_form(workflow_temp)
+
+    # ========= BUILD WIC COMPILE INPUT =========
+    tools_cwl: Tools = {StepId(content["id"], "global"):
+                        Tool(".", content["run"]) for content in workflow_can["steps"]}
+    # run tag will have the actual CommandLineTool
+    wic_obj = {'wic': workflow_can.get('wic', {})}
+    plugin_ns = wic_obj['wic'].get('namespace', 'global')
+
+    graph = get_graph_reps(wkflw_name)
+    yaml_tree: YamlTree = YamlTree(StepId(wkflw_name, plugin_ns), workflow_can)
+
+    # ========= COMPILE WORKFLOW ================
+    compiler_info: CompilerInfo = compiler.compile_workflow(yaml_tree, args, [], [graph], {}, {}, {}, {},
+                                                            tools_cwl, True, relative_run_path=True, testing=False)
+
+    # =========== OPTIONAL RUN ==============
+    print('---------- Run Workflow locally! ---------')
+    retval = run_workflow(compiler_info, args)
+
+    compute_workflow: Json = {}
+    compute_workflow["retval"] = str(retval)
+    return compute_workflow
+
+
+if __name__ == '__main__':
+    uvicorn.run(app, host="0.0.0.0", port=3000)
+
+
+# # ========= PROCESS COMPILED OBJECT =========
+    # sub_node_data: NodeData = compiler_info.rose.data
+    # yaml_stem = sub_node_data.name
+    # cwl_tree = sub_node_data.compiled_cwl
+    # yaml_inputs = sub_node_data.workflow_inputs_file
+
+    # # ======== OUTPUT PROCESSING ================
+    # cwl_tree_no_dd = remove_dot_dollar(cwl_tree)
+    # yaml_inputs_no_dd = remove_dot_dollar(yaml_inputs)
+
+    # # Convert the compiled yaml file to json for labshare Compute.
+    # cwl_tree_run = copy.deepcopy(cwl_tree_no_dd)
+    # for step_key in cwl_tree['steps']:
+    #     step_name_i = step_key
+    #     step_name_i = step_name_i.replace('.yml', '_yml')  # Due to calling remove_dot_dollar above
+    #     step_name = '__'.join(step_key.split('__')[3:])  # Remove prefix
+
+    #     # Get step CWL from templates
+    #     run_val = next((tval['cwlScript']
+    #                    for _, tval in ict_plugins.items() if step_name == tval['name']), None)
+    #     cwl_tree_run['steps'][step_name_i]['run'] = run_val
+
+    # TODO: set name and driver in workflow builder ui
+    # compute_workflow: Json = {}
+    # compute_workflow = {
+    #     "name": yaml_stem,
+    #     "driver": "argo",
+    #     # "driver": "cwltool",
+    #     "cwlJobInputs": yaml_inputs_no_dd,
+    #     **cwl_tree_run
+    # }

tests/single_node_helloworld.json

@@ -0,0 +1,40 @@
+{
+	"nodes": [
+		{
+			"id": 1,
+			"name": "PythonHelloWorld",
+			"pluginId": "",
+			"cwlScript": {
+				"steps": {
+					"one": {
+						"run": {
+							"baseCommand": [
+								"python",
+								"-c",
+								"print('hello world'); print('sqr of 7 : %.2f' % 7**2)"
+							],
+							"class": "CommandLineTool",
+							"cwlVersion": "v1.2",
+							"inputs": {},
+							"outputs": {
+								"pyout": {
+									"outputBinding": {
+										"glob": "output"
+									},
+									"type": "File"
+								}
+							},
+							"stdout": "output",
+							"requirements": {
+								"InlineJavascriptRequirement": {}
+							}
+						}
+					}
+				}
+			},
+			"settings": {},
+			"internal": false
+		}
+	],
+	"links": []
+}