diff --git a/.github/workflows/run_workflows.yml b/.github/workflows/run_workflows.yml index 1b6da8cd..54a7dfb5 100644 --- a/.github/workflows/run_workflows.yml +++ b/.github/workflows/run_workflows.yml @@ -168,13 +168,18 @@ jobs: - name: cwl-docker-extract (i.e. recursively docker pull) if: always() - run: cd workflow-inference-compiler/ && pytest -k test_cwl_docker_extract + run: cd workflow-inference-compiler/ && pytest tests/test_examples.py -k test_cwl_docker_extract # For self-hosted runners, make sure the docker cache is up-to-date. - name: PyTest Run Workflows if: always() # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 - run: cd workflow-inference-compiler/ && pytest -k test_run_workflows_on_push --workers 8 --cwl_runner cwltool # --cov + run: cd workflow-inference-compiler/ && pytest tests/test_examples.py -k test_run_workflows_on_push --workers 8 --cwl_runner cwltool # --cov + + - name: PyTest Run REST Core Tests + if: always() + # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 + run: cd workflow-inference-compiler/ && pytest tests/test_rest_core.py -k test_rest_core --cwl_runner cwltool # NOTE: The steps below are for repository_dispatch only. For all other steps, please insert above # this comment. diff --git a/install/system_deps.yml b/install/system_deps.yml index 022f345f..dda22365 100644 --- a/install/system_deps.yml +++ b/install/system_deps.yml @@ -35,8 +35,10 @@ dependencies: # Similarly, toil[cwl] depends on ruamel.yaml.clib for performance. # Install it with conda/mamba here. - ruamel.yaml.clib -# Simiarly, cryptography needs to build binary wheels +# Similarly, cryptography needs to build binary wheels - cryptography # Needs binary PyQt5 dependencies. - kubernetes-helm - zstandard +# Needed for orjson wheels + - orjson diff --git a/pyproject.toml b/pyproject.toml index 68fad553..bc63ed0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,11 +33,15 @@ dependencies = [ # See https://github.com/common-workflow-language/cwl-utils/releases/ "typeguard", "pydantic>=2.6", + "pydantic-settings", "docker", # FYI also need uidmap to run podman rootless "podman", # We are using the official release for these packages for now "toil[cwl]", + "fastapi", + "python-jose", + "uvicorn" ] [project.readme] diff --git a/src/sophios/api/http/__init__.py b/src/sophios/api/http/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/sophios/api/http/restapi.py b/src/sophios/api/http/restapi.py new file mode 100644 index 00000000..0c108749 --- /dev/null +++ b/src/sophios/api/http/restapi.py @@ -0,0 +1,179 @@ +from pathlib import Path +import argparse +import yaml + +import uvicorn +from fastapi import FastAPI, Request, status +from fastapi.middleware.cors import CORSMiddleware + +from sophios import __version__, compiler +from sophios import run_local, input_output +from sophios.utils_graphs import get_graph_reps +from sophios.utils_yaml import wic_loader +from sophios import utils_cwl +from sophios.cli import get_args +from sophios.wic_types import CompilerInfo, Json, Tool, Tools, StepId, YamlTree, Cwl +# from sophios.api.utils import converter +# from .auth.auth import authenticate + + +# helper functions + + +def remove_dot_dollar(tree: Cwl) -> Cwl: + """Removes . and $ from dictionary keys, e.g. $namespaces and $schemas. Otherwise, you will get + {'error': {'statusCode': 500, 'message': 'Internal Server Error'}} + This is due to MongoDB: + See https://www.mongodb.com/docs/manual/reference/limits/#Restrictions-on-Field-Names + Args: + tree (Cwl): A Cwl document + Returns: + Cwl: A Cwl document with . and $ removed from $namespaces and $schemas + """ + tree_str = str(yaml.dump(tree, sort_keys=False, line_break='\n', indent=2)) + tree_str_no_dd = tree_str.replace('$namespaces', 'namespaces').replace( + '$schemas', 'schemas').replace('.wic', '_wic') + tree_no_dd: Cwl = yaml.load(tree_str_no_dd, Loader=wic_loader()) # This effectively copies tree + return tree_no_dd + + +def get_yaml_tree(req: Json) -> Json: + """ + Get the Sophios yaml tree from incoming JSON + Args: + req (JSON): A raw JSON content of incoming JSON object + Returns: + Cwl: A Cwl document with . and $ removed from $namespaces and $schemas + """ + wkflw_name = "generic_workflow" + # args = converter.get_args(wkflw_name) + # yaml_tree_json: Json = converter.wfb_to_wic(req) + yaml_tree_json: Json = {} + return yaml_tree_json + + +def run_workflow(compiler_info: CompilerInfo, args: argparse.Namespace) -> int: + """ + Get the Sophios yaml tree from incoming JSON + Args: + req (JSON): A raw JSON content of incoming JSON object + Returns: + Cwl: A Cwl document with . and $ removed from $namespaces and $schemas + """ + # ========= WRITE OUT ======================= + input_output.write_to_disk(compiler_info.rose, Path('autogenerated/'), relative_run_path=True) + # ======== TEST RUN ========================= + retval = run_local.run_local(args, compiler_info.rose, args.cachedir, 'cwltool', False) + return retval + + +app = FastAPI() + +origins = ["*"] + +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/", status_code=status.HTTP_200_OK) +# @authenticate +async def root(request: Request) -> Json: + """The api has 1 route: compile + + Returns: + Dict[str, str]: {"message": "The api has 1 route: compile"} + """ + return {"message": "The api has 1 route: compile"} + + +@app.post("/compile") +# @authenticate +async def compile_wf(request: Request) -> Json: + """The compile route compiles the json object from http request object built elsewhere + + Args: + request (Request): request object built elsewhere + + Returns: + compute_workflow (JSON): workflow json object ready to submit to compute + """ + print('---------- Compile Workflow! ---------') + # ========= PROCESS REQUEST OBJECT ========== + req: Json = await request.json() + wkflw_name = "generic_workflow" + args = get_args(wkflw_name) + + workflow_temp = {} + if req["links"] != []: + for node in req["nodes"]: + workflow_temp["id"] = node["id"] + workflow_temp["step"] = node["cwlScript"] # Assume dict form + else: # A single node workflow + node = req["nodes"][0] + workflow_temp = node["cwlScript"] + + workflow_can = utils_cwl.desugar_into_canonical_normal_form(workflow_temp) + + # ========= BUILD WIC COMPILE INPUT ========= + tools_cwl: Tools = {StepId(content["id"], "global"): + Tool(".", content["run"]) for content in workflow_can["steps"]} + # run tag will have the actual CommandLineTool + wic_obj = {'wic': workflow_can.get('wic', {})} + plugin_ns = wic_obj['wic'].get('namespace', 'global') + + graph = get_graph_reps(wkflw_name) + yaml_tree: YamlTree = YamlTree(StepId(wkflw_name, plugin_ns), workflow_can) + + # ========= COMPILE WORKFLOW ================ + compiler_info: CompilerInfo = compiler.compile_workflow(yaml_tree, args, [], [graph], {}, {}, {}, {}, + tools_cwl, True, relative_run_path=True, testing=False) + + # =========== OPTIONAL RUN ============== + print('---------- Run Workflow locally! ---------') + retval = run_workflow(compiler_info, args) + + compute_workflow: Json = {} + compute_workflow["retval"] = str(retval) + return compute_workflow + + +if __name__ == '__main__': + uvicorn.run(app, host="0.0.0.0", port=3000) + + +# # ========= PROCESS COMPILED OBJECT ========= + # sub_node_data: NodeData = compiler_info.rose.data + # yaml_stem = sub_node_data.name + # cwl_tree = sub_node_data.compiled_cwl + # yaml_inputs = sub_node_data.workflow_inputs_file + + # # ======== OUTPUT PROCESSING ================ + # cwl_tree_no_dd = remove_dot_dollar(cwl_tree) + # yaml_inputs_no_dd = remove_dot_dollar(yaml_inputs) + + # # Convert the compiled yaml file to json for labshare Compute. + # cwl_tree_run = copy.deepcopy(cwl_tree_no_dd) + # for step_key in cwl_tree['steps']: + # step_name_i = step_key + # step_name_i = step_name_i.replace('.yml', '_yml') # Due to calling remove_dot_dollar above + # step_name = '__'.join(step_key.split('__')[3:]) # Remove prefix + + # # Get step CWL from templates + # run_val = next((tval['cwlScript'] + # for _, tval in ict_plugins.items() if step_name == tval['name']), None) + # cwl_tree_run['steps'][step_name_i]['run'] = run_val + + # TODO: set name and driver in workflow builder ui + # compute_workflow: Json = {} + # compute_workflow = { + # "name": yaml_stem, + # "driver": "argo", + # # "driver": "cwltool", + # "cwlJobInputs": yaml_inputs_no_dd, + # **cwl_tree_run + # } diff --git a/tests/single_node_helloworld.json b/tests/single_node_helloworld.json new file mode 100644 index 00000000..099bf9ce --- /dev/null +++ b/tests/single_node_helloworld.json @@ -0,0 +1,40 @@ +{ + "nodes": [ + { + "id": 1, + "name": "PythonHelloWorld", + "pluginId": "", + "cwlScript": { + "steps": { + "one": { + "run": { + "baseCommand": [ + "python", + "-c", + "print('hello world'); print('sqr of 7 : %.2f' % 7**2)" + ], + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "inputs": {}, + "outputs": { + "pyout": { + "outputBinding": { + "glob": "output" + }, + "type": "File" + } + }, + "stdout": "output", + "requirements": { + "InlineJavascriptRequirement": {} + } + } + } + } + }, + "settings": {}, + "internal": false + } + ], + "links": [] +} \ No newline at end of file diff --git a/tests/test_rest_core.py b/tests/test_rest_core.py new file mode 100644 index 00000000..f85bf2e9 --- /dev/null +++ b/tests/test_rest_core.py @@ -0,0 +1,166 @@ +import json +# import subprocess as sub +from pathlib import Path +# import signal +# import sys +# from typing import List +# import argparse +import asyncio +from jsonschema import Draft202012Validator + +from fastapi import Request + +import pytest +from sophios.wic_types import Json + + +from sophios.api.http import restapi + +SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Link": { + "properties": { + "id": { + "type": "number" + }, + "inletIndex": { + "type": "number" + }, + "outletIndex": { + "type": "number" + }, + "sourceId": { + "type": "number" + }, + "targetId": { + "type": "number" + }, + "x1": { + "type": "number" + }, + "x2": { + "type": "number" + }, + "y1": { + "type": "number" + }, + "y2": { + "type": "number" + } + }, + "type": "object", + "required": ["id", "inletIndex", "outletIndex", "sourceId", "targetId"] + }, + "NodeSettings": { + "properties": { + "inputs": { + "additionalProperties": { + "$ref": "#/definitions/T" + }, + "type": "object" + }, + "outputs": { + "additionalProperties": { + "$ref": "#/definitions/T" + }, + "type": "object" + } + }, + "type": "object" + }, + "NodeX": { + "properties": { + "expanded": { + "type": "boolean" + }, + "height": { + "type": "number" + }, + "id": { + "type": "number" + }, + "internal": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "pluginId": { + "type": "string" + }, + "settings": { + "$ref": "#/definitions/NodeSettings" + }, + "width": { + "type": "number" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + }, + "z": { + "type": "number" + }, + }, + "type": "object", + "required": ["id", "name", "pluginId", "settings", "internal"] + }, + "T": { + "type": "object" + } + }, + "properties": { + "links": { + "items": { + "$ref": "#/definitions/Link" + }, + "type": "array" + }, + "nodes": { + "items": { + "$ref": "#/definitions/NodeX" + }, + "type": "array" + }, + "selection": { + "items": { + "type": "number" + }, + "type": "array" + } + }, + "type": "object", + "required": ["links", "nodes"] +} + + +@pytest.mark.fast +def test_rest_core_single_node() -> None: + """A simple single node 'hello world' test""" + # validate schema + Draft202012Validator.check_schema(SCHEMA) + df2012 = Draft202012Validator(SCHEMA) + inp_file = "single_node_helloworld.json" + inp: Json = {} + yaml_path = "workflow.json" + inp_path = Path(__file__).with_name(inp_file) + with open(inp_path, 'r', encoding='utf-8') as f: + inp = json.load(f) + # check if object is conformant with our schema + df2012.is_valid(inp) + print('----------- from rest api ----------- \n\n') + scope = {} + scope['type'] = 'http' + + async def receive() -> Json: + inp_byte = json.dumps(inp).encode('utf-8') + return {"type": "http.request", "body": inp_byte} + + # create a request object and pack it with our json payload + req: Request = Request(scope) + req._receive = receive + res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api + assert int(res['retval']) == 0