From 49935d1be0b408474bca877d24a594e9cdb71e75 Mon Sep 17 00:00:00 2001 From: VasuJ <145879890+vjaganat90@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:49:58 -0400 Subject: [PATCH 01/15] REST API : wfb to wic transformation and compile (with tests) (#270) Co-authored-by: Vasu Jaganath --- src/sophios/api/http/restapi.py | 87 ++------- src/sophios/api/utils/converter.py | 129 ++++++------ tests/rest_wfb_objects/multi_node.json | 76 ++++++++ .../multi_node_inline_cwl.json | 181 +++++++++++++++++ .../single_node.json} | 0 tests/test_rest_core.py | 184 +++++------------- 6 files changed, 394 insertions(+), 263 deletions(-) create mode 100644 tests/rest_wfb_objects/multi_node.json create mode 100644 tests/rest_wfb_objects/multi_node_inline_cwl.json rename tests/{single_node_helloworld.json => rest_wfb_objects/single_node.json} (100%) diff --git a/src/sophios/api/http/restapi.py b/src/sophios/api/http/restapi.py index f7e49b8c..f2bb950f 100644 --- a/src/sophios/api/http/restapi.py +++ b/src/sophios/api/http/restapi.py @@ -16,6 +16,7 @@ from sophios.cli import get_args from sophios.wic_types import CompilerInfo, Json, Tool, Tools, StepId, YamlTree, Cwl, NodeData from sophios.api.utils import converter +import sophios.plugins as plugins # from .auth.auth import authenticate @@ -39,21 +40,6 @@ def remove_dot_dollar(tree: Cwl) -> Cwl: return tree_no_dd -def get_yaml_tree(req: Json) -> Json: - """ - Get the Sophios yaml tree from incoming JSON - Args: - req (JSON): A raw JSON content of incoming JSON object - Returns: - Cwl: A Cwl document with . and $ removed from $namespaces and $schemas - """ - wkflw_name = "generic_workflow" - # args = converter.get_args(wkflw_name) - # yaml_tree_json: Json = converter.wfb_to_wic(req) - yaml_tree_json: Json = {} - return yaml_tree_json - - def run_workflow(compiler_info: CompilerInfo, args: argparse.Namespace) -> int: """ Get the Sophios yaml tree from incoming JSON @@ -108,25 +94,31 @@ async def compile_wf(request: Request) -> Json: # ========= PROCESS REQUEST OBJECT ========== req: Json = await request.json() # clean up and convert the incoming object + # schema preserving req = converter.raw_wfb_to_lean_wfb(req) + # schema non-preserving + workflow_temp = converter.wfb_to_wic(req) wkflw_name = "generic_workflow" - args = get_args(wkflw_name) - - workflow_temp = {} - if req["links"] != []: - for node in req["nodes"]: - workflow_temp["id"] = node["id"] - workflow_temp["step"] = node["cwlScript"] # Assume dict form - else: # A single node workflow - node = req["nodes"][0] - workflow_temp = node["cwlScript"] + args = get_args(wkflw_name, ['--inline_cwl_runtag']) + # Build canonical workflow object workflow_can = utils_cwl.desugar_into_canonical_normal_form(workflow_temp) # ========= BUILD WIC COMPILE INPUT ========= - tools_cwl: Tools = {StepId(content["id"], "global"): - Tool(".", content["run"]) for content in workflow_can["steps"]} + # Build a list of CLTs + # The default list + tools_cwl: Tools = {} + global_config = input_output.get_config(Path(args.config_file), Path(args.homedir)/'wic'/'global_config.json') + tools_cwl = plugins.get_tools_cwl(global_config, + args.validate_plugins, + not args.no_skip_dollar_schemas, + args.quiet) + # Add to the default list if the tool is 'inline' in run tag # run tag will have the actual CommandLineTool + for can_step in workflow_can["steps"]: + if can_step.get("run", None): + # add a new tool + tools_cwl[StepId(can_step["id"], "global")] = Tool(".", can_step["run"]) wic_obj = {'wic': workflow_can.get('wic', {})} plugin_ns = wic_obj['wic'].get('namespace', 'global') @@ -152,14 +144,6 @@ async def compile_wf(request: Request) -> Json: # Convert the compiled yaml file to json for labshare Compute. cwl_tree_run = copy.deepcopy(cwl_tree_no_dd) - # for step_key in cwl_tree['steps']: - # step_name_i = step_key - # step_name_i = step_name_i.replace('.yml', '_yml') # Due to calling remove_dot_dollar above - # # step_name = '__'.join(step_key.split('__')[3:]) # Remove prefix - # # Get step CWL from templates - # # run_val = next((tval['cwlScript'] - # # for _, tval in ict_plugins.items() if step_name == tval['name']), None) - # # cwl_tree_run['steps'][step_name_i]['run'] = run_val compute_workflow: Json = {} compute_workflow = { @@ -173,36 +157,3 @@ async def compile_wf(request: Request) -> Json: if __name__ == '__main__': uvicorn.run(app, host="0.0.0.0", port=3000) - - -# # ========= PROCESS COMPILED OBJECT ========= - # sub_node_data: NodeData = compiler_info.rose.data - # yaml_stem = sub_node_data.name - # cwl_tree = sub_node_data.compiled_cwl - # yaml_inputs = sub_node_data.workflow_inputs_file - - # # ======== OUTPUT PROCESSING ================ - # cwl_tree_no_dd = remove_dot_dollar(cwl_tree) - # yaml_inputs_no_dd = remove_dot_dollar(yaml_inputs) - - # # Convert the compiled yaml file to json for labshare Compute. - # cwl_tree_run = copy.deepcopy(cwl_tree_no_dd) - # for step_key in cwl_tree['steps']: - # step_name_i = step_key - # step_name_i = step_name_i.replace('.yml', '_yml') # Due to calling remove_dot_dollar above - # step_name = '__'.join(step_key.split('__')[3:]) # Remove prefix - - # # Get step CWL from templates - # run_val = next((tval['cwlScript'] - # for _, tval in ict_plugins.items() if step_name == tval['name']), None) - # cwl_tree_run['steps'][step_name_i]['run'] = run_val - - # TODO: set name and driver in workflow builder ui - # compute_workflow: Json = {} - # compute_workflow = { - # "name": yaml_stem, - # "driver": "argo", - # # "driver": "cwltool", - # "cwlJobInputs": yaml_inputs_no_dd, - # **cwl_tree_run - # } diff --git a/src/sophios/api/utils/converter.py b/src/sophios/api/utils/converter.py index 67289d87..8a1e5ba6 100644 --- a/src/sophios/api/utils/converter.py +++ b/src/sophios/api/utils/converter.py @@ -1,10 +1,10 @@ import copy from typing import Any, Dict, List - +import yaml from jsonschema import Draft202012Validator +from sophios.utils_yaml import wic_loader -# from sophios import cli -from sophios.wic_types import Json +from sophios.wic_types import Json, Cwl SCHEMA: Json = { "$schema": "http://json-schema.org/draft-07/schema#", @@ -153,7 +153,7 @@ def raw_wfb_to_lean_wfb(inp: Json) -> Json: prop_req = SCHEMA['required'] nodes_req = SCHEMA['definitions']['NodeX']['required'] links_req = SCHEMA['definitions']['Link']['required'] - do_not_rem_nodes_prop = ['cwlScript'] + do_not_rem_nodes_prop = ['cwlScript', 'run'] do_not_rem_links_prop: list = [] for k in keys: @@ -174,59 +174,68 @@ def raw_wfb_to_lean_wfb(inp: Json) -> Json: return inp_restrict -# def wfb_to_wic(request: Json) -> Json: -# """Convert the json object from http request object to a json object that can be used as input to wic compiler. - -# Args: -# request (Json): json object from http request - -# Returns: -# converted_json (Json): json object that can be used as input to wic compiler""" - -# converted_steps: list[Any] = [] - -# for step in request['steps']: -# step_template = step['template'] -# arguments = step['arguments'] -# # Get the template name from the step template -# template_name = next((tval['name'] -# for tname, tval in request['templates'].items() if step_template == tname), None) -# # template_name = None -# # for tname, tval in request['templates'].items(): -# # if tname == step_template and tval['name']: -# # template_name = tval['name'] -# # break -# # elif tname == step_template and not tval['name']: -# # break -# # else: -# # pass - -# converted_step: Json = {} -# if template_name: -# converted_step[template_name] = { -# "in": {} -# } - -# for key, value in arguments.items(): -# # be aware of usage of periods in the values as delimiters, this may cause an issue when storing in MongoDB -# if value.startswith("steps."): -# parts = value.split('.') -# src_step_idx = int(parts[1][len("step"):]) -# src_output_key = parts[3] - -# # Get the src template name from the src step name -# src_template = next((step.get("template") -# for step in request['steps'] if step.get("name") == parts[1]), None) -# src_template_name = next((stval['name'] for stname, stval in request['templates'].items() -# if src_template == stname), None) -# src_converted_step = next((step.get(src_template_name) -# for step in converted_steps if step.get(src_template_name)), None) -# if src_converted_step: -# src_converted_step["in"][src_output_key] = f"&{src_template_name}.{src_output_key}.{src_step_idx}" -# converted_step[template_name]["in"][key] = f"*{src_template_name}.{src_output_key}.{src_step_idx}" -# else: -# converted_step[template_name]["in"][key] = value -# converted_steps.append(converted_step) - -# converted_json: Json = {"steps": converted_steps} -# return converted_json +def wfb_to_wic(inp: Json) -> Cwl: + """convert lean wfb json to compliant wic""" + # non-schema preserving changes + inp_restrict = copy.deepcopy(inp) + + for node in inp_restrict['nodes']: + if node.get('settings'): + node['in'] = node['settings'].get('inputs') + if node['settings'].get('outputs'): + node['out'] = list({k: yaml.load('!& ' + v, Loader=wic_loader())} for k, v in node['settings'] + ['outputs'].items()) # outputs always have to be list + # remove these (now) superfluous keys + node.pop('settings', None) + node.pop('pluginId', None) + node.pop('internal', None) + + # setting the inputs of the non-sink nodes i.e. whose input doesn't depend on any other node's output + # first get all target node ids + target_node_ids = [] + for edg in inp_restrict['links']: + target_node_ids.append(edg['targetId']) + # now set inputs on non-sink nodes as inline input '!ii ' + # if inputs exist + non_sink_nodes = [node for node in inp_restrict['nodes'] if node['id'] not in target_node_ids] + for node in non_sink_nodes: + if node.get('in'): + for nkey in node['in']: + node['in'][nkey] = yaml.load('!ii ' + node['in'][nkey], Loader=wic_loader()) + + # After outs are set + for edg in inp_restrict['links']: + # links = edge. nodes and edges is the correct terminology! + src_id = edg['sourceId'] + tgt_id = edg['targetId'] + src_node = next((node for node in inp_restrict['nodes'] if node['id'] == src_id), None) + tgt_node = next((node for node in inp_restrict['nodes'] if node['id'] == tgt_id), None) + assert src_node, f'output(s) of source node of edge{edg} must exist!' + assert tgt_node, f'input(s) of target node of edge{edg} must exist!' + # flattened list of keys + if src_node.get('out') and tgt_node.get('in'): + src_out_keys = [sk for sout in src_node['out'] for sk in sout.keys()] + tgt_in_keys = tgt_node['in'].keys() + # we match the source output tag type to target input tag type + # and connect them through '!* ' for input, all outputs are '!& ' before this + for sk in src_out_keys: + tgt_node['in'][sk] = yaml.load('!* ' + tgt_node['in'][sk], Loader=wic_loader()) + # the inputs which aren't dependent on previous/other steps + # they are by default inline input + diff_keys = set(tgt_in_keys) - set(src_out_keys) + for dfk in diff_keys: + tgt_node['in'][dfk] = yaml.load('!ii ' + tgt_node['in'][dfk], Loader=wic_loader()) + + for node in inp_restrict['nodes']: + node['id'] = node['name'] # just reuse name as node's id, wic id is same as wfb name + node.pop('name', None) + + workflow_temp: Cwl = {} + if inp_restrict["links"] != []: + workflow_temp["steps"] = [] + for node in inp_restrict["nodes"]: + workflow_temp["steps"].append(node) # node["cwlScript"] # Assume dict form + else: # A single node workflow + node = inp_restrict["nodes"][0] + workflow_temp = node["cwlScript"] + return workflow_temp diff --git a/tests/rest_wfb_objects/multi_node.json b/tests/rest_wfb_objects/multi_node.json new file mode 100644 index 00000000..ffe42014 --- /dev/null +++ b/tests/rest_wfb_objects/multi_node.json @@ -0,0 +1,76 @@ +{ + "nodes": [ + { + "id": 7, + "x": 462, + "y": 206, + "z": 2, + "name": "touch", + "pluginId": "touch", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "filename": "empty.txt" + }, + "outputs": { + "file": "file_touch" + } + }, + "internal": false + }, + { + "id": 18, + "x": 155, + "y": 195, + "z": 1, + "name": "append", + "pluginId": "append", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "str": "Hello", + "file": "file_touch" + }, + "outputs": { + "file": "file_append1" + } + }, + "internal": false + }, + { + "id": 9, + "x": 790.3254637299812, + "y": 449.8103498684344, + "z": 5, + "name": "append", + "pluginId": "append", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "str": "World!", + "file": "file_append1" + }, + "outputs": { + "file": "file_append2" + } + }, + "internal": false + } + ], + "links": [ + { + "sourceId": 7, + "targetId": 18, + "id": 1 + }, + { + "sourceId": 18, + "targetId": 9, + "id": 5 + } + ], + "selection": [] +} \ No newline at end of file diff --git a/tests/rest_wfb_objects/multi_node_inline_cwl.json b/tests/rest_wfb_objects/multi_node_inline_cwl.json new file mode 100644 index 00000000..c972ee36 --- /dev/null +++ b/tests/rest_wfb_objects/multi_node_inline_cwl.json @@ -0,0 +1,181 @@ +{ + "nodes": [ + { + "id": 7, + "x": 462, + "y": 206, + "z": 2, + "name": "touch", + "pluginId": "touch", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "filename": "empty.txt" + }, + "outputs": { + "file": "file_touch" + } + }, + "run": { + "cwlVersion": "v1.0", + "class": "CommandLineTool", + "requirements": { + "DockerRequirement": { + "dockerPull": "docker.io/bash:4.4" + }, + "InlineJavascriptRequirement": {} + }, + "baseCommand": "touch", + "inputs": { + "filename": { + "type": "string", + "inputBinding": { + "position": 1 + } + } + }, + "outputs": { + "file": { + "type": "File", + "outputBinding": { + "glob": "$(inputs.filename)" + } + } + } + }, + "internal": false + }, + { + "id": 18, + "x": 155, + "y": 195, + "z": 1, + "name": "append", + "pluginId": "append", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "str": "Hello", + "file": "file_touch" + }, + "outputs": { + "file": "file_append1" + } + }, + "run": { + "class": "CommandLineTool", + "cwlVersion": "v1.0", + "requirements": { + "ShellCommandRequirement": {}, + "InlineJavascriptRequirement": {}, + "InitialWorkDirRequirement": { + "listing": [ + "$(inputs.file)" + ] + } + }, + "inputs": { + "str": { + "type": "string", + "inputBinding": { + "shellQuote": false, + "position": 1, + "prefix": "echo" + } + }, + "file": { + "type": "File", + "inputBinding": { + "shellQuote": false, + "position": 2, + "prefix": ">>" + } + } + }, + "outputs": { + "file": { + "type": "File", + "outputBinding": { + "glob": "$(inputs.file.basename)" + } + } + } + }, + "internal": false + }, + { + "id": 9, + "x": 790.3254637299812, + "y": 449.8103498684344, + "z": 5, + "name": "append", + "pluginId": "append", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "str": "World!", + "file": "file_append1" + }, + "outputs": { + "file": "file_append2" + } + }, + "run": { + "class": "CommandLineTool", + "cwlVersion": "v1.0", + "requirements": { + "ShellCommandRequirement": {}, + "InlineJavascriptRequirement": {}, + "InitialWorkDirRequirement": { + "listing": [ + "$(inputs.file)" + ] + } + }, + "inputs": { + "str": { + "type": "string", + "inputBinding": { + "shellQuote": false, + "position": 1, + "prefix": "echo" + } + }, + "file": { + "type": "File", + "inputBinding": { + "shellQuote": false, + "position": 2, + "prefix": ">>" + } + } + }, + "outputs": { + "file": { + "type": "File", + "outputBinding": { + "glob": "$(inputs.file.basename)" + } + } + } + }, + "internal": false + } + ], + "links": [ + { + "sourceId": 7, + "targetId": 18, + "id": 1 + }, + { + "sourceId": 18, + "targetId": 9, + "id": 5 + } + ], + "selection": [] +} \ No newline at end of file diff --git a/tests/single_node_helloworld.json b/tests/rest_wfb_objects/single_node.json similarity index 100% rename from tests/single_node_helloworld.json rename to tests/rest_wfb_objects/single_node.json diff --git a/tests/test_rest_core.py b/tests/test_rest_core.py index f85bf2e9..54639d02 100644 --- a/tests/test_rest_core.py +++ b/tests/test_rest_core.py @@ -1,12 +1,6 @@ import json -# import subprocess as sub from pathlib import Path -# import signal -# import sys -# from typing import List -# import argparse import asyncio -from jsonschema import Draft202012Validator from fastapi import Request @@ -16,141 +10,61 @@ from sophios.api.http import restapi -SCHEMA = { - "$schema": "http://json-schema.org/draft-07/schema#", - "definitions": { - "Link": { - "properties": { - "id": { - "type": "number" - }, - "inletIndex": { - "type": "number" - }, - "outletIndex": { - "type": "number" - }, - "sourceId": { - "type": "number" - }, - "targetId": { - "type": "number" - }, - "x1": { - "type": "number" - }, - "x2": { - "type": "number" - }, - "y1": { - "type": "number" - }, - "y2": { - "type": "number" - } - }, - "type": "object", - "required": ["id", "inletIndex", "outletIndex", "sourceId", "targetId"] - }, - "NodeSettings": { - "properties": { - "inputs": { - "additionalProperties": { - "$ref": "#/definitions/T" - }, - "type": "object" - }, - "outputs": { - "additionalProperties": { - "$ref": "#/definitions/T" - }, - "type": "object" - } - }, - "type": "object" - }, - "NodeX": { - "properties": { - "expanded": { - "type": "boolean" - }, - "height": { - "type": "number" - }, - "id": { - "type": "number" - }, - "internal": { - "type": "boolean" - }, - "name": { - "type": "string" - }, - "pluginId": { - "type": "string" - }, - "settings": { - "$ref": "#/definitions/NodeSettings" - }, - "width": { - "type": "number" - }, - "x": { - "type": "number" - }, - "y": { - "type": "number" - }, - "z": { - "type": "number" - }, - }, - "type": "object", - "required": ["id", "name", "pluginId", "settings", "internal"] - }, - "T": { - "type": "object" - } - }, - "properties": { - "links": { - "items": { - "$ref": "#/definitions/Link" - }, - "type": "array" - }, - "nodes": { - "items": { - "$ref": "#/definitions/NodeX" - }, - "type": "array" - }, - "selection": { - "items": { - "type": "number" - }, - "type": "array" - } - }, - "type": "object", - "required": ["links", "nodes"] -} - @pytest.mark.fast def test_rest_core_single_node() -> None: - """A simple single node 'hello world' test""" - # validate schema - Draft202012Validator.check_schema(SCHEMA) - df2012 = Draft202012Validator(SCHEMA) - inp_file = "single_node_helloworld.json" + """A simple single node sophios/restapi test""" + inp_file = "single_node.json" + inp: Json = {} + inp_path = Path(__file__).parent / 'rest_wfb_objects' / inp_file + with open(inp_path, 'r', encoding='utf-8') as f: + inp = json.load(f) + print('----------- from rest api ----------- \n\n') + scope = {} + scope['type'] = 'http' + + async def receive() -> Json: + inp_byte = json.dumps(inp).encode('utf-8') + return {"type": "http.request", "body": inp_byte} + + # create a request object and pack it with our json payload + req: Request = Request(scope) + req._receive = receive + res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api + assert int(res['retval']) == 0 + + +@pytest.mark.fast +def test_rest_core_multi_node() -> None: + """A simple multi node sophios/restapi test""" + inp_file = "multi_node.json" + inp: Json = {} + inp_path = Path(__file__).parent / 'rest_wfb_objects' / inp_file + with open(inp_path, 'r', encoding='utf-8') as f: + inp = json.load(f) + print('----------- from rest api ----------- \n\n') + scope = {} + scope['type'] = 'http' + + async def receive() -> Json: + inp_byte = json.dumps(inp).encode('utf-8') + return {"type": "http.request", "body": inp_byte} + + # create a request object and pack it with our json payload + req: Request = Request(scope) + req._receive = receive + res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api + assert int(res['retval']) == 0 + + +@pytest.mark.fast +def test_rest_core_multi_node_inline_cwl() -> None: + """A simple multi node (inline cwl) sophios/restapi test""" + inp_file = "multi_node_inline_cwl.json" inp: Json = {} - yaml_path = "workflow.json" - inp_path = Path(__file__).with_name(inp_file) + inp_path = Path(__file__).parent / 'rest_wfb_objects' / inp_file with open(inp_path, 'r', encoding='utf-8') as f: inp = json.load(f) - # check if object is conformant with our schema - df2012.is_valid(inp) print('----------- from rest api ----------- \n\n') scope = {} scope['type'] = 'http' From 32c9ec39c3a86d24eb495405a4ebe3ab76721af1 Mon Sep 17 00:00:00 2001 From: VasuJ <145879890+vjaganat90@users.noreply.github.com> Date: Wed, 18 Sep 2024 10:40:40 -0400 Subject: [PATCH 02/15] adjust for modified 'schema' of input objects with 'plugins(ICT)' (#273) Co-authored-by: Vasu Jaganath --- pyproject.toml | 2 +- src/sophios/api/utils/converter.py | 156 ++------ .../api/utils/input_object_schema.json | 377 ++++++++++++++++++ tests/rest_wfb_objects/multi_node.json | 139 +++---- .../multi_node_inline_cwl.json | 313 ++++++++------- tests/rest_wfb_objects/single_node.json | 71 ++-- 6 files changed, 673 insertions(+), 385 deletions(-) create mode 100644 src/sophios/api/utils/input_object_schema.json diff --git a/pyproject.toml b/pyproject.toml index 4990bb38..d0aa2e77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,7 +141,7 @@ namespaces = true [tool.setuptools.package-data] "*" = ["*.txt"] -"sophios" = ["*.json"] +"sophios" = ["**/*.json"] [tool.aliases] test = "pytest --workers 8" diff --git a/src/sophios/api/utils/converter.py b/src/sophios/api/utils/converter.py index 8a1e5ba6..8cbccc8c 100644 --- a/src/sophios/api/utils/converter.py +++ b/src/sophios/api/utils/converter.py @@ -1,130 +1,17 @@ import copy +from pathlib import Path from typing import Any, Dict, List +import json import yaml from jsonschema import Draft202012Validator from sophios.utils_yaml import wic_loader from sophios.wic_types import Json, Cwl -SCHEMA: Json = { - "$schema": "http://json-schema.org/draft-07/schema#", - "definitions": { - "Link": { - "properties": { - "id": { - "type": "number" - }, - "inletIndex": { - "type": "number" - }, - "outletIndex": { - "type": "number" - }, - "sourceId": { - "type": "number" - }, - "targetId": { - "type": "number" - }, - "x1": { - "type": "number" - }, - "x2": { - "type": "number" - }, - "y1": { - "type": "number" - }, - "y2": { - "type": "number" - } - }, - "type": "object", - "required": ["id", "sourceId", "targetId"] - }, - "NodeSettings": { - "properties": { - "inputs": { - "additionalProperties": { - "$ref": "#/definitions/T" - }, - "type": "object" - }, - "outputs": { - "additionalProperties": { - "$ref": "#/definitions/T" - }, - "type": "object" - } - }, - "type": "object" - }, - "NodeX": { - "properties": { - "expanded": { - "type": "boolean" - }, - "height": { - "type": "number" - }, - "id": { - "type": "number" - }, - "internal": { - "type": "boolean" - }, - "name": { - "type": "string" - }, - "pluginId": { - "type": "string" - }, - "settings": { - "$ref": "#/definitions/NodeSettings" - }, - "width": { - "type": "number" - }, - "x": { - "type": "number" - }, - "y": { - "type": "number" - }, - "z": { - "type": "number" - }, - }, - "type": "object", - "required": ["id", "name", "pluginId", "settings", "internal"] - }, - "T": { - "type": "object" - } - }, - "properties": { - "links": { - "items": { - "$ref": "#/definitions/Link" - }, - "type": "array" - }, - "nodes": { - "items": { - "$ref": "#/definitions/NodeX" - }, - "type": "array" - }, - "selection": { - "items": { - "type": "number" - }, - "type": "array" - } - }, - "type": "object", - "required": ["links", "nodes"] -} +SCHEMA_FILE = Path(__file__).parent / "input_object_schema.json" +SCHEMA: Json = {} +with open(SCHEMA_FILE, 'r', encoding='utf-8') as f: + SCHEMA = json.load(f) def del_irrelevant_keys(ldict: List[Dict[Any, Any]], relevant_keys: List[Any]) -> None: @@ -137,20 +24,36 @@ def del_irrelevant_keys(ldict: List[Dict[Any, Any]], relevant_keys: List[Any]) - elem.pop(ek, None) -def validate_schema_and_object(schema: Json, jobj: Json) -> None: +def validate_schema_and_object(schema: Json, jobj: Json) -> bool: """Validate schema object""" Draft202012Validator.check_schema(schema) df2012 = Draft202012Validator(schema) - df2012.is_valid(jobj) + return df2012.is_valid(jobj) + + +def extract_state(inp: Json) -> Json: + """Extract only the state information from the incoming wfb object. + It includes converting "ICT" nodes to "CLT" using "plugins" tag of the object. + """ + inp_restrict: Json = {} + if not inp.get('plugins'): + inp_restrict = copy.deepcopy(inp['state']) + else: + inp_inter = copy.deepcopy(inp) + # Here goes the ICT to CLT extraction logic + inp_restrict = inp_inter['state'] + return inp_restrict def raw_wfb_to_lean_wfb(inp: Json) -> Json: - """drop all the unnecessary info from incoming wfb object""" - inp_restrict = copy.deepcopy(inp) - keys = list(inp.keys()) + """Drop all the unnecessary info from incoming wfb object""" + if validate_schema_and_object(SCHEMA, inp): + print('incoming object is valid against input object schema') + inp_restrict = extract_state(inp) + keys = list(inp_restrict.keys()) # To avoid deserialization # required attributes from schema - prop_req = SCHEMA['required'] + prop_req = SCHEMA['definitions']['State']['required'] nodes_req = SCHEMA['definitions']['NodeX']['required'] links_req = SCHEMA['definitions']['Link']['required'] do_not_rem_nodes_prop = ['cwlScript', 'run'] @@ -170,12 +73,11 @@ def raw_wfb_to_lean_wfb(inp: Json) -> Json: else: pass - validate_schema_and_object(SCHEMA, inp_restrict) return inp_restrict def wfb_to_wic(inp: Json) -> Cwl: - """convert lean wfb json to compliant wic""" + """Convert lean wfb json to compliant wic""" # non-schema preserving changes inp_restrict = copy.deepcopy(inp) diff --git a/src/sophios/api/utils/input_object_schema.json b/src/sophios/api/utils/input_object_schema.json new file mode 100644 index 00000000..bd3b0fc8 --- /dev/null +++ b/src/sophios/api/utils/input_object_schema.json @@ -0,0 +1,377 @@ +{ + "$ref": "#/definitions/WicPayload", + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Dictionary": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "Dictionary": { + "additionalProperties": {}, + "type": "object" + }, + "Link": { + "properties": { + "id": { + "type": "number" + }, + "inletIndex": { + "type": "number" + }, + "outletIndex": { + "type": "number" + }, + "sourceId": { + "type": "number" + }, + "targetId": { + "type": "number" + }, + "x1": { + "type": "number" + }, + "x2": { + "type": "number" + }, + "y1": { + "type": "number" + }, + "y2": { + "type": "number" + } + }, + "required": [ + "id", + "sourceId", + "targetId" + ], + "type": "object" + }, + "NodeInput": { + "properties": { + "description": { + "type": "string" + }, + "format": { + "items": { + "type": "string" + }, + "type": "array" + }, + "name": { + "type": "string" + }, + "options": { + "$ref": "#/definitions/NodeInputOptions" + }, + "required": { + "type": "boolean" + }, + "type": { + "type": "string" + } + }, + "required": [ + "name", + "type" + ], + "type": "object" + }, + "NodeInputOptions": { + "properties": { + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "values" + ], + "type": "object" + }, + "NodeInputUI": { + "properties": { + "condition": { + "type": "string" + }, + "description": { + "type": "string" + }, + "fields": { + "items": { + "type": "string" + }, + "type": "array" + }, + "format": { + "items": { + "type": "string" + }, + "type": "array" + }, + "key": { + "type": "string" + }, + "placeholder": { + "type": "string" + }, + "required": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "key", + "title", + "description", + "type" + ], + "type": "object" + }, + "NodeOutput": { + "properties": { + "description": { + "type": "string" + }, + "format": { + "items": { + "type": "string" + }, + "type": "array" + }, + "name": { + "type": "string" + }, + "required": { + "type": "boolean" + }, + "type": { + "type": "string" + } + }, + "required": [ + "name", + "type" + ], + "type": "object" + }, + "NodeSettings": { + "properties": { + "inputs": { + "$ref": "#/definitions/Dictionary%3Cunknown%3E" + }, + "outputs": { + "$ref": "#/definitions/Dictionary%3Cstring%3E" + } + }, + "type": "object" + }, + "NodeX": { + "properties": { + "expanded": { + "type": "boolean" + }, + "height": { + "type": "number" + }, + "id": { + "type": "number" + }, + "internal": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "pluginId": { + "type": "string" + }, + "settings": { + "$ref": "#/definitions/NodeSettings" + }, + "width": { + "type": "number" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + }, + "z": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "pluginId", + "settings", + "internal" + ], + "type": "object" + }, + "PluginX": { + "properties": { + "author": { + "type": "string" + }, + "baseCommand": { + "items": { + "type": "string" + }, + "type": "array" + }, + "contact": { + "type": "string" + }, + "container": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "createdBy": { + "type": "string" + }, + "description": { + "type": "string" + }, + "documentation": { + "type": "string" + }, + "entrypoint": { + "type": "string" + }, + "hardware": { + "type": "object" + }, + "id": { + "type": "string" + }, + "inputs": { + "items": { + "$ref": "#/definitions/NodeInput" + }, + "type": "array" + }, + "institution": { + "type": "string" + }, + "internal": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "outputs": { + "items": { + "$ref": "#/definitions/NodeOutput" + }, + "type": "array" + }, + "path": { + "type": "string" + }, + "pid": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "specVersion": { + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "title": { + "type": "string" + }, + "ui": { + "items": { + "$ref": "#/definitions/NodeInputUI" + }, + "type": "array" + }, + "updatedAt": { + "type": "string" + }, + "updatedBy": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "id", + "name", + "version", + "title", + "description", + "inputs", + "outputs", + "ui" + ], + "type": "object" + }, + "State": { + "properties": { + "links": { + "items": { + "$ref": "#/definitions/Link" + }, + "type": "array" + }, + "nodes": { + "items": { + "$ref": "#/definitions/NodeX" + }, + "type": "array" + }, + "selection": { + "items": { + "type": "number" + }, + "type": "array" + } + }, + "required": [ + "links", + "nodes" + ], + "type": "object" + }, + "WicPayload": { + "properties": { + "plugins": { + "items": { + "$ref": "#/definitions/PluginX" + }, + "type": "array" + }, + "state": { + "$ref": "#/definitions/State" + } + }, + "required": [ + "state", + "plugins" + ], + "type": "object" + } + } +} \ No newline at end of file diff --git a/tests/rest_wfb_objects/multi_node.json b/tests/rest_wfb_objects/multi_node.json index ffe42014..27b9894a 100644 --- a/tests/rest_wfb_objects/multi_node.json +++ b/tests/rest_wfb_objects/multi_node.json @@ -1,76 +1,79 @@ { - "nodes": [ - { - "id": 7, - "x": 462, - "y": 206, - "z": 2, - "name": "touch", - "pluginId": "touch", - "height": 50, - "width": 250, - "settings": { - "inputs": { - "filename": "empty.txt" + "state": { + "nodes": [ + { + "id": 7, + "x": 462, + "y": 206, + "z": 2, + "name": "touch", + "pluginId": "touch", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "filename": "empty.txt" + }, + "outputs": { + "file": "file_touch" + } }, - "outputs": { - "file": "file_touch" - } + "internal": false }, - "internal": false - }, - { - "id": 18, - "x": 155, - "y": 195, - "z": 1, - "name": "append", - "pluginId": "append", - "height": 50, - "width": 250, - "settings": { - "inputs": { - "str": "Hello", - "file": "file_touch" + { + "id": 18, + "x": 155, + "y": 195, + "z": 1, + "name": "append", + "pluginId": "append", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "str": "Hello", + "file": "file_touch" + }, + "outputs": { + "file": "file_append1" + } }, - "outputs": { - "file": "file_append1" - } + "internal": false }, - "internal": false - }, - { - "id": 9, - "x": 790.3254637299812, - "y": 449.8103498684344, - "z": 5, - "name": "append", - "pluginId": "append", - "height": 50, - "width": 250, - "settings": { - "inputs": { - "str": "World!", - "file": "file_append1" + { + "id": 9, + "x": 790.3254637299812, + "y": 449.8103498684344, + "z": 5, + "name": "append", + "pluginId": "append", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "str": "World!", + "file": "file_append1" + }, + "outputs": { + "file": "file_append2" + } }, - "outputs": { - "file": "file_append2" - } + "internal": false + } + ], + "links": [ + { + "sourceId": 7, + "targetId": 18, + "id": 1 }, - "internal": false - } - ], - "links": [ - { - "sourceId": 7, - "targetId": 18, - "id": 1 - }, - { - "sourceId": 18, - "targetId": 9, - "id": 5 - } - ], - "selection": [] + { + "sourceId": 18, + "targetId": 9, + "id": 5 + } + ], + "selection": [] + }, + "plugins": [] } \ No newline at end of file diff --git a/tests/rest_wfb_objects/multi_node_inline_cwl.json b/tests/rest_wfb_objects/multi_node_inline_cwl.json index c972ee36..410a1d48 100644 --- a/tests/rest_wfb_objects/multi_node_inline_cwl.json +++ b/tests/rest_wfb_objects/multi_node_inline_cwl.json @@ -1,181 +1,184 @@ { - "nodes": [ - { - "id": 7, - "x": 462, - "y": 206, - "z": 2, - "name": "touch", - "pluginId": "touch", - "height": 50, - "width": 250, - "settings": { - "inputs": { - "filename": "empty.txt" - }, - "outputs": { - "file": "file_touch" - } - }, - "run": { - "cwlVersion": "v1.0", - "class": "CommandLineTool", - "requirements": { - "DockerRequirement": { - "dockerPull": "docker.io/bash:4.4" + "state": { + "nodes": [ + { + "id": 7, + "x": 462, + "y": 206, + "z": 2, + "name": "touch", + "pluginId": "touch", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "filename": "empty.txt" }, - "InlineJavascriptRequirement": {} - }, - "baseCommand": "touch", - "inputs": { - "filename": { - "type": "string", - "inputBinding": { - "position": 1 - } + "outputs": { + "file": "file_touch" } }, - "outputs": { - "file": { - "type": "File", - "outputBinding": { - "glob": "$(inputs.filename)" + "run": { + "cwlVersion": "v1.0", + "class": "CommandLineTool", + "requirements": { + "DockerRequirement": { + "dockerPull": "docker.io/bash:4.4" + }, + "InlineJavascriptRequirement": {} + }, + "baseCommand": "touch", + "inputs": { + "filename": { + "type": "string", + "inputBinding": { + "position": 1 + } + } + }, + "outputs": { + "file": { + "type": "File", + "outputBinding": { + "glob": "$(inputs.filename)" + } } } - } - }, - "internal": false - }, - { - "id": 18, - "x": 155, - "y": 195, - "z": 1, - "name": "append", - "pluginId": "append", - "height": 50, - "width": 250, - "settings": { - "inputs": { - "str": "Hello", - "file": "file_touch" }, - "outputs": { - "file": "file_append1" - } + "internal": false }, - "run": { - "class": "CommandLineTool", - "cwlVersion": "v1.0", - "requirements": { - "ShellCommandRequirement": {}, - "InlineJavascriptRequirement": {}, - "InitialWorkDirRequirement": { - "listing": [ - "$(inputs.file)" - ] + { + "id": 18, + "x": 155, + "y": 195, + "z": 1, + "name": "append", + "pluginId": "append", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "str": "Hello", + "file": "file_touch" + }, + "outputs": { + "file": "file_append1" } }, - "inputs": { - "str": { - "type": "string", - "inputBinding": { - "shellQuote": false, - "position": 1, - "prefix": "echo" + "run": { + "class": "CommandLineTool", + "cwlVersion": "v1.0", + "requirements": { + "ShellCommandRequirement": {}, + "InlineJavascriptRequirement": {}, + "InitialWorkDirRequirement": { + "listing": [ + "$(inputs.file)" + ] } }, - "file": { - "type": "File", - "inputBinding": { - "shellQuote": false, - "position": 2, - "prefix": ">>" + "inputs": { + "str": { + "type": "string", + "inputBinding": { + "shellQuote": false, + "position": 1, + "prefix": "echo" + } + }, + "file": { + "type": "File", + "inputBinding": { + "shellQuote": false, + "position": 2, + "prefix": ">>" + } } - } - }, - "outputs": { - "file": { - "type": "File", - "outputBinding": { - "glob": "$(inputs.file.basename)" + }, + "outputs": { + "file": { + "type": "File", + "outputBinding": { + "glob": "$(inputs.file.basename)" + } } } - } - }, - "internal": false - }, - { - "id": 9, - "x": 790.3254637299812, - "y": 449.8103498684344, - "z": 5, - "name": "append", - "pluginId": "append", - "height": 50, - "width": 250, - "settings": { - "inputs": { - "str": "World!", - "file": "file_append1" }, - "outputs": { - "file": "file_append2" - } + "internal": false }, - "run": { - "class": "CommandLineTool", - "cwlVersion": "v1.0", - "requirements": { - "ShellCommandRequirement": {}, - "InlineJavascriptRequirement": {}, - "InitialWorkDirRequirement": { - "listing": [ - "$(inputs.file)" - ] + { + "id": 9, + "x": 790.3254637299812, + "y": 449.8103498684344, + "z": 5, + "name": "append", + "pluginId": "append", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "str": "World!", + "file": "file_append1" + }, + "outputs": { + "file": "file_append2" } }, - "inputs": { - "str": { - "type": "string", - "inputBinding": { - "shellQuote": false, - "position": 1, - "prefix": "echo" + "run": { + "class": "CommandLineTool", + "cwlVersion": "v1.0", + "requirements": { + "ShellCommandRequirement": {}, + "InlineJavascriptRequirement": {}, + "InitialWorkDirRequirement": { + "listing": [ + "$(inputs.file)" + ] } }, - "file": { - "type": "File", - "inputBinding": { - "shellQuote": false, - "position": 2, - "prefix": ">>" + "inputs": { + "str": { + "type": "string", + "inputBinding": { + "shellQuote": false, + "position": 1, + "prefix": "echo" + } + }, + "file": { + "type": "File", + "inputBinding": { + "shellQuote": false, + "position": 2, + "prefix": ">>" + } } - } - }, - "outputs": { - "file": { - "type": "File", - "outputBinding": { - "glob": "$(inputs.file.basename)" + }, + "outputs": { + "file": { + "type": "File", + "outputBinding": { + "glob": "$(inputs.file.basename)" + } } } - } + }, + "internal": false + } + ], + "links": [ + { + "sourceId": 7, + "targetId": 18, + "id": 1 }, - "internal": false - } - ], - "links": [ - { - "sourceId": 7, - "targetId": 18, - "id": 1 - }, - { - "sourceId": 18, - "targetId": 9, - "id": 5 - } - ], - "selection": [] + { + "sourceId": 18, + "targetId": 9, + "id": 5 + } + ], + "selection": [] + }, + "plugins": [] } \ No newline at end of file diff --git a/tests/rest_wfb_objects/single_node.json b/tests/rest_wfb_objects/single_node.json index 099bf9ce..196fa80c 100644 --- a/tests/rest_wfb_objects/single_node.json +++ b/tests/rest_wfb_objects/single_node.json @@ -1,40 +1,43 @@ { - "nodes": [ - { - "id": 1, - "name": "PythonHelloWorld", - "pluginId": "", - "cwlScript": { - "steps": { - "one": { - "run": { - "baseCommand": [ - "python", - "-c", - "print('hello world'); print('sqr of 7 : %.2f' % 7**2)" - ], - "class": "CommandLineTool", - "cwlVersion": "v1.2", - "inputs": {}, - "outputs": { - "pyout": { - "outputBinding": { - "glob": "output" - }, - "type": "File" + "state": { + "nodes": [ + { + "id": 1, + "name": "PythonHelloWorld", + "pluginId": "", + "cwlScript": { + "steps": { + "one": { + "run": { + "baseCommand": [ + "python", + "-c", + "print('hello world'); print('sqr of 7 : %.2f' % 7**2)" + ], + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "inputs": {}, + "outputs": { + "pyout": { + "outputBinding": { + "glob": "output" + }, + "type": "File" + } + }, + "stdout": "output", + "requirements": { + "InlineJavascriptRequirement": {} } - }, - "stdout": "output", - "requirements": { - "InlineJavascriptRequirement": {} } } } - } - }, - "settings": {}, - "internal": false - } - ], - "links": [] + }, + "settings": {}, + "internal": false + } + ], + "links": [] + }, + "plugins": [] } \ No newline at end of file From 982ed4a253d431ea45feb048f46b0ff69e158606 Mon Sep 17 00:00:00 2001 From: VasuJ <145879890+vjaganat90@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:32:22 -0400 Subject: [PATCH 03/15] (Modified) Add flag to only compile workflow to cwl without running (#274) Co-authored-by: JesseMckinzie --- src/sophios/cli.py | 2 ++ src/sophios/main.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/sophios/cli.py b/src/sophios/cli.py index bca6c548..c892d589 100644 --- a/src/sophios/cli.py +++ b/src/sophios/cli.py @@ -83,6 +83,8 @@ help='Just generates run.sh and exits. Does not actually invoke ./run.sh') group_run.add_argument('--run_local', default=False, action="store_true", help='After generating the cwl file(s), run it on your local machine.') +group_run.add_argument('--generate_cwl_workflow', required=False, default=False, action="store_true", + help='Compile the workflow without pulling the docker image') parser.add_argument('--cwl_inline_subworkflows', default=False, action="store_true", help='Before generating the cwl file, inline all subworkflows.') diff --git a/src/sophios/main.py b/src/sophios/main.py index 3c1d4b08..e7ba57a0 100644 --- a/src/sophios/main.py +++ b/src/sophios/main.py @@ -194,6 +194,9 @@ def main() -> None: print("(This may happen if you installed the graphviz python package") print("but not the graphviz system package.)") + if args.generate_cwl_workflow: + io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) + if args.run_local or args.generate_run_script: # cwl-docker-extract recursively `docker pull`s all images in all subworkflows. # This is important because cwltool only uses `docker run` when executing From c69e897a36526623f9db4efbc4b1d4e22e985bcc Mon Sep 17 00:00:00 2001 From: VasuJ <145879890+vjaganat90@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:37:19 -0400 Subject: [PATCH 04/15] (Modified) Add ict to clt conversion (#275) Co-authored-by: JesseMckinzie --- .github/workflows/run_workflows.yml | 5 + pyproject.toml | 1 + src/sophios/api/utils/converter.py | 22 +- src/sophios/api/utils/ict/__init__.py | 0 .../api/utils/ict/ict_spec/__init__.py | 0 src/sophios/api/utils/ict/ict_spec/cast.py | 27 ++ .../utils/ict/ict_spec/hardware/__init__.py | 10 + .../utils/ict/ict_spec/hardware/objects.py | 108 ++++++++ .../api/utils/ict/ict_spec/io/__init__.py | 5 + .../api/utils/ict/ict_spec/io/objects.py | 148 +++++++++++ .../utils/ict/ict_spec/metadata/__init__.py | 5 + .../utils/ict/ict_spec/metadata/objects.py | 186 +++++++++++++ src/sophios/api/utils/ict/ict_spec/model.py | 109 ++++++++ .../api/utils/ict/ict_spec/tools/__init__.py | 5 + .../api/utils/ict/ict_spec/tools/cwl_ict.py | 119 +++++++++ .../api/utils/ict/ict_spec/ui/__init__.py | 27 ++ .../api/utils/ict/ict_spec/ui/objects.py | 246 ++++++++++++++++++ tests/data/__init__.py | 0 .../ict_data/czi_extract/czi_extract_clt.json | 43 +++ .../ict_data/czi_extract/czi_extract_ict.json | 53 ++++ .../label_to_vector/label_to_vector_clt.json | 49 ++++ .../label_to_vector/label_to_vector_ict.json | 69 +++++ .../ome_conversion/ome_conversion_clt.json | 55 ++++ .../ome_conversion/ome_conversion_ict.json | 89 +++++++ tests/test_ict_to_clt_conversion.py | 61 +++++ 25 files changed, 1441 insertions(+), 1 deletion(-) create mode 100644 src/sophios/api/utils/ict/__init__.py create mode 100644 src/sophios/api/utils/ict/ict_spec/__init__.py create mode 100644 src/sophios/api/utils/ict/ict_spec/cast.py create mode 100644 src/sophios/api/utils/ict/ict_spec/hardware/__init__.py create mode 100644 src/sophios/api/utils/ict/ict_spec/hardware/objects.py create mode 100644 src/sophios/api/utils/ict/ict_spec/io/__init__.py create mode 100644 src/sophios/api/utils/ict/ict_spec/io/objects.py create mode 100644 src/sophios/api/utils/ict/ict_spec/metadata/__init__.py create mode 100644 src/sophios/api/utils/ict/ict_spec/metadata/objects.py create mode 100644 src/sophios/api/utils/ict/ict_spec/model.py create mode 100644 src/sophios/api/utils/ict/ict_spec/tools/__init__.py create mode 100644 src/sophios/api/utils/ict/ict_spec/tools/cwl_ict.py create mode 100644 src/sophios/api/utils/ict/ict_spec/ui/__init__.py create mode 100644 src/sophios/api/utils/ict/ict_spec/ui/objects.py create mode 100644 tests/data/__init__.py create mode 100644 tests/data/ict_data/czi_extract/czi_extract_clt.json create mode 100644 tests/data/ict_data/czi_extract/czi_extract_ict.json create mode 100644 tests/data/ict_data/label_to_vector/label_to_vector_clt.json create mode 100644 tests/data/ict_data/label_to_vector/label_to_vector_ict.json create mode 100644 tests/data/ict_data/ome_conversion/ome_conversion_clt.json create mode 100644 tests/data/ict_data/ome_conversion/ome_conversion_ict.json create mode 100644 tests/test_ict_to_clt_conversion.py diff --git a/.github/workflows/run_workflows.yml b/.github/workflows/run_workflows.yml index 6a7e6fa7..b074fa80 100644 --- a/.github/workflows/run_workflows.yml +++ b/.github/workflows/run_workflows.yml @@ -181,6 +181,11 @@ jobs: # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 run: cd workflow-inference-compiler/ && pytest tests/test_rest_core.py -k test_rest_core --cwl_runner cwltool + - name: PyTest Run ICT to CLT conversion Tests + if: always() + # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 + run: cd workflow-inference-compiler/ && pytest tests/test_ict_to_clt_conversion.py -k test_ict_to_clt + # NOTE: The steps below are for repository_dispatch only. For all other steps, please insert above # this comment. diff --git a/pyproject.toml b/pyproject.toml index d0aa2e77..d9df3db9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "typeguard", "pydantic>=2.6", "pydantic-settings", + "pydantic[email]", "docker", # FYI also need uidmap to run podman rootless "podman", diff --git a/src/sophios/api/utils/converter.py b/src/sophios/api/utils/converter.py index 8cbccc8c..8fdec813 100644 --- a/src/sophios/api/utils/converter.py +++ b/src/sophios/api/utils/converter.py @@ -1,12 +1,15 @@ + import copy from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Union import json import yaml from jsonschema import Draft202012Validator from sophios.utils_yaml import wic_loader from sophios.wic_types import Json, Cwl +from sophios.api.utils.ict.ict_spec.model import ICT +from sophios.api.utils.ict.ict_spec.cast import cast_to_ict SCHEMA_FILE = Path(__file__).parent / "input_object_schema.json" SCHEMA: Json = {} @@ -141,3 +144,20 @@ def wfb_to_wic(inp: Json) -> Cwl: node = inp_restrict["nodes"][0] workflow_temp = node["cwlScript"] return workflow_temp + + +def ict_to_clt(ict: Union[ICT, Path, str, dict], network_access: bool = False) -> dict: + """ + Convert ICT to CWL CommandLineTool + + Args: + ict (Union[ICT, Path, str, dict]): ICT to convert to CLT. ICT can be an ICT object, + a path to a yaml file, or a dictionary containing ICT + + Returns: + dict: A dictionary containing the CLT + """ + + ict_local = ict if isinstance(ict, ICT) else cast_to_ict(ict) + + return ict_local.to_clt(network_access=network_access) diff --git a/src/sophios/api/utils/ict/__init__.py b/src/sophios/api/utils/ict/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/sophios/api/utils/ict/ict_spec/__init__.py b/src/sophios/api/utils/ict/ict_spec/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/sophios/api/utils/ict/ict_spec/cast.py b/src/sophios/api/utils/ict/ict_spec/cast.py new file mode 100644 index 00000000..ea84394c --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/cast.py @@ -0,0 +1,27 @@ +import json +from pathlib import Path +from typing import Union + +from yaml import safe_load + +from sophios.api.utils.ict.ict_spec.model import ICT + + +def cast_to_ict(ict: Union[Path, str, dict]) -> ICT: + + if isinstance(ict, str): + ict = Path(ict) + + if isinstance(ict, Path): + if str(ict).endswith(".yaml") or str(ict).endswith(".yml"): + with open(ict, "r", encoding="utf-8") as f_o: + data = safe_load(f_o) + elif str(ict).endswith(".json"): + with open(ict, "r", encoding="utf-8") as f_o: + data = json.load(f_o) + else: + raise ValueError(f"File extension not supported: {ict}") + + return ICT(**data) + + return ICT(**ict) diff --git a/src/sophios/api/utils/ict/ict_spec/hardware/__init__.py b/src/sophios/api/utils/ict/ict_spec/hardware/__init__.py new file mode 100644 index 00000000..7b3bc5f3 --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/hardware/__init__.py @@ -0,0 +1,10 @@ +"""Hardware Requirements for ICT.""" + +from sophios.api.utils.ict.ict_spec.hardware.objects import ( + CPU, + GPU, + HardwareRequirements, + Memory, +) + +__all__ = ["CPU", "Memory", "GPU", "HardwareRequirements"] diff --git a/src/sophios/api/utils/ict/ict_spec/hardware/objects.py b/src/sophios/api/utils/ict/ict_spec/hardware/objects.py new file mode 100644 index 00000000..023da730 --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/hardware/objects.py @@ -0,0 +1,108 @@ +# pylint: disable=no-member +"""Hardware Requirements for ICT.""" +from typing import Annotated, Optional, Union, Any + +from pydantic import BaseModel, BeforeValidator, Field + + +def validate_str(s_t: Union[int, float, str]) -> Union[str, None]: + """Return a string from int, float, or str.""" + if s_t is None: + return None + if isinstance(s_t, str): + return s_t + if not isinstance(s_t, (int, float)) or isinstance(s_t, bool): + raise ValueError("must be an int, float, or str") + return str(s_t) + + +StrInt = Annotated[str, BeforeValidator(validate_str)] + + +class CPU(BaseModel): + """CPU object.""" + + cpu_type: Optional[str] = Field( + None, + alias="type", + description="Any non-standard or specific processor limitations.", + examples=["arm64"], + ) + cpu_min: Optional[StrInt] = Field( + None, + alias="min", + description="Minimum requirement for CPU allocation where 1 CPU unit is equivalent to 1 physical CPU core or 1 virtual core.", + examples=["100m"], + ) + cpu_recommended: Optional[StrInt] = Field( + None, + alias="recommended", + description="Recommended requirement for CPU allocation for optimal performance.", + examples=["200m"], + ) + + +class Memory(BaseModel): + """Memory object.""" + + memory_min: Optional[StrInt] = Field( + None, + alias="min", + description="Minimum requirement for memory allocation, measured in bytes.", + examples=["129Mi"], + ) + memory_recommended: Optional[StrInt] = Field( + None, + alias="recommended", + description="Recommended requirement for memory allocation for optimal performance.", + examples=["200Mi"], + ) + + +class GPU(BaseModel): + """GPU object.""" + + gpu_enabled: Optional[bool] = Field( + None, + alias="enabled", + description="Boolean value indicating if the plugin is optimized for GPU.", + examples=[False], + ) + gpu_required: Optional[bool] = Field( + None, + alias="required", + description="Boolean value indicating if the plugin requires a GPU to run.", + examples=[False], + ) + gpu_type: Optional[str] = Field( + None, + alias="type", + description=" Any identifying label for GPU hardware specificity.", + examples=["cuda11"], + ) + + +ATTRIBUTES = [ + "cpu_type", + "cpu_min", + "cpu_recommended", + "memory_min", + "memory_recommended", + "gpu_enabled", + "gpu_required", + "gpu_type", +] + + +class HardwareRequirements(BaseModel): + """HardwareRequirements object.""" + + cpu: Optional[CPU] = Field(None, description="CPU requirements.") + memory: Optional[Memory] = Field(None, description="Memory requirements.") + gpu: Optional[GPU] = Field(None, description="GPU requirements.") + + def __getattribute__(self, name: str) -> Any: + """Get attribute.""" + if name in ATTRIBUTES: + return super().__getattribute__(name.split("_")[0]).__getattribute__(name) + return super().__getattribute__(name) diff --git a/src/sophios/api/utils/ict/ict_spec/io/__init__.py b/src/sophios/api/utils/ict/ict_spec/io/__init__.py new file mode 100644 index 00000000..bfb31c11 --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/io/__init__.py @@ -0,0 +1,5 @@ +"""IO objects.""" + +from .objects import IO + +__all__ = ["IO"] diff --git a/src/sophios/api/utils/ict/ict_spec/io/objects.py b/src/sophios/api/utils/ict/ict_spec/io/objects.py new file mode 100644 index 00000000..3cbee409 --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/io/objects.py @@ -0,0 +1,148 @@ +"""IO objects for ICT.""" + +import enum +from typing import Optional, Union, Any + +from pydantic import BaseModel, Field + + +CWL_IO_DICT: dict[str, str] = { + "string": "string", + "number": "double", + "array": "string", + "boolean": "boolean", + # TODO: File vs Directory? +} + + +class TypesEnum(str, enum.Enum): + """Types enum for ICT IO.""" + + STRING = "string" + NUMBER = "number" + ARRAY = "array" + BOOLEAN = "boolean" + PATH = "path" + + +# def _get_cwl_type(io_name: str, io_type: str) -> str: +def _get_cwl_type(io_type: str) -> str: + """Return the CWL type from the ICT IO type.""" + if io_type == "path": + # NOTE: for now, default to directory + # this needs to be addressed + # path could be File or Directory + return "Directory" + # if bool(re.search("dir", io_name, re.I)): + # return "Directory" + # return "File" + return CWL_IO_DICT[io_type] + + +class IO(BaseModel): + """IO BaseModel.""" + + name: str = Field( + description=( + "Unique input or output name for this plugin, case-sensitive match to" + "corresponding variable expected by tool." + ), + examples=["thresholdtype"], + ) + io_type: TypesEnum = Field( + ..., + alias="type", + description="Defines the parameter passed to the ICT tool based on broad categories of basic types.", + examples=["string"], + ) + description: Optional[str] = Field( + None, + description="Short text description of expected value for field.", + examples=["Algorithm type for thresholding"], + ) + defaultValue: Optional[Any] = Field( + None, + description="Optional default value.", + examples=["42"], + ) + required: bool = Field( + description="Boolean (true/false) value indicating whether this " + + "field needs an associated value.", + examples=["true"], + ) + io_format: Union[list[str], dict] = Field( + ..., + alias="format", + description="Defines the actual value(s) that the input/output parameter" + + "represents using an ontology schema.", + ) # TODO ontology + + @property + def _is_optional(self) -> str: + """Return '' if required, '?' if default exists, else '?'.""" + if self.defaultValue is not None: + return "?" + if self.required: + return "" + + return "?" + + def convert_uri_format(self, uri_format: Any) -> str: + """Convert to cwl format + Args: + format (_type_): _description_ + """ + return f"edam:format_{uri_format.split('_')[-1]}" + + def _input_to_cwl(self) -> dict: + """Convert inputs to CWL.""" + cwl_dict_ = { + "inputBinding": {"prefix": f"--{self.name}"}, + # "type": f"{_get_cwl_type(self.name, self.io_type)}{self._is_optional}", + "type": f"{_get_cwl_type(self.io_type)}{self._is_optional}", + } + + if ( + isinstance(self.io_format, dict) + and self.io_format.get("uri", None) is not None # pylint: disable=no-member + ): + # pylint: disable-next=unsubscriptable-object + cwl_dict_["format"] = self.convert_uri_format(self.io_format["uri"]) + if self.defaultValue is not None: + cwl_dict_["default"] = self.defaultValue + return cwl_dict_ + + def _output_to_cwl(self, inputs: Any) -> dict: + """Convert outputs to CWL.""" + if self.io_type == "path": + if self.name in inputs: + if ( + not isinstance(self.io_format, list) + and self.io_format["term"].lower() + == "directory" # pylint: disable=unsubscriptable-object + ): + cwl_type = "Directory" + elif ( + not isinstance(self.io_format, list) + and self.io_format["term"].lower() + == "file" # pylint: disable=unsubscriptable-object + ): + cwl_type = "File" + else: + cwl_type = "File" + + cwl_dict_ = { + "outputBinding": {"glob": f"$(inputs.{self.name}.basename)"}, + "type": cwl_type, + } + if ( + not isinstance(self.io_format, list) + and self.io_format.get("uri", None) + is not None # pylint: disable=no-member + ): + # pylint: disable-next=unsubscriptable-object + cwl_dict_["format"] = self.convert_uri_format(self.io_format["uri"]) + return cwl_dict_ + + raise ValueError(f"Output {self.name} not found in inputs") + raise NotImplementedError(f"Output not supported {self.name}") diff --git a/src/sophios/api/utils/ict/ict_spec/metadata/__init__.py b/src/sophios/api/utils/ict/ict_spec/metadata/__init__.py new file mode 100644 index 00000000..37665e4c --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/metadata/__init__.py @@ -0,0 +1,5 @@ +"""Metadata objects.""" + +from .objects import Metadata + +__all__ = ["Metadata"] diff --git a/src/sophios/api/utils/ict/ict_spec/metadata/objects.py b/src/sophios/api/utils/ict/ict_spec/metadata/objects.py new file mode 100644 index 00000000..88bd945a --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/metadata/objects.py @@ -0,0 +1,186 @@ +"""Metadata Model.""" + +import re +from functools import singledispatchmethod +from pathlib import Path +from typing import Any, Optional, Union + +from pydantic import ( + AnyHttpUrl, + BaseModel, + EmailStr, + Field, + RootModel, + WithJsonSchema, + field_validator, + model_validator, +) +from typing_extensions import Annotated + + +class Author(RootModel): + """Author object.""" + + root: str + + @field_validator("root") + @classmethod + def check_author(cls: Any, value: Any) -> Any: + """Check the author follows the correct format.""" + if not len(value.split(" ")) == 2: + raise ValueError( + "The author must be in the format " + ) + return value + + def __repr__(self) -> str: + """Repr.""" + return self.root + + def __str__(self) -> str: + """Str.""" + return self.root + + @singledispatchmethod + def __eq__(self, other: Any) -> bool: # type: ignore + """Compare if two Author objects are equal.""" + msg = "invalid type for comparison." + raise TypeError(msg) + + +@Author.__eq__.register(str) # type: ignore # pylint: disable=no-member +def _(self: Author, other: Author) -> Any: + return self.root == other + + +@Author.__eq__.register(Author) # type: ignore # pylint: disable=no-member +def _(self: Author, other: Author) -> Any: + return self.root == other.root + + +class DOI(RootModel): + """DOI object.""" + + root: str + + @field_validator("root") + @classmethod + def check_doi(cls: Any, value: Any) -> Any: + """Check the doi follows the correct format.""" + if not value.startswith("10."): + raise ValueError("The DOI must start with 10.") + if not len(value.split("/")) == 2: + raise ValueError("The DOI must be in the format /") + return value + + def __repr__(self) -> str: + """Repr.""" + return self.root + + def __str__(self) -> str: + """Str.""" + return self.root + + @singledispatchmethod + def __eq__(self, other: Any) -> bool: # type: ignore + """Compare if two DOI objects are equal.""" + msg = "invalid type for comparison." + raise TypeError(msg) + + +@DOI.__eq__.register(str) # type: ignore # pylint: disable=no-member +def _(self, other): + return self.root == other + + +@DOI.__eq__.register(DOI) # type: ignore # pylint: disable=no-member +def _(self, other): + return self.root == other.root + + +EntrypointPath = Annotated[Path, WithJsonSchema({"type": "string", "format": "uri"})] + + +class Metadata(BaseModel): + """Metadata BaseModel.""" + + name: str = Field( + description=( + "Unique identifier for ICT tool scoped on organization or user," + "should take the format /." + ), + examples=["wipp/threshold"], + ) + container: str = Field( + description=( + "Direct link to hosted ICT container image, should take the format" + "/:, registry path may be omitted" + "and will default to Docker Hub." + ), + examples=["wipp/threshold:1.1.1"], + ) + entrypoint: Union[EntrypointPath, str] = Field( + description="Absolute path to initial script or command within packaged image." + ) + title: Optional[str] = Field( + None, + description="(optional) Descriptive human-readable name, will default to `name` if omitted.", + examples=["Thresholding Plugin"], + ) + description: Optional[str] = Field( + None, + description="(optional) Brief description of plugin.", + examples=["Thresholding methods from ImageJ"], + ) + author: list[Author] = Field( + description=( + "Comma separated list of authors, each author name should take the format" + " ." + ), + examples=["Mohammed Ouladi"], + ) + contact: Union[EmailStr, AnyHttpUrl] = Field( + description="Email or link to point of contact (ie. GitHub user page) for questions or issues.", + examples=["mohammed.ouladi@labshare.org"], + ) + repository: AnyHttpUrl = Field( + description="Url for public or private repository hosting source code.", + examples=["https://github.com/polusai/polus-plugins"], + ) + documentation: Optional[AnyHttpUrl] = Field( + None, + description="Url for hosted documentation about using or modifying the plugin.", + ) + citation: Optional[DOI] = Field( + None, + description="DOI link to relevant citation, plugin user should use this citation when using this plugin.", + ) + + @field_validator("name") + @classmethod + def check_name(cls: Any, value: Any) -> Any: + """Check the name follows the correct format.""" + if not len(value.split("/")) in [2, 3]: + raise ValueError( + "The name must be in the format /" + ) + return value + + @field_validator("container") + @classmethod + def check_container(cls: Any, value: Any) -> Any: + """Check the container follows the correct format.""" + if not bool( + re.match(r"^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+:[a-zA-Z0-9_\.\-]+$", value) + ): + raise ValueError( + "The name must be in the format /:" + ) + return value + + @model_validator(mode="after") + def default_title(self) -> Any: + """Set the title to the name if not provided.""" + if self.title is None: + self.title = self.name + return self diff --git a/src/sophios/api/utils/ict/ict_spec/model.py b/src/sophios/api/utils/ict/ict_spec/model.py new file mode 100644 index 00000000..42609045 --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/model.py @@ -0,0 +1,109 @@ +# pylint: disable=no-member, no-name-in-module, import-error +"""ICT model.""" + +import logging +from pathlib import Path +from typing import Optional, TypeVar + +import yaml +from pydantic import model_validator + +from sophios.api.utils.ict.ict_spec.hardware import HardwareRequirements +from sophios.api.utils.ict.ict_spec.io import IO +from sophios.api.utils.ict.ict_spec.metadata import Metadata +from sophios.api.utils.ict.ict_spec.tools import clt_dict, ict_dict +from sophios.api.utils.ict.ict_spec.ui import UIItem + +StrPath = TypeVar("StrPath", str, Path) + +logger = logging.getLogger("ict") + + +class ICT(Metadata): + """ICT object.""" + + inputs: list[IO] + outputs: list[IO] + ui: list[UIItem] + hardware: Optional[HardwareRequirements] = None + + @model_validator(mode="after") + def validate_ui(self) -> "ICT": + """Validate that the ui matches the inputs and outputs.""" + io_dict = {"inputs": [], "outputs": []} # type: ignore + ui_keys = [ui.key.root.split(".") for ui in self.ui] + for ui_ in ui_keys: + io_dict[ui_[0]].append(ui_[1]) + input_names = [io.name for io in self.inputs] + output_names = [io.name for io in self.outputs] + inp_bool = [x in input_names for x in io_dict["inputs"]] + out_bool = [x in output_names for x in io_dict["outputs"]] + + if not all(inp_bool): + raise ValueError( + f"The ui keys must match the inputs and outputs keys. Unmatched: inputs.{set(io_dict['inputs'])-set(input_names)}" + ) + if not all(out_bool): + raise ValueError( + f"The ui keys must match the inputs and outputs keys. Unmatched: outputs.{set(io_dict['outputs'])-set(output_names)}" + ) + return self + + def to_clt(self, network_access: bool = False) -> dict: + """Convert ICT to CWL CommandLineTool. + + + Args: + network_access: bool + Default is `False`. If set to `True`, the + requirements of the CLT will include + `networkAccess`: `True`. + + Returns: `dict` representation of the CLT. + """ + return clt_dict(self, network_access) + + @property + def clt(self) -> dict: + """CWL CommandLineTool from an ICT object.""" + return clt_dict(self, network_access=False) + + @property + def ict(self) -> dict: + """ICT yaml from an ICT object.""" + return ict_dict(self) + + def save_clt(self, cwl_path: StrPath, network_access: bool = False) -> Path: + """Save the ICT as CommandLineTool to a file.""" + assert ( + str(cwl_path).rsplit(".", maxsplit=1)[-1] == "cwl" + ), "Path must end in .cwl" + with Path(cwl_path).open("w", encoding="utf-8") as file: + yaml.dump(self.to_clt(network_access), file) + return Path(cwl_path) + + def save_cwl(self, cwl_path: StrPath, network_access: bool = False) -> Path: + """Save the ICT as CommandLineTool to a file. + + Alias for `save_clt`. + """ + return self.save_clt(cwl_path, network_access) + + def save_yaml(self, yaml_path: StrPath) -> Path: + """Save the ICT as yaml to a file.""" + assert str(yaml_path).rsplit(".", maxsplit=1)[-1] in [ + "yaml", + "yml", + ], "Path must end in .yaml or .yml" + with Path(yaml_path).open("w", encoding="utf-8") as file: + yaml.dump( + self.model_dump(mode="json", exclude_none=True, by_alias=True), file + ) + return Path(yaml_path) + + def save_yml(self, yml_path: StrPath) -> Path: + """Save the ICT as yaml to a file. + + Alias for `save_yaml`. + """ + return self.save_yaml(yml_path) diff --git a/src/sophios/api/utils/ict/ict_spec/tools/__init__.py b/src/sophios/api/utils/ict/ict_spec/tools/__init__.py new file mode 100644 index 00000000..c97c6d2f --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/tools/__init__.py @@ -0,0 +1,5 @@ +"""CWL generation for ICT objects.""" + +from .cwl_ict import clt_dict, ict_dict + +__all__ = ["clt_dict", "ict_dict"] diff --git a/src/sophios/api/utils/ict/ict_spec/tools/cwl_ict.py b/src/sophios/api/utils/ict/ict_spec/tools/cwl_ict.py new file mode 100644 index 00000000..1d87fcdf --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/tools/cwl_ict.py @@ -0,0 +1,119 @@ +"""CWL generation for ICT objects.""" "" +from typing import Union, Dict, Any, TYPE_CHECKING + +if TYPE_CHECKING: + from sophios.api.utils.ict.ict_spec.model import ICT + + +def requirements(ict_: "ICT", network_access: bool) -> dict: + """Return the requirements from an ICT object.""" + reqs: Dict[Any, Any] = {} + reqs["DockerRequirement"] = {"dockerPull": ict_.container} + output_names = [io.name for io in ict_.outputs] + if "outDir" in output_names: + reqs["InitialWorkDirRequirement"] = { + "listing": [{"entry": "$(inputs.outDir)", "writable": True}] + } + reqs["InlineJavascriptRequirement"] = {} + if network_access: + reqs["NetworkAccess"] = {"networkAccess": True} + return reqs + + +def clt_dict(ict_: "ICT", network_access: bool) -> dict: + """Return a dict of a CommandLineTool from an ICT object.""" + + clt_: Dict[Any, Any] = { + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "inputs": { + io.name: io._input_to_cwl() # pylint: disable=W0212 + for io in ict_.inputs + ict_.outputs + }, + "outputs": { + io.name: io._output_to_cwl( + [io.name for io in ict_.inputs] + ) # pylint: disable=W0212 + for io in ict_.outputs + }, + "requirements": requirements(ict_, network_access), + "baseCommand": ict_.entrypoint, + "label": ict_.title, + "doc": str(ict_.documentation), + } + + return clt_ + + +def remove_none(d: Union[dict, str]) -> Union[dict, str]: + """Recursively remove keys with None values.""" + if isinstance(d, dict): + return {k: remove_none(v) for k, v in d.items() if v is not None} + elif isinstance(d, str): + return d # Return the string unchanged + else: + return d # Return other types of values unchanged + + +def input_output_dict(ict_: "ICT") -> Union[dict, str]: + """Return a input or output dictionary from an ICT object.""" + io_dict: Dict[Any, Any] = {} + for prop in ict_: + io_dict[prop.name] = { # type: ignore + "type": prop.io_type.value, # type: ignore + "description": prop.description, # type: ignore + "defaultValue": prop.defaultValue, # type: ignore + "required": prop.required, # type: ignore + "format": prop.io_format, # type: ignore + } + # recursively remove keys with None values + return remove_none(io_dict) + + +def ui_dict(ict_: "ICT") -> list: + """Return a CommandLineTool from an ICT object.""" + ui_list = [] + for prop in ict_: + prop_dict: Dict[Any, Any] = { + "key": prop.key.root, # type: ignore # Assuming 'root' attribute contains the actual key + "title": prop.title, # type: ignore + "description": prop.description, # type: ignore + "type": prop.ui_type, # type: ignore + } + if prop.customType: # type: ignore + prop_dict["customType"] = prop.customType # type: ignore + if prop.condition: # type: ignore + prop_dict["condition"] = prop.condition.root # type: ignore + if prop.ui_type == "select": # type: ignore + prop_dict["fields"] = prop.fields # type: ignore + ui_list.append(prop_dict) + return ui_list + + +def hardware_dict(ict_: "ICT") -> dict: + """Return a CommandLineTool from an ICT object.""" + hardware_dict = { + "cpu.type": ict_.cpu_type, # type: ignore + "cpu.min": ict_.cpu_min, # type: ignore + "cpu.recommended": ict_.cpu_recommended, # type: ignore + "memory.min": ict_.memory_min, # type: ignore + "memory.recommended": ict_.memory_recommended, # type: ignore + "gpu.enabled": ict_.gpu_enabled, # type: ignore + "gpu.required": ict_.gpu_required, # type: ignore + "gpu.type": ict_.gpu_type, # type: ignore + } + return hardware_dict + + +def ict_dict(ict_: "ICT") -> dict: + """Return a CommandLineTool from an ICT object.""" + inputs_dict = input_output_dict(ict_.inputs) # type: ignore + outputs_dict = input_output_dict(ict_.outputs) # type: ignore + clt_ = { + "inputs": inputs_dict, + "outputs": outputs_dict, + "ui": ui_dict(ict_.ui), # type: ignore + } + if ict_.hardware is not None: + clt_["hardware"] = hardware_dict(ict_.hardware) # type: ignore + return clt_ diff --git a/src/sophios/api/utils/ict/ict_spec/ui/__init__.py b/src/sophios/api/utils/ict/ict_spec/ui/__init__.py new file mode 100644 index 00000000..d6fdf5ee --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/ui/__init__.py @@ -0,0 +1,27 @@ +"""UI objects.""" + +from .objects import ( + UICheckbox, + UIColor, + UIDatetime, + UIFile, + UIItem, + UIMultiselect, + UINumber, + UIPath, + UISelect, + UIText, +) + +__all__ = [ + "UICheckbox", + "UIColor", + "UIDatetime", + "UIFile", + "UIItem", + "UIMultiselect", + "UINumber", + "UIPath", + "UISelect", + "UIText", +] diff --git a/src/sophios/api/utils/ict/ict_spec/ui/objects.py b/src/sophios/api/utils/ict/ict_spec/ui/objects.py new file mode 100644 index 00000000..1a5111ca --- /dev/null +++ b/src/sophios/api/utils/ict/ict_spec/ui/objects.py @@ -0,0 +1,246 @@ +"""UI objects.""" + +import enum +import re +from typing import Annotated, Literal, Optional, Union, Any + +from pydantic import BaseModel, Field, RootModel, field_validator + + +class UIKey(RootModel): + """UIKey object.""" + + root: str + + @field_validator("root") + @classmethod + def check_ui_key(cls: Any, value: str) -> str: + """Check the UI key follows the correct format.""" + sp_ = value.split(".") # ruff: noqa: PLR2004 + if not len(sp_) == 2: + raise ValueError( + "The UI key must be in the format ." + ) + if not sp_[0] in ["inputs", "outputs"]: + raise ValueError( + "The UI key must be in the format ." + ) + return value + + def __repr__(self) -> str: + """Repr.""" + return f"'{self.root}'" + + +class TypesEnum(str, enum.Enum): + """Types enum.""" + + TEXT = "text" + NUMBER = "number" + CHECKBOX = "checkbox" + SELECT = "select" + MULTISELECT = "multiselect" + COLOR = "color" + DATETIME = "datetime" + PATH = "path" + FILE = "file" + + +class ConditionalStatement(RootModel): + """ConditionalStatement object.""" + + root: str + + @field_validator("root") + @classmethod + def check_conditional_statement(cls: Any, value: str) -> str: + """Check the conditional statement follows the correct format.""" + if not bool( + re.match(r"^(inputs|outputs)\.\w+(==|!=|<|>|<=|>=|&&)'?\w+'?$", value) + ): + raise ValueError( + "The conditional statement must be in the format ." + ) + return value + + def __repr__(self) -> str: + """Repr.""" + return f"'{self.root}'" + + +class UIBase(BaseModel): + """UI BaseModel.""" + + key: UIKey = Field( + description="Identifier to connect UI configuration to specific parameter, " + + "should take the form ..", + examples=["inputs.thresholdvalue"], + ) + title: str = Field( + description="User friendly label used in UI.", + examples=["Thresholding Value"], + ) + description: Optional[str] = Field( + None, + description="Short user friendly instructions for selecting appropriate parameter.", + examples=["Enter a threshold value"], + ) + customType: Optional[str] = Field( + None, description="Optional label for a non-standard expected user interface." + ) + condition: Optional[ConditionalStatement] = Field( + None, + json_schema_extra={"pattern": "^(inputs|outputs)\.\w+(==|!=|<|>|<=|>=|&&)\w+$"}, + description="Conditional statement that resolves to a boolean value based on UI configuration and selected value, " + + "used to dictate relationship between parameters.", + examples=["inputs.thresholdtype=='Manual'"], + ) + + +class UIText(UIBase, extra="forbid"): + """Any arbitrary length string.""" + + default: Optional[str] = Field(None, description="Prefilled value.") + regex: Optional[str] = Field(None, description="Regular expression for validation.") + toolbar: Optional[bool] = Field( + None, description="Boolean value to add text formatting toolbar." + ) + ui_type: Literal["text"] = Field( + ..., + alias="type", + description="Defines the expected user interface based on a set of basic UI types.", + ) + + +class UINumber(UIBase, extra="forbid"): + """Any numerical value.""" + + default: Optional[Union[int, float]] = Field(None, description="Prefilled value.") + integer: Optional[bool] = Field( + None, description="Boolean value to force integers only." + ) + number_range: Optional[tuple[Union[int, float], Union[int, float]]] = Field( + None, alias="range", description="Minimum and maximum range as a tuple." + ) + ui_type: Literal["number"] = Field( + ..., + alias="type", + description="Defines the expected user interface based on a set of basic UI types.", + ) + + +class UICheckbox(UIBase, extra="forbid"): + """Boolean operator, checked for `true` unchecked for `false`.""" + + default: Optional[bool] = Field( + None, description="Prefilled value, either `true` or `false`." + ) + ui_type: Literal["checkbox"] = Field( + ..., + alias="type", + description="Defines the expected user interface based on a set of basic UI types.", + ) + + +class UISelect(UIBase, extra="forbid"): + """Single string value from a set of options.""" + + fields: list[str] = Field(description="Required array of options.") + optional: Optional[bool] = Field(None, description="Leave blank by default.") + ui_type: Literal["select"] = Field( + ..., + alias="type", + description="Defines the expected user interface based on a set of basic UI types.", + ) + + +class UIMultiselect(UIBase, extra="forbid"): + """One or more string values from a set of options.""" + + fields: list[str] = Field(description="Required array of options.") + optional: Optional[bool] = Field(None, description="Leave blank by default.") + limit: Optional[int] = Field(None, description="Maximum number of selections.") + ui_type: Literal["multiselect"] = Field( + ..., + alias="type", + description="Defines the expected user interface based on a set of basic UI types.", + ) + + +class UIColor(UIBase, extra="forbid"): + """Color values passed as RGB color values.""" + + fields: list[int] = Field(description="Array of preset RGB selections.") + ui_type: Literal["color"] = Field( + ..., + alias="type", + description="Defines the expected user interface based on a set of basic UI types.", + ) + + +class W3Format(str, enum.Enum): + """W3Format enum.""" + + YEAR = "YYYY" + YEAR_MONTH = "YYYY-MM" + COMPLETE_DATE = "YYYY-MM-DD" + COMPLETE_DATE_TIME = "YYYY-MM-DDThh:mmTZD" + COMPLETE_DATE_TIME_SEC = "YYYY-MM-DDThh:mm:ssTZD" + COMPLETE_DATE_TIME_MS = "YYYY-MM-DDThh:mm:ss.sTZD" + + +class UIDatetime(UIBase, extra="forbid"): + """Standardized date and time values.""" + + w3_format: W3Format = Field( + alias="format", description="Datetime format using W3C conventions." + ) + ui_type: Literal["datetime"] = Field( + ..., + alias="type", + description="Defines the expected user interface based on a set of basic UI types.", + ) + + +class UIPath(UIBase, extra="forbid"): + """Absolute or relative path to file/directory using Unix conventions.""" + + ext: Optional[list[str]] = Field( + None, description="Array of allowed file extensions." + ) + ui_type: Literal["path"] = Field( + ..., + alias="type", + description="Defines the expected user interface based on a set of basic UI types.", + ) + + +class UIFile(UIBase, extra="forbid"): + """User uploaded binary data.""" + + ext: Optional[list[str]] = Field( + None, description="Array of allowed file extensions." + ) + limit: Optional[int] = Field(None, description="Maximum number of uploaded files.") + size: Optional[int] = Field(None, description="Total size file limit.") + ui_type: Literal["file"] = Field( + ..., + alias="type", + description="Defines the expected user interface based on a set of basic UI types.", + ) + + +UIItem = Annotated[ + Union[ + UIText, + UINumber, + UICheckbox, + UISelect, + UIMultiselect, + UIColor, + UIDatetime, + UIPath, + UIFile, + ], + Field(discriminator="ui_type"), +] diff --git a/tests/data/__init__.py b/tests/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/ict_data/czi_extract/czi_extract_clt.json b/tests/data/ict_data/czi_extract/czi_extract_clt.json new file mode 100644 index 00000000..9fcd1ae2 --- /dev/null +++ b/tests/data/ict_data/czi_extract/czi_extract_clt.json @@ -0,0 +1,43 @@ +{ + "baseCommand": "[python3, main.py]", + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "doc": "None", + "inputs": { + "inpDir": { + "inputBinding": { + "prefix": "--inpDir" + }, + "type": "Directory" + }, + "outDir": { + "inputBinding": { + "prefix": "--outDir" + }, + "type": "Directory" + } + }, + "label": "Extract TIFFs From CZI", + "outputs": { + "outDir": { + "outputBinding": { + "glob": "$(inputs.outDir.basename)" + }, + "type": "File" + } + }, + "requirements": { + "DockerRequirement": { + "dockerPull": "polusai/czi-extract-plugin:1.1.1" + }, + "InitialWorkDirRequirement": { + "listing": [ + { + "entry": "$(inputs.outDir)", + "writable": true + } + ] + }, + "InlineJavascriptRequirement": {} + } +} \ No newline at end of file diff --git a/tests/data/ict_data/czi_extract/czi_extract_ict.json b/tests/data/ict_data/czi_extract/czi_extract_ict.json new file mode 100644 index 00000000..625bc9c5 --- /dev/null +++ b/tests/data/ict_data/czi_extract/czi_extract_ict.json @@ -0,0 +1,53 @@ +{ + "author": [ + "Author One" + ], + "contact": "test@test.com", + "container": "polusai/czi-extract-plugin:1.1.1", + "description": "Extracts individual fields of view from a CZI file. Saves as OME TIFF.", + "entrypoint": "[python3, main.py]", + "inputs": [ + { + "description": "Input collection", + "format": [ + "genericData" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Input collection", + "format": [ + "genericData" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "name": "polusai/ExtractTIFFsFromCZI", + "outputs": [ + { + "description": "Output data for the plugin", + "format": [ + "collection" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/polusai/image-tools", + "specVersion": "1.0.0", + "title": "Extract TIFFs From CZI", + "ui": [ + { + "description": "Collection name...", + "key": "inputs.inpDir", + "title": "Input collection: ", + "type": "path" + } + ], + "version": "1.1.1" +} \ No newline at end of file diff --git a/tests/data/ict_data/label_to_vector/label_to_vector_clt.json b/tests/data/ict_data/label_to_vector/label_to_vector_clt.json new file mode 100644 index 00000000..4bf5370c --- /dev/null +++ b/tests/data/ict_data/label_to_vector/label_to_vector_clt.json @@ -0,0 +1,49 @@ +{ + "baseCommand": "[python3, main.py]", + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "doc": "None", + "inputs": { + "filePattern": { + "inputBinding": { + "prefix": "--filePattern" + }, + "type": "string?" + }, + "inpDir": { + "inputBinding": { + "prefix": "--inpDir" + }, + "type": "Directory" + }, + "outDir": { + "inputBinding": { + "prefix": "--outDir" + }, + "type": "Directory" + } + }, + "label": "Label to Vector", + "outputs": { + "outDir": { + "outputBinding": { + "glob": "$(inputs.outDir.basename)" + }, + "type": "File" + } + }, + "requirements": { + "DockerRequirement": { + "dockerPull": "polusai/label-to-vector-tool:0.7.1-dev0" + }, + "InitialWorkDirRequirement": { + "listing": [ + { + "entry": "$(inputs.outDir)", + "writable": true + } + ] + }, + "InlineJavascriptRequirement": {} + } +} \ No newline at end of file diff --git a/tests/data/ict_data/label_to_vector/label_to_vector_ict.json b/tests/data/ict_data/label_to_vector/label_to_vector_ict.json new file mode 100644 index 00000000..8387c88e --- /dev/null +++ b/tests/data/ict_data/label_to_vector/label_to_vector_ict.json @@ -0,0 +1,69 @@ +{ + "author": [ + "Author One", + "Author Two" + ], + "contact": "test@test.com", + "container": "polusai/label-to-vector-tool:0.7.1-dev0", + "description": "Convert labelled masks to flow-field vectors.", + "entrypoint": "[python3, main.py]", + "inputs": [ + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "genericData" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Image-name pattern to use when selecting images for processing.", + "format": [ + "string" + ], + "name": "filePattern", + "required": false, + "type": "string" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "genericData" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "name": "polusai/LabeltoVector", + "outputs": [ + { + "description": "Output collection", + "format": [ + "genericData" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins/tree/dev/formats/polus-vector-converter-plugins", + "specVersion": "1.0.0", + "title": "Label to Vector", + "ui": [ + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input collection", + "type": "path" + }, + { + "description": "Image-name pattern to use when selecting images for processing.", + "key": "inputs.filePattern", + "title": "File Pattern", + "type": "text" + } + ], + "version": "0.7.1-dev0" +} \ No newline at end of file diff --git a/tests/data/ict_data/ome_conversion/ome_conversion_clt.json b/tests/data/ict_data/ome_conversion/ome_conversion_clt.json new file mode 100644 index 00000000..3656cf47 --- /dev/null +++ b/tests/data/ict_data/ome_conversion/ome_conversion_clt.json @@ -0,0 +1,55 @@ +{ + "baseCommand": "python3 -m polus.images.formats.ome_converter", + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "doc": "None", + "inputs": { + "fileExtension": { + "inputBinding": { + "prefix": "--fileExtension" + }, + "type": "string" + }, + "filePattern": { + "inputBinding": { + "prefix": "--filePattern" + }, + "type": "string" + }, + "inpDir": { + "inputBinding": { + "prefix": "--inpDir" + }, + "type": "Directory" + }, + "outDir": { + "inputBinding": { + "prefix": "--outDir" + }, + "type": "Directory" + } + }, + "label": "OME Converter", + "outputs": { + "outDir": { + "outputBinding": { + "glob": "$(inputs.outDir.basename)" + }, + "type": "File" + } + }, + "requirements": { + "DockerRequirement": { + "dockerPull": "polusai/ome-converter-tool:0.3.2-dev0" + }, + "InitialWorkDirRequirement": { + "listing": [ + { + "entry": "$(inputs.outDir)", + "writable": true + } + ] + }, + "InlineJavascriptRequirement": {} + } +} \ No newline at end of file diff --git a/tests/data/ict_data/ome_conversion/ome_conversion_ict.json b/tests/data/ict_data/ome_conversion/ome_conversion_ict.json new file mode 100644 index 00000000..7e81c3fa --- /dev/null +++ b/tests/data/ict_data/ome_conversion/ome_conversion_ict.json @@ -0,0 +1,89 @@ +{ + "author": [ + "Author One", + "Author Two" + ], + "contact": "test@test.com", + "container": "polusai/ome-converter-tool:0.3.2-dev0", + "description": "Convert Bioformats supported format to OME Zarr or OME TIF", + "entrypoint": "python3 -m polus.images.formats.ome_converter", + "inputs": [ + { + "description": "Input generic data collection to be processed by this plugin", + "format": [ + "genericData" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "A filepattern, used to select data to be converted", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Type of data conversion", + "format": [ + "enum" + ], + "name": "fileExtension", + "required": true, + "type": "string" + }, + { + "description": "Output collection", + "format": [ + "genericData" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "name": "polusai/OMEConverter", + "outputs": [ + { + "description": "Output collection", + "format": [ + "genericData" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "title": "OME Converter", + "ui": [ + { + "description": "Input generic data collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input generic collection", + "type": "path" + }, + { + "description": "A filepattern, used to select data for conversion", + "key": "inputs.filePattern", + "title": "Filepattern", + "type": "text" + }, + { + "description": "Type of data conversion", + "fields": [ + ".ome.tif", + ".ome.zarr", + "default" + ], + "key": "inputs.fileExtension", + "title": "fileExtension", + "type": "select" + } + ], + "version": "0.3.2-dev0" +} \ No newline at end of file diff --git a/tests/test_ict_to_clt_conversion.py b/tests/test_ict_to_clt_conversion.py new file mode 100644 index 00000000..69c62a71 --- /dev/null +++ b/tests/test_ict_to_clt_conversion.py @@ -0,0 +1,61 @@ +import pytest +import json +import pathlib + +from sophios.api.utils.converter import ict_to_clt + + +@pytest.mark.fast +def test_ict_to_clt_label_to_vector_conversion() -> None: + + path = pathlib.Path(__file__).parent.resolve() + + with open( + path / "data/ict_data/label_to_vector/label_to_vector_ict.json", "r" + ) as file: + label_to_vector_ict = json.load(file) + + with open( + path / "data/ict_data/label_to_vector/label_to_vector_clt.json", "r" + ) as file: + label_to_vector_clt = json.load(file) + + result = ict_to_clt(label_to_vector_ict) + + assert result == label_to_vector_clt + + +@pytest.mark.fast +def test_ict_to_clt_ome_conversion() -> None: + + path = pathlib.Path(__file__).parent.resolve() + + with open( + path / "data/ict_data/ome_conversion/ome_conversion_ict.json", "r" + ) as file: + ome_conversion_ict = json.load(file) + + with open( + path / "data/ict_data/ome_conversion/ome_conversion_clt.json", "r" + ) as file: + ome_conversion_clt = json.load(file) + + result = ict_to_clt(ome_conversion_ict) + + assert result == ome_conversion_clt + + +@pytest.mark.fast +def test_ict_to_clt_czi_extract_conversion() -> None: + + path = pathlib.Path(__file__).parent.resolve() + + with open(path / "data/ict_data/czi_extract/czi_extract_ict.json", "r") as file: + czi_extract_ict = json.load(file) + + with open(path / "data/ict_data/czi_extract/czi_extract_clt.json", "r") as file: + czi_extract_clt = json.load(file) + + result = ict_to_clt(czi_extract_ict) + + assert result == czi_extract_clt From 2a427d8e2f82fa7effba5330a5ba2f2f7129e805 Mon Sep 17 00:00:00 2001 From: Nazanin Donyapour <31333163+ndonyapour@users.noreply.github.com> Date: Tue, 1 Oct 2024 11:04:49 -0600 Subject: [PATCH 05/15] replace --user-space-docker-cmd with --singularity (#278) * replace --user-space-docker-cmd with --singularity * fix cwl-docker-extract --- src/sophios/main.py | 2 +- src/sophios/run_local.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/sophios/main.py b/src/sophios/main.py index e7ba57a0..23837057 100644 --- a/src/sophios/main.py +++ b/src/sophios/main.py @@ -204,7 +204,7 @@ def main() -> None: # `docker run` will NOT query the remote repository for the latest image! # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. if args.container_engine == 'singularity': - cmd = ['cwl-docker-extract', f'-s --dir {args.homedir}', f'autogenerated/{yaml_stem}.cwl'] + cmd = ['cwl-docker-extract', '-s', '--dir', f'{args.homedir}', f'autogenerated/{yaml_stem}.cwl'] else: cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{yaml_stem}.cwl'] sub.run(cmd, check=True) diff --git a/src/sophios/run_local.py b/src/sophios/run_local.py index 0b8436af..57034123 100644 --- a/src/sophios/run_local.py +++ b/src/sophios/run_local.py @@ -134,7 +134,13 @@ def run_local(args: argparse.Namespace, rose_tree: RoseTree, cachedir: Optional[ cachedir_ = ['--cachedir', cachedir] if cachedir else [] net = ['--custom-net', args.custom_net] if args.custom_net else [] provenance = ['--provenance', f'provenance/{yaml_stem}'] if not args.no_provenance else [] - docker_cmd_ = [] if docker_cmd == 'docker' else ['--user-space-docker-cmd', docker_cmd] + docker_cmd_: List[str] = [] + if docker_cmd == 'docker': + docker_cmd_ = [] + elif docker_cmd == 'singularity': + docker_cmd_ = ['--singularity'] + else: + docker_cmd_ = ['--user-space-docker-cmd', docker_cmd] write_summary = ['--write-summary', f'output_{yaml_stem}.json'] path_check = ['--relax-path-checks'] # See https://github.com/common-workflow-language/cwltool/blob/5a645dfd4b00e0a704b928cc0bae135b0591cc1a/cwltool/command_line_tool.py#L94 @@ -205,7 +211,13 @@ def run_local(args: argparse.Namespace, rose_tree: RoseTree, cachedir: Optional[ # NOTE: toil-cwl-runner always runs in parallel net = ['--custom-net', args.custom_net] if args.custom_net else [] provenance = ['--provenance', f'provenance/{yaml_stem}'] if not args.no_provenance else [] - docker_cmd_ = [] if docker_cmd == 'docker' else ['--user-space-docker-cmd', docker_cmd] + docker_cmd_ = [] + if docker_cmd == 'docker': + docker_cmd_ = [] + elif docker_cmd == 'singularity': + docker_cmd_ = ['--singularity'] + else: + docker_cmd_ = ['--user-space-docker-cmd', docker_cmd] path_check = ['--relax-path-checks'] # See https://github.com/common-workflow-language/cwltool/blob/5a645dfd4b00e0a704b928cc0bae135b0591cc1a/cwltool/command_line_tool.py#L94 # https://github.com/DataBiosphere/toil/blob/6558c7f97fb37c6ef6f469c7ae614109050322f4/src/toil/options/cwl.py#L152 From 47caf2548d1c88d7864f464a33a6ab254c235de6 Mon Sep 17 00:00:00 2001 From: Sameeul Bashir Samee Date: Tue, 1 Oct 2024 21:22:24 -0400 Subject: [PATCH 06/15] Replace mambaforge with miniforge (#279) --- .github/workflows/fuzzy_compile_weekly.yml | 4 ++-- .github/workflows/lint_and_test.yml | 8 +++---- .github/workflows/lint_and_test_macos.yml | 4 ++-- .github/workflows/run_workflows.yml | 24 ++++++++++---------- .github/workflows/run_workflows_weekly.yml | 8 +++---- docker/Dockerfile_amazon | 6 ++--- docker/Dockerfile_amazon_pypy | 6 ++--- docker/Dockerfile_debian | 6 ++--- docker/Dockerfile_debian_pypy | 6 ++--- docker/Dockerfile_fedora | 6 ++--- docker/Dockerfile_fedora_pypy | 6 ++--- docker/Dockerfile_redhat | 6 ++--- docker/Dockerfile_redhat_pypy | 6 ++--- docker/Dockerfile_ubuntu | 6 ++--- docker/Dockerfile_ubuntu_pypy | 6 ++--- examples/scripts/Dockerfile_check_linear_fit | 2 +- examples/scripts/Dockerfile_scatter_plot | 2 +- install/install_conda.bat | 2 +- install/install_conda.sh | 4 ++-- install/install_pypy.sh | 2 +- install/install_system_deps.bat | 2 +- install/install_system_deps.sh | 2 +- 22 files changed, 62 insertions(+), 62 deletions(-) diff --git a/.github/workflows/fuzzy_compile_weekly.yml b/.github/workflows/fuzzy_compile_weekly.yml index 45c4589b..6fe24c56 100644 --- a/.github/workflows/fuzzy_compile_weekly.yml +++ b/.github/workflows/fuzzy_compile_weekly.yml @@ -67,11 +67,11 @@ jobs: ref: main path: image-workflows - - name: Setup mamba (linux, macos) + - name: Setup miniforge (linux, macos) if: runner.os != 'Windows' uses: conda-incubator/setup-miniconda@v3.0.1 with: - miniforge-variant: Mambaforge-pypy3 + miniforge-variant: Miniforge-pypy3 miniforge-version: latest environment-file: workflow-inference-compiler/install/system_deps.yml activate-environment: wic diff --git a/.github/workflows/lint_and_test.yml b/.github/workflows/lint_and_test.yml index 753b9a2e..ee1bdf54 100644 --- a/.github/workflows/lint_and_test.yml +++ b/.github/workflows/lint_and_test.yml @@ -133,22 +133,22 @@ jobs: # if: runner.os == 'Windows' # run: cd workflow-inference-compiler/install/ && echo " - pypy" >> system_deps_windows.yml - - name: Setup mamba (linux, macos) + - name: Setup miniforge (linux, macos) if: runner.os != 'Windows' uses: conda-incubator/setup-miniconda@v3.0.1 with: - miniforge-variant: Mambaforge-pypy3 + miniforge-variant: Miniforge-pypy3 miniforge-version: latest environment-file: workflow-inference-compiler/install/system_deps.yml activate-environment: wic channels: conda-forge python-version: "3.9.*" # pypy is not yet compatible with 3.10 and 3.11 - - name: Setup mamba (windows) + - name: Setup miniforge (windows) if: runner.os == 'Windows' uses: conda-incubator/setup-miniconda@v3.0.1 with: - miniforge-variant: Mambaforge-pypy3 + miniforge-variant: Miniforge-pypy3 miniforge-version: latest environment-file: workflow-inference-compiler/install/system_deps_windows.yml activate-environment: wic diff --git a/.github/workflows/lint_and_test_macos.yml b/.github/workflows/lint_and_test_macos.yml index 57773105..6c1d46c1 100644 --- a/.github/workflows/lint_and_test_macos.yml +++ b/.github/workflows/lint_and_test_macos.yml @@ -67,11 +67,11 @@ jobs: ref: main path: image-workflows - - name: Setup mamba (linux, macos) + - name: Setup miniforge (linux, macos) if: runner.os != 'Windows' uses: conda-incubator/setup-miniconda@v2.2.0 with: - miniforge-variant: Mambaforge-pypy3 + miniforge-variant: Miniforge-pypy3 miniforge-version: latest environment-file: workflow-inference-compiler/install/system_deps.yml activate-environment: wic diff --git a/.github/workflows/run_workflows.yml b/.github/workflows/run_workflows.yml index b074fa80..093abbf3 100644 --- a/.github/workflows/run_workflows.yml +++ b/.github/workflows/run_workflows.yml @@ -111,22 +111,22 @@ jobs: if: runner.os != 'Windows' run: cd workflow-inference-compiler/install/ && echo " - pypy" >> system_deps.yml - - name: Remove old mamba environment - if: always() - run: rm -rf "/home/$(whoami)/miniconda3/envs/wic_github_actions/" - # For self-hosted runners, make sure we install into a new mamba environment - # NOTE: Every time the github self-hosted runner executes, it sets "set -e" in ~/.bash_profile - # So if we rm -rf the old mamba environment without also removing the mamba init code in ~/.bash_profile - # (or removing the file altogether), then unless we immediately re-create the environment, - # (i.e. if we try to run any other commands between removing and re-creating the environment) - # we will get "EnvironmentNameNotFound: Could not find conda environment: wic_github_actions" - # and (again, due to "set -e") the workflow step will fail. + # - name: Remove old mamba environment + # if: always() + # run: rm -rf "/home/$(whoami)/miniconda3/envs/wic_github_actions/" + # # For self-hosted runners, make sure we install into a new mamba environment + # # NOTE: Every time the github self-hosted runner executes, it sets "set -e" in ~/.bash_profile + # # So if we rm -rf the old mamba environment without also removing the mamba init code in ~/.bash_profile + # # (or removing the file altogether), then unless we immediately re-create the environment, + # # (i.e. if we try to run any other commands between removing and re-creating the environment) + # # we will get "EnvironmentNameNotFound: Could not find conda environment: wic_github_actions" + # # and (again, due to "set -e") the workflow step will fail. - - name: Setup mamba (linux, macos) + - name: Setup miniforge (linux, macos) if: always() uses: conda-incubator/setup-miniconda@v3.0.1 with: - miniforge-variant: Mambaforge-pypy3 + miniforge-variant: Miniforge-pypy3 miniforge-version: latest environment-file: workflow-inference-compiler/install/system_deps.yml activate-environment: wic_github_actions diff --git a/.github/workflows/run_workflows_weekly.yml b/.github/workflows/run_workflows_weekly.yml index 6b5a2289..a6eab21f 100644 --- a/.github/workflows/run_workflows_weekly.yml +++ b/.github/workflows/run_workflows_weekly.yml @@ -80,15 +80,15 @@ jobs: # if: runner.os != 'Windows' # run: cd workflow-inference-compiler/install/ && echo " - pypy" >> system_deps.yml - - name: Setup mamba (linux, macos) + - name: Setup miniforge (linux, macos) if: always() uses: conda-incubator/setup-miniconda@v3.0.1 with: - miniforge-variant: Mambaforge # NOT Mambaforge-pypy3 - # Toil is (currently) incompatible with pypy. Mambaforge-pypy3 + miniforge-variant: Miniforge # NOT Miniforge-pypy3 + # Toil is (currently) incompatible with pypy. Miniforge-pypy3 # installs pypy in the base environment (only). Although we are using # another environment, better to avoid the problem altogether by - # not using Mambaforge-pypy3 + # not using Miniforge-pypy3 miniforge-version: latest environment-file: workflow-inference-compiler/install/system_deps.yml activate-environment: wic_github_actions diff --git a/docker/Dockerfile_amazon b/docker/Dockerfile_amazon index ac4cedea..6816d402 100644 --- a/docker/Dockerfile_amazon +++ b/docker/Dockerfile_amazon @@ -5,12 +5,12 @@ FROM amazonlinux # Install conda / mamba RUN yum update -y && yum install -y wget -RUN CONDA="Mambaforge-Linux-x86_64.sh" && \ +RUN CONDA="Miniforge3-Linux-x86_64.sh" && \ wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \ chmod +x $CONDA && \ - ./$CONDA -b -p /mambaforge && \ + ./$CONDA -b -p /miniforge && \ rm -f $CONDA -ENV PATH /mambaforge/bin:$PATH +ENV PATH /miniforge/bin:$PATH # Install wic RUN yum install -y git diff --git a/docker/Dockerfile_amazon_pypy b/docker/Dockerfile_amazon_pypy index 1069e18d..2d4d8f32 100644 --- a/docker/Dockerfile_amazon_pypy +++ b/docker/Dockerfile_amazon_pypy @@ -5,12 +5,12 @@ FROM amazonlinux # Install conda / mamba RUN yum update -y && yum install -y wget -RUN CONDA="Mambaforge-pypy3-Linux-x86_64.sh" && \ +RUN CONDA="Miniforge-pypy3-Linux-x86_64.sh" && \ wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \ chmod +x $CONDA && \ - ./$CONDA -b -p /mambaforge-pypy3 && \ + ./$CONDA -b -p /Miniforge-pypy3 && \ rm -f $CONDA -ENV PATH /mambaforge-pypy3/bin:$PATH +ENV PATH /Miniforge-pypy3/bin:$PATH # Install wic RUN yum install -y git diff --git a/docker/Dockerfile_debian b/docker/Dockerfile_debian index c7196e71..569128c4 100644 --- a/docker/Dockerfile_debian +++ b/docker/Dockerfile_debian @@ -5,12 +5,12 @@ FROM debian:stable-slim # Install conda / mamba RUN apt-get update -y && apt-get install -y wget -RUN CONDA="Mambaforge-Linux-x86_64.sh" && \ +RUN CONDA="Miniforge3-Linux-x86_64.sh" && \ wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \ chmod +x $CONDA && \ - ./$CONDA -b -p /mambaforge && \ + ./$CONDA -b -p /miniforge && \ rm -f $CONDA -ENV PATH /mambaforge/bin:$PATH +ENV PATH /miniforge/bin:$PATH # Install wic RUN apt-get install -y git diff --git a/docker/Dockerfile_debian_pypy b/docker/Dockerfile_debian_pypy index d52a4382..251912ac 100644 --- a/docker/Dockerfile_debian_pypy +++ b/docker/Dockerfile_debian_pypy @@ -5,12 +5,12 @@ FROM debian:stable-slim # Install conda / mamba RUN apt-get update -y && apt-get install -y wget -RUN CONDA="Mambaforge-pypy3-Linux-x86_64.sh" && \ +RUN CONDA="Miniforge-pypy3-Linux-x86_64.sh" && \ wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \ chmod +x $CONDA && \ - ./$CONDA -b -p /mambaforge-pypy3 && \ + ./$CONDA -b -p /Miniforge-pypy3 && \ rm -f $CONDA -ENV PATH /mambaforge-pypy3/bin:$PATH +ENV PATH /Miniforge-pypy3/bin:$PATH # Install wic RUN apt-get install -y git diff --git a/docker/Dockerfile_fedora b/docker/Dockerfile_fedora index b5b9429c..d96aba04 100644 --- a/docker/Dockerfile_fedora +++ b/docker/Dockerfile_fedora @@ -5,12 +5,12 @@ FROM fedora # Install conda / mamba RUN yum update -y && yum install -y wget -RUN CONDA="Mambaforge-Linux-x86_64.sh" && \ +RUN CONDA="Miniforge3-Linux-x86_64.sh" && \ wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \ chmod +x $CONDA && \ - ./$CONDA -b -p /mambaforge && \ + ./$CONDA -b -p /miniforge && \ rm -f $CONDA -ENV PATH /mambaforge/bin:$PATH +ENV PATH /miniforge/bin:$PATH # Install wic RUN yum install -y git diff --git a/docker/Dockerfile_fedora_pypy b/docker/Dockerfile_fedora_pypy index 30434d17..b67bad5c 100644 --- a/docker/Dockerfile_fedora_pypy +++ b/docker/Dockerfile_fedora_pypy @@ -5,12 +5,12 @@ FROM fedora # Install conda / mamba RUN yum update -y && yum install -y wget -RUN CONDA="Mambaforge-pypy3-Linux-x86_64.sh" && \ +RUN CONDA="Miniforge-pypy3-Linux-x86_64.sh" && \ wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \ chmod +x $CONDA && \ - ./$CONDA -b -p /mambaforge-pypy3 && \ + ./$CONDA -b -p /Miniforge-pypy3 && \ rm -f $CONDA -ENV PATH /mambaforge-pypy3/bin:$PATH +ENV PATH /Miniforge-pypy3/bin:$PATH # Install wic RUN yum install -y git diff --git a/docker/Dockerfile_redhat b/docker/Dockerfile_redhat index a05be53f..06304acf 100644 --- a/docker/Dockerfile_redhat +++ b/docker/Dockerfile_redhat @@ -6,12 +6,12 @@ FROM redhat/ubi9-minimal # Install conda / mamba RUN microdnf update -y && microdnf install -y wget -RUN CONDA="Mambaforge-Linux-x86_64.sh" && \ +RUN CONDA="Miniforge3-Linux-x86_64.sh" && \ wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \ chmod +x $CONDA && \ - ./$CONDA -b -p /mambaforge && \ + ./$CONDA -b -p /miniforge && \ rm -f $CONDA -ENV PATH /mambaforge/bin:$PATH +ENV PATH /miniforge/bin:$PATH # Install wic RUN microdnf install -y git diff --git a/docker/Dockerfile_redhat_pypy b/docker/Dockerfile_redhat_pypy index 852ea872..9e2a5556 100644 --- a/docker/Dockerfile_redhat_pypy +++ b/docker/Dockerfile_redhat_pypy @@ -6,12 +6,12 @@ FROM redhat/ubi9-minimal # Install conda / mamba RUN microdnf update -y && microdnf install -y wget -RUN CONDA="Mambaforge-pypy3-Linux-x86_64.sh" && \ +RUN CONDA="Miniforge-pypy3-Linux-x86_64.sh" && \ wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \ chmod +x $CONDA && \ - ./$CONDA -b -p /mambaforge-pypy3 && \ + ./$CONDA -b -p /Miniforge-pypy3 && \ rm -f $CONDA -ENV PATH /mambaforge-pypy3/bin:$PATH +ENV PATH /Miniforge-pypy3/bin:$PATH # Install wic RUN microdnf install -y git diff --git a/docker/Dockerfile_ubuntu b/docker/Dockerfile_ubuntu index 9cd5dc5d..2162d5c5 100644 --- a/docker/Dockerfile_ubuntu +++ b/docker/Dockerfile_ubuntu @@ -5,12 +5,12 @@ FROM ubuntu # Install conda / mamba RUN apt-get update -y && apt-get install -y wget -RUN CONDA="Mambaforge-Linux-x86_64.sh" && \ +RUN CONDA="Miniforge3-Linux-x86_64.sh" && \ wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \ chmod +x $CONDA && \ - ./$CONDA -b -p /mambaforge && \ + ./$CONDA -b -p /miniforge && \ rm -f $CONDA -ENV PATH /mambaforge/bin:$PATH +ENV PATH /miniforge/bin:$PATH # Install wic RUN apt-get install -y git diff --git a/docker/Dockerfile_ubuntu_pypy b/docker/Dockerfile_ubuntu_pypy index 132ec1bb..27ab9c3e 100644 --- a/docker/Dockerfile_ubuntu_pypy +++ b/docker/Dockerfile_ubuntu_pypy @@ -5,12 +5,12 @@ FROM ubuntu # Install conda / mamba RUN apt-get update -y && apt-get install -y wget -RUN CONDA="Mambaforge-pypy3-Linux-x86_64.sh" && \ +RUN CONDA="Miniforge-pypy3-Linux-x86_64.sh" && \ wget --quiet https://github.com/conda-forge/miniforge/releases/latest/download/$CONDA && \ chmod +x $CONDA && \ - ./$CONDA -b -p /mambaforge-pypy3 && \ + ./$CONDA -b -p /Miniforge-pypy3 && \ rm -f $CONDA -ENV PATH /mambaforge-pypy3/bin:$PATH +ENV PATH /Miniforge-pypy3/bin:$PATH # Install wic RUN apt-get install -y git diff --git a/examples/scripts/Dockerfile_check_linear_fit b/examples/scripts/Dockerfile_check_linear_fit index 8aef3a94..b611ff69 100644 --- a/examples/scripts/Dockerfile_check_linear_fit +++ b/examples/scripts/Dockerfile_check_linear_fit @@ -1,4 +1,4 @@ -FROM condaforge/mambaforge-pypy3 +FROM condaforge/Miniforge-pypy3 RUN mamba install -c conda-forge numpy scipy diff --git a/examples/scripts/Dockerfile_scatter_plot b/examples/scripts/Dockerfile_scatter_plot index 68d6dc12..c3072ba5 100644 --- a/examples/scripts/Dockerfile_scatter_plot +++ b/examples/scripts/Dockerfile_scatter_plot @@ -1,4 +1,4 @@ -FROM condaforge/mambaforge-pypy3 +FROM condaforge/Miniforge-pypy3 RUN mamba install -c conda-forge matplotlib diff --git a/install/install_conda.bat b/install/install_conda.bat index 5c8e1715..6afc2c31 100644 --- a/install/install_conda.bat +++ b/install/install_conda.bat @@ -1,4 +1,4 @@ -set CONDA="Mambaforge-pypy3-Windows-x86_64.exe" +set CONDA="Miniforge-pypy3-Windows-x86_64.exe" curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/%CONDA% %CONDA% del %CONDA% \ No newline at end of file diff --git a/install/install_conda.sh b/install/install_conda.sh index 07a75954..3c5f0c70 100755 --- a/install/install_conda.sh +++ b/install/install_conda.sh @@ -1,7 +1,7 @@ #!/bin/bash -e -CONDA="Mambaforge-pypy3-$(uname)-$(uname -m).sh" +CONDA="Miniforge-pypy3-$(uname)-$(uname -m).sh" curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/"$CONDA" chmod +x "$CONDA" ./"$CONDA" -b -~/mambaforge-pypy3/bin/mamba init +~/Miniforge-pypy3/bin/mamba init rm -f "$CONDA" \ No newline at end of file diff --git a/install/install_pypy.sh b/install/install_pypy.sh index 6456ef14..d380412f 100755 --- a/install/install_pypy.sh +++ b/install/install_pypy.sh @@ -1,7 +1,7 @@ #!/bin/bash -e # NOTE: mamba is a drop-in replacement for conda, just much faster. # (i.e. You can replace mamba with conda below.) -# See https://github.com/conda-forge/miniforge#mambaforge-pypy3 +# See https://github.com/conda-forge/miniforge#Miniforge-pypy3 CONDA=conda if [ "$(which mamba)" ]; then CONDA=mamba diff --git a/install/install_system_deps.bat b/install/install_system_deps.bat index b6f4c470..9d137f14 100644 --- a/install/install_system_deps.bat +++ b/install/install_system_deps.bat @@ -1,6 +1,6 @@ :: NOTE: mamba is a drop-in replacement for conda, just much faster. :: (i.e. You can replace mamba with conda below.) -:: See https://github.com/conda-forge/miniforge#mambaforge-pypy3 +:: See https://github.com/conda-forge/miniforge#Miniforge-pypy3 set CONDA=conda where /q mamba if not ERRORLEVEL 1 (set CONDA=mamba) diff --git a/install/install_system_deps.sh b/install/install_system_deps.sh index 54a04192..f890896a 100755 --- a/install/install_system_deps.sh +++ b/install/install_system_deps.sh @@ -1,7 +1,7 @@ #!/bin/bash -e # NOTE: mamba is a drop-in replacement for conda, just much faster. # (i.e. You can replace mamba with conda below.) -# See https://github.com/conda-forge/miniforge#mambaforge-pypy3 +# See https://github.com/conda-forge/miniforge#Miniforge-pypy3 CONDA=conda if [ "$(which mamba)" ]; then CONDA=mamba From 32f51e411974cace00c229de0c6951bbfe8244fd Mon Sep 17 00:00:00 2001 From: Sameeul Bashir Samee Date: Tue, 1 Oct 2024 21:47:21 -0400 Subject: [PATCH 07/15] Fix ci miniforge v3 (#280) --- .github/workflows/run_workflows.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run_workflows.yml b/.github/workflows/run_workflows.yml index 093abbf3..737a674b 100644 --- a/.github/workflows/run_workflows.yml +++ b/.github/workflows/run_workflows.yml @@ -111,16 +111,16 @@ jobs: if: runner.os != 'Windows' run: cd workflow-inference-compiler/install/ && echo " - pypy" >> system_deps.yml - # - name: Remove old mamba environment - # if: always() - # run: rm -rf "/home/$(whoami)/miniconda3/envs/wic_github_actions/" - # # For self-hosted runners, make sure we install into a new mamba environment - # # NOTE: Every time the github self-hosted runner executes, it sets "set -e" in ~/.bash_profile - # # So if we rm -rf the old mamba environment without also removing the mamba init code in ~/.bash_profile - # # (or removing the file altogether), then unless we immediately re-create the environment, - # # (i.e. if we try to run any other commands between removing and re-creating the environment) - # # we will get "EnvironmentNameNotFound: Could not find conda environment: wic_github_actions" - # # and (again, due to "set -e") the workflow step will fail. + - name: Remove old mamba environment + if: always() + run: rm -rf "/home/$(whoami)/miniconda3/envs/wic_github_actions/" + # For self-hosted runners, make sure we install into a new mamba environment + # NOTE: Every time the github self-hosted runner executes, it sets "set -e" in ~/.bash_profile + # So if we rm -rf the old mamba environment without also removing the mamba init code in ~/.bash_profile + # (or removing the file altogether), then unless we immediately re-create the environment, + # (i.e. if we try to run any other commands between removing and re-creating the environment) + # we will get "EnvironmentNameNotFound: Could not find conda environment: wic_github_actions" + # and (again, due to "set -e") the workflow step will fail. - name: Setup miniforge (linux, macos) if: always() From e889cfacbd3493e59946d098e50e5a73ad1c8190 Mon Sep 17 00:00:00 2001 From: VasuJ <145879890+vjaganat90@users.noreply.github.com> Date: Thu, 3 Oct 2024 09:30:27 -0400 Subject: [PATCH 08/15] Fix for running workflows with singularity (#281) * singularity flag and cmd fix * fix checking for container_engine unlike docker on the platform --------- Co-authored-by: Vasu Jaganath --- src/sophios/api/pythonapi.py | 3 +- src/sophios/cli.py | 3 ++ src/sophios/main.py | 3 +- src/sophios/run_local.py | 98 ++++++++++++++++++++++-------------- tests/test_examples.py | 3 +- 5 files changed, 69 insertions(+), 41 deletions(-) diff --git a/src/sophios/api/pythonapi.py b/src/sophios/api/pythonapi.py index a37680c4..a5984c5f 100644 --- a/src/sophios/api/pythonapi.py +++ b/src/sophios/api/pythonapi.py @@ -760,7 +760,8 @@ def run(self) -> None: # `docker run` will NOT query the remote repository for the latest image! # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. if args.container_engine == 'singularity': - cmd = ['cwl-docker-extract', f'-s --dir {args.homedir}', f'autogenerated/{self.process_name}.cwl'] + cmd = ['cwl-docker-extract', '-s', '--dir', + f'{args.singularity_pull_dir}', f'autogenerated/{self.process_name}.cwl'] else: cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{self.process_name}.cwl'] sub.run(cmd, check=True) diff --git a/src/sophios/cli.py b/src/sophios/cli.py index c892d589..f5089601 100644 --- a/src/sophios/cli.py +++ b/src/sophios/cli.py @@ -28,6 +28,9 @@ parser.add_argument('--homedir', type=str, required=False, default=str(Path().home()), help='''The users home directory. This is necessary because CWL clears environment variables (e.g. HOME)''') +parser.add_argument('--singularity_pull_dir', type=str, required=False, default=str(Path().cwd()), + help='''The user specified pull directory for singularity image pull. + The default is the current working directory i.e. `pwd`''') parser.add_argument('--insert_steps_automatically', default=False, action="store_true", help='''Attempt to fix inference failures by speculatively inserting workflow steps from a curated whitelist.''') diff --git a/src/sophios/main.py b/src/sophios/main.py index 23837057..24a6dbdd 100644 --- a/src/sophios/main.py +++ b/src/sophios/main.py @@ -204,7 +204,8 @@ def main() -> None: # `docker run` will NOT query the remote repository for the latest image! # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. if args.container_engine == 'singularity': - cmd = ['cwl-docker-extract', '-s', '--dir', f'{args.homedir}', f'autogenerated/{yaml_stem}.cwl'] + cmd = ['cwl-docker-extract', '-s', '--dir', + f'{args.singularity_pull_dir}', f'autogenerated/{yaml_stem}.cwl'] else: cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{yaml_stem}.cwl'] sub.run(cmd, check=True) diff --git a/src/sophios/run_local.py b/src/sophios/run_local.py index 57034123..fe407775 100644 --- a/src/sophios/run_local.py +++ b/src/sophios/run_local.py @@ -63,47 +63,69 @@ def run_local(args: argparse.Namespace, rose_tree: RoseTree, cachedir: Optional[ Returns: retval: The return value """ - + docker_like_engines = ['docker', 'podman'] docker_cmd: str = args.container_engine # Check that docker is installed, so users don't get a nasty runtime error. - cmd = [docker_cmd, 'run', '--rm', 'hello-world'] - output = '' - try: - docker_cmd_exists = True - proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT) - output = proc.stdout.decode("utf-8") - except FileNotFoundError: - docker_cmd_exists = False - out_d = "Hello from Docker!" - out_p = "Hello Podman World" - permission_denied = 'permission denied while trying to connect to the Docker daemon socket at' - if (not docker_cmd_exists or not (proc.returncode == 0 and out_d in output or out_p in output)) and not args.ignore_docker_install: - if permission_denied in output: - print('Warning! docker appears to be installed, but not configured as a non-root user.') - print('See https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user') - print('TL;DR you probably just need to run the following command (and then restart your machine)') - print('sudo usermod -aG docker $USER') + if docker_cmd in docker_like_engines: + cmd = [docker_cmd, 'run', '--rm', 'hello-world'] + output = '' + try: + docker_cmd_exists = True + proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT) + output = proc.stdout.decode("utf-8") + except FileNotFoundError: + docker_cmd_exists = False + out_d = "Hello from Docker!" + out_p = "Hello Podman World" + permission_denied = 'permission denied while trying to connect to the Docker daemon socket at' + if ((not docker_cmd_exists + or not (proc.returncode == 0 and out_d in output or out_p in output)) + and not args.ignore_docker_install): + + if permission_denied in output: + print('Warning! docker appears to be installed, but not configured as a non-root user.') + print('See https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user') + print('TL;DR you probably just need to run the following command (and then restart your machine)') + print('sudo usermod -aG docker $USER') + sys.exit(1) + + print(f'Warning! The {docker_cmd} command does not appear to be installed.') + print(f"""Most workflows require docker containers and + will fail at runtime if {docker_cmd} is not installed.""") + print('If you want to try running the workflow anyway, use --ignore_docker_install') + print("""Note that --ignore_docker_install does + NOT change whether or not any step in your workflow uses docker""") sys.exit(1) - print(f'Warning! The {docker_cmd} command does not appear to be installed.') - print(f'Most workflows require docker containers and will fail at runtime if {docker_cmd} is not installed.') - print('If you want to try running the workflow anyway, use --ignore_docker_install') - print('Note that --ignore_docker_install does NOT change whether or not any step in your workflow uses docker') - sys.exit(1) - - # If docker is installed, check for too many running processes. (on linux, macos) - if docker_cmd == 'docker' and docker_cmd_exists and sys.platform != "win32": - cmd = 'pgrep com.docker | wc -l' # type: ignore - proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT, shell=True) - output = proc.stdout.decode("utf-8") - num_processes = int(output.strip()) - max_processes = 1000 - if num_processes > max_processes and not args.ignore_docker_processes: - print(f'Warning! There are {num_processes} running docker processes.') - print(f'More than {max_processes} may potentially cause intermittent hanging issues.') - print('It is recommended to terminate the processes using the command') - print('`sudo pkill com.docker && sudo pkill Docker`') - print('and then restart Docker.') - print('If you want to run the workflow anyway, use --ignore_docker_processes') + + # If docker is installed, check for too many running processes. (on linux, macos) + if docker_cmd_exists and sys.platform != "win32": + cmd = 'pgrep com.docker | wc -l' # type: ignore + proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT, shell=True) + output = proc.stdout.decode("utf-8") + num_processes = int(output.strip()) + max_processes = 1000 + if num_processes > max_processes and not args.ignore_docker_processes: + print(f'Warning! There are {num_processes} running docker processes.') + print(f'More than {max_processes} may potentially cause intermittent hanging issues.') + print('It is recommended to terminate the processes using the command') + print('`sudo pkill com.docker && sudo pkill Docker`') + print('and then restart Docker.') + print('If you want to run the workflow anyway, use --ignore_docker_processes') + sys.exit(1) + else: + cmd = [docker_cmd, '--version'] + output = '' + try: + docker_cmd_exists = True + proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT) + output = proc.stdout.decode("utf-8") + except FileNotFoundError: + docker_cmd_exists = False + if not docker_cmd_exists and not args.ignore_docker_install: + print(f'Warning! The {docker_cmd} command does not appear to be installed.') + print('If you want to try running the workflow anyway, use --ignore_docker_install') + print('Note that --ignore_docker_install does NOT change whether or not') + print('any step in your workflow uses docker or any other containers') sys.exit(1) yaml_path = args.yaml diff --git a/tests/test_examples.py b/tests/test_examples.py index abc60f59..153fdedb 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -218,7 +218,8 @@ def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str, args: argp # `docker run` will NOT query the remote repository for the latest image! # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. if args.container_engine == 'singularity': - cmd = ['cwl-docker-extract', f'-s --dir {args.homedir}', f'autogenerated/{Path(yml_path).stem}.cwl'] + cmd = ['cwl-docker-extract', '-s', '--dir', + f'{args.singularity_pull_dir}', f'autogenerated/{Path(yml_path).stem}.cwl'] else: cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{Path(yml_path).stem}.cwl'] sub.run(cmd, check=True) From 1e797b5676ac10027a252f59b14a2a6160e63627 Mon Sep 17 00:00:00 2001 From: JesseMckinzie <72471813+JesseMckinzie@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:11:38 -0600 Subject: [PATCH 09/15] Add flag to only compile workflow to cwl without running (#272) * Add flag to only compile workflow to cwl without running * Switch --generate_cwl_workflow to group_run * Add test for --generate_cwl_workflow * Update generate cwl test to use helloworld from tutorials --------- Co-authored-by: Sameeul Bashir Samee --- src/sophios/cli.py | 1 - tests/data/cwl/helloworld.cwl | 24 ++++++++++++++++++++++++ tests/test_cli_flags.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tests/data/cwl/helloworld.cwl create mode 100644 tests/test_cli_flags.py diff --git a/src/sophios/cli.py b/src/sophios/cli.py index f5089601..6e7f3c68 100644 --- a/src/sophios/cli.py +++ b/src/sophios/cli.py @@ -88,7 +88,6 @@ help='After generating the cwl file(s), run it on your local machine.') group_run.add_argument('--generate_cwl_workflow', required=False, default=False, action="store_true", help='Compile the workflow without pulling the docker image') - parser.add_argument('--cwl_inline_subworkflows', default=False, action="store_true", help='Before generating the cwl file, inline all subworkflows.') parser.add_argument('--inference_disable', default=False, action="store_true", diff --git a/tests/data/cwl/helloworld.cwl b/tests/data/cwl/helloworld.cwl new file mode 100644 index 00000000..8bc6867c --- /dev/null +++ b/tests/data/cwl/helloworld.cwl @@ -0,0 +1,24 @@ +#!/usr/bin/env cwl-runner +# This file was autogenerated using the Workflow Inference Compiler, version 0+unknown +# https://github.com/PolusAI/workflow-inference-compiler +steps: +- id: helloworld__step__1__echo + in: + message: + source: helloworld__step__1__echo___message + run: helloworld__step__1__echo/echo.cwl + out: + - stdout +cwlVersion: v1.2 +class: Workflow +$namespaces: + edam: https://edamontology.org/ +$schemas: +- https://raw.githubusercontent.com/edamontology/edamontology/master/EDAM_dev.owl +inputs: + helloworld__step__1__echo___message: + type: string +outputs: + helloworld__step__1__echo___stdout: + type: File + outputSource: helloworld__step__1__echo/stdout diff --git a/tests/test_cli_flags.py b/tests/test_cli_flags.py new file mode 100644 index 00000000..c45fba48 --- /dev/null +++ b/tests/test_cli_flags.py @@ -0,0 +1,35 @@ +import pathlib +import subprocess +import yaml +import shutil + +from sophios.main import main +from sophios.cli import get_args + + +def test_generate_cwl_workflow() -> None: + """ + Test that running sophios with --generate_cwl_workflow produces the correct cwl file + """ + # remove output directory in case it exists to avoid false test pass + out_path = pathlib.Path('autogenerated') + + if out_path.exists() and out_path.is_dir(): + shutil.rmtree(out_path) + + yaml_path = str(pathlib.Path(__file__).parent.parent.resolve() / "docs/tutorials/helloworld.wic") + + args = ["sophios", "--yaml", yaml_path, "--generate_cwl_workflow"] + + # run sophios with args + subprocess.run(args) + + with open("autogenerated/helloworld.cwl", "r") as cwl_file: + result_dict = yaml.safe_load(cwl_file) + + with open( + str(pathlib.Path(__file__).parent.resolve() / "data/cwl/helloworld.cwl"), "r" + ) as cwl_file: + actual_dict = yaml.safe_load(cwl_file) + + assert result_dict == actual_dict From a908c882c1da122e77d19346ad44e3ebff03a524 Mon Sep 17 00:00:00 2001 From: VasuJ <145879890+vjaganat90@users.noreply.github.com> Date: Wed, 9 Oct 2024 11:13:07 -0400 Subject: [PATCH 10/15] REST API : add the logic of converting wfb plugins (ICT) to CWL (CLT) (#276) * add the logic of converting plugins (ICT) to CWL (CLT) and add a test for given wf.json * get output from /compile endpoint and run locally for CI * semi-fix the inline_cwl when using stock CLT names with REST API --------- Co-authored-by: Vasu Jaganath --- .github/workflows/run_workflows.yml | 5 + src/sophios/api/http/restapi.py | 19 +- src/sophios/api/utils/converter.py | 46 +- src/sophios/api/utils/ict/ict_spec/model.py | 16 +- .../api/utils/input_object_schema.json | 3 +- src/sophios/cli.py | 3 + src/sophios/compiler.py | 13 +- tests/rest_wfb_objects/multi_node_wfb.json | 815 ++++++++++++++++++ tests/test_rest_core.py | 127 ++- tests/test_rest_wfb.py | 34 + 10 files changed, 1047 insertions(+), 34 deletions(-) create mode 100644 tests/rest_wfb_objects/multi_node_wfb.json create mode 100644 tests/test_rest_wfb.py diff --git a/.github/workflows/run_workflows.yml b/.github/workflows/run_workflows.yml index 737a674b..4bf1d0ec 100644 --- a/.github/workflows/run_workflows.yml +++ b/.github/workflows/run_workflows.yml @@ -181,6 +181,11 @@ jobs: # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 run: cd workflow-inference-compiler/ && pytest tests/test_rest_core.py -k test_rest_core --cwl_runner cwltool + - name: PyTest Run REST WFB Tests + if: always() + # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 + run: cd workflow-inference-compiler/ && pytest tests/test_rest_wfb.py -k test_rest_wfb --cwl_runner cwltool + - name: PyTest Run ICT to CLT conversion Tests if: always() # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 diff --git a/src/sophios/api/http/restapi.py b/src/sophios/api/http/restapi.py index f2bb950f..01c337e0 100644 --- a/src/sophios/api/http/restapi.py +++ b/src/sophios/api/http/restapi.py @@ -95,11 +95,11 @@ async def compile_wf(request: Request) -> Json: req: Json = await request.json() # clean up and convert the incoming object # schema preserving - req = converter.raw_wfb_to_lean_wfb(req) + wfb_payload = converter.raw_wfb_to_lean_wfb(req) # schema non-preserving - workflow_temp = converter.wfb_to_wic(req) + workflow_temp = converter.wfb_to_wic(wfb_payload) wkflw_name = "generic_workflow" - args = get_args(wkflw_name, ['--inline_cwl_runtag']) + args = get_args(wkflw_name, ['--inline_cwl_runtag', '--generate_cwl_workflow']) # Build canonical workflow object workflow_can = utils_cwl.desugar_into_canonical_normal_form(workflow_temp) @@ -126,16 +126,17 @@ async def compile_wf(request: Request) -> Json: yaml_tree: YamlTree = YamlTree(StepId(wkflw_name, plugin_ns), workflow_can) # ========= COMPILE WORKFLOW ================ + args.ignore_dir_path = True compiler_info: CompilerInfo = compiler.compile_workflow(yaml_tree, args, [], [graph], {}, {}, {}, {}, tools_cwl, True, relative_run_path=True, testing=False) - # =========== OPTIONAL RUN ============== - print('---------- Run Workflow locally! ---------') - retval = run_workflow(compiler_info, args) - + rose_tree = compiler_info.rose + if args.inline_cwl_runtag: + input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) + rose_tree = plugins.cwl_update_inline_runtag_rosetree(rose_tree, Path('autogenerated/'), True) # ======== OUTPUT PROCESSING ================ # ========= PROCESS COMPILED OBJECT ========= - sub_node_data: NodeData = compiler_info.rose.data + sub_node_data: NodeData = rose_tree.data yaml_stem = sub_node_data.name cwl_tree = sub_node_data.compiled_cwl yaml_inputs = sub_node_data.workflow_inputs_file @@ -151,7 +152,7 @@ async def compile_wf(request: Request) -> Json: "cwlJobInputs": yaml_inputs_no_dd, **cwl_tree_run } - compute_workflow["retval"] = str(retval) + compute_workflow["retval"] = str(0) return compute_workflow diff --git a/src/sophios/api/utils/converter.py b/src/sophios/api/utils/converter.py index 8fdec813..c9df678b 100644 --- a/src/sophios/api/utils/converter.py +++ b/src/sophios/api/utils/converter.py @@ -43,7 +43,35 @@ def extract_state(inp: Json) -> Json: inp_restrict = copy.deepcopy(inp['state']) else: inp_inter = copy.deepcopy(inp) + # drop all 'internal' nodes and all edges with 'internal' nodes + step_nodes = [snode for snode in inp['state']['nodes'] if not snode['internal']] + step_node_ids = [step_node['id'] for step_node in step_nodes] + step_edges = [edg for edg in inp_inter['state']['links'] if edg['sourceId'] + in step_node_ids and edg['targetId'] in step_node_ids] + # overwrite 'links' and 'nodes' + inp_inter['state'].pop('nodes', None) + inp_inter['state'].pop('links', None) + inp_inter['state']['nodes'] = step_nodes + inp_inter['state']['links'] = step_edges + # massage the plugins + plugins = inp_inter['plugins'] + # drop incorrect/superfluous UI fields from plugins + # 'required' and 'format' + for ict_plugin in plugins: + for ui_elem in ict_plugin['ui']: + _, _ = ui_elem.pop('required', None), ui_elem.pop('format', None) + for out in ict_plugin['outputs']: + if out['name'] == 'outDir': + ict_plugin['inputs'].append(out) # Here goes the ICT to CLT extraction logic + for node in inp_inter['state']['nodes']: + node_pid = node["pluginId"] + plugin = next((ict for ict in plugins if ict['pid'] == node_pid), None) + clt: Json = {} + if plugin: + clt = ict_to_clt(plugin) + # just have the clt payload in run + node['run'] = clt inp_restrict = inp_inter['state'] return inp_restrict @@ -92,7 +120,7 @@ def wfb_to_wic(inp: Json) -> Cwl: ['outputs'].items()) # outputs always have to be list # remove these (now) superfluous keys node.pop('settings', None) - node.pop('pluginId', None) + node.pop('name', None) node.pop('internal', None) # setting the inputs of the non-sink nodes i.e. whose input doesn't depend on any other node's output @@ -106,7 +134,7 @@ def wfb_to_wic(inp: Json) -> Cwl: for node in non_sink_nodes: if node.get('in'): for nkey in node['in']: - node['in'][nkey] = yaml.load('!ii ' + node['in'][nkey], Loader=wic_loader()) + node['in'][nkey] = yaml.load('!ii ' + str(node['in'][nkey]), Loader=wic_loader()) # After outs are set for edg in inp_restrict['links']: @@ -124,16 +152,20 @@ def wfb_to_wic(inp: Json) -> Cwl: # we match the source output tag type to target input tag type # and connect them through '!* ' for input, all outputs are '!& ' before this for sk in src_out_keys: - tgt_node['in'][sk] = yaml.load('!* ' + tgt_node['in'][sk], Loader=wic_loader()) + # It maybe possible that (explicit) outputs of src nodes might not have corresponding + # (explicit) inputs in target node + if tgt_node['in'].get(sk): + tgt_node['in'][sk] = yaml.load('!* ' + tgt_node['in'][sk], Loader=wic_loader()) # the inputs which aren't dependent on previous/other steps # they are by default inline input diff_keys = set(tgt_in_keys) - set(src_out_keys) for dfk in diff_keys: - tgt_node['in'][dfk] = yaml.load('!ii ' + tgt_node['in'][dfk], Loader=wic_loader()) + tgt_node['in'][dfk] = yaml.load('!ii ' + str(tgt_node['in'][dfk]), Loader=wic_loader()) for node in inp_restrict['nodes']: - node['id'] = node['name'] # just reuse name as node's id, wic id is same as wfb name - node.pop('name', None) + # just reuse name as node's pluginId, wic id is same as wfb name + node['id'] = node['pluginId'].split('@')[0].replace('/', '_') + node.pop('pluginId', None) workflow_temp: Cwl = {} if inp_restrict["links"] != []: @@ -142,7 +174,7 @@ def wfb_to_wic(inp: Json) -> Cwl: workflow_temp["steps"].append(node) # node["cwlScript"] # Assume dict form else: # A single node workflow node = inp_restrict["nodes"][0] - workflow_temp = node["cwlScript"] + workflow_temp = node["cwlScript"] if node.get("cwlScript") else node['run'] return workflow_temp diff --git a/src/sophios/api/utils/ict/ict_spec/model.py b/src/sophios/api/utils/ict/ict_spec/model.py index 42609045..c542e2d0 100644 --- a/src/sophios/api/utils/ict/ict_spec/model.py +++ b/src/sophios/api/utils/ict/ict_spec/model.py @@ -39,14 +39,14 @@ def validate_ui(self) -> "ICT": inp_bool = [x in input_names for x in io_dict["inputs"]] out_bool = [x in output_names for x in io_dict["outputs"]] - if not all(inp_bool): - raise ValueError( - f"The ui keys must match the inputs and outputs keys. Unmatched: inputs.{set(io_dict['inputs'])-set(input_names)}" - ) - if not all(out_bool): - raise ValueError( - f"The ui keys must match the inputs and outputs keys. Unmatched: outputs.{set(io_dict['outputs'])-set(output_names)}" - ) + # if not all(inp_bool): + # raise ValueError( + # f"The ui keys must match the inputs and outputs keys. Unmatched: inputs.{set(io_dict['inputs'])-set(input_names)}" + # ) + # if not all(out_bool): + # raise ValueError( + # f"The ui keys must match the inputs and outputs keys. Unmatched: outputs.{set(io_dict['outputs'])-set(output_names)}" + # ) return self def to_clt(self, network_access: bool = False) -> dict: diff --git a/src/sophios/api/utils/input_object_schema.json b/src/sophios/api/utils/input_object_schema.json index bd3b0fc8..801f2fbf 100644 --- a/src/sophios/api/utils/input_object_schema.json +++ b/src/sophios/api/utils/input_object_schema.json @@ -223,7 +223,7 @@ "PluginX": { "properties": { "author": { - "type": "string" + "type": ["string", "array"] }, "baseCommand": { "items": { @@ -318,6 +318,7 @@ }, "required": [ "id", + "pid", "name", "version", "title", diff --git a/src/sophios/cli.py b/src/sophios/cli.py index 6e7f3c68..1f9fa8fe 100644 --- a/src/sophios/cli.py +++ b/src/sophios/cli.py @@ -41,6 +41,9 @@ help='Copies output files from the cachedir to outdir/ (automatically enabled with --run_local)') parser.add_argument('--inline_cwl_runtag', default=False, action="store_true", help='Copies cwl adapter file contents inline into the final .cwl in autogenerated/') +# This is a hidden flag +parser.add_argument('--ignore_dir_path', type=bool, + required=False, default=False, help=argparse.SUPPRESS) parser.add_argument('--parallel', default=False, action="store_true", help='''When running locally, execute independent steps in parallel. diff --git a/src/sophios/compiler.py b/src/sophios/compiler.py index f3abb0fb..353c7837 100644 --- a/src/sophios/compiler.py +++ b/src/sophios/compiler.py @@ -880,10 +880,15 @@ def compile_workflow_once(yaml_tree_ast: YamlTree, newval['format'] = in_format new_keyval = {key: newval} elif 'Directory' == in_dict['type']: - dir = Path(in_dict['value']) - if not dir.is_absolute(): - dir = Path('autogenerated') / dir - dir.mkdir(parents=True, exist_ok=True) + if not args.ignore_dir_path: + if in_dict['value'].startswith('/'): + print("Warning! directory can not start with '/'") + print("It is most likely an incorrect path! Can't create directories!") + sys.exit(1) + ldir = Path(in_dict['value']) + if not ldir.is_absolute(): + ldir = Path('autogenerated') / ldir + ldir.mkdir(parents=True, exist_ok=True) newval = {'class': 'Directory', 'location': in_dict['value']} new_keyval = {key: newval} # TODO: Check for all valid types? diff --git a/tests/rest_wfb_objects/multi_node_wfb.json b/tests/rest_wfb_objects/multi_node_wfb.json new file mode 100644 index 00000000..eee562b5 --- /dev/null +++ b/tests/rest_wfb_objects/multi_node_wfb.json @@ -0,0 +1,815 @@ +{ + "state": { + "nodes": [ + { + "id": 1, + "x": 167.75, + "y": 619.25, + "name": "Input Data Directory", + "expanded": true, + "pluginId": "core.input-path", + "height": 160, + "width": 250, + "settings": { + "outputs": { + "inputPath": "/viz_workflow_BBBC001__step__1__BbbcDownload/InpDirFileRenaming/BBBC/BBBC001/raw/Images/human_ht29_colon_cancer_1_images" + }, + "inputs": {} + }, + "internal": true + }, + { + "id": 2, + "x": 469.25, + "y": 277.75, + "z": 1, + "name": "OME Converter", + "expanded": true, + "pluginId": "polusai/OMEConverter@0.3.2-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "fileExtension": ".ome.tif", + "filePattern": ".*.tif" + }, + "outputs": { + "outDir": "omeconverter_2-outDir" + } + }, + "internal": false + }, + { + "id": 3, + "x": 104.25, + "y": 233.75, + "z": 4, + "name": "File Renaming", + "expanded": true, + "pluginId": "polusai/FileRenaming@0.2.4-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "mapDirectory": "", + "outFilePattern": "x{row:dd}_y{col:dd}_p{f:dd}_c{channel:d}.tif", + "filePattern": ".*_{row:c}{col:dd}f{f:dd}d{channel:d}.tif", + "inpDir": "/viz_workflow_BBBC001__step__1__BbbcDownload/InpDirFileRenaming/BBBC/BBBC001/raw/Images/human_ht29_colon_cancer_1_images" + }, + "outputs": { + "outDir": "filerenaming_3-outDir" + } + }, + "internal": false + }, + { + "id": 4, + "x": 770.5754637299812, + "y": 514.5603498684344, + "z": 2, + "name": "Montage", + "expanded": true, + "pluginId": "polusai/Montage@0.5.1-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "flipAxis": "", + "gridSpacing": 0, + "imageSpacing": 0, + "layout": "p", + "filePattern": "x00_y03_p{p:dd}_c0.ome.tif" + }, + "outputs": { + "outDir": "montage_4-outDir" + } + }, + "internal": false + }, + { + "id": 5, + "x": 1055.25, + "y": 294.75, + "z": 3, + "name": "Image Assembler", + "expanded": true, + "pluginId": "polusai/ImageAssembler@1.4.1-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": {}, + "outputs": { + "outDir": "imageassembler_5-outDir" + } + }, + "internal": false + }, + { + "id": 6, + "x": 1343.75, + "y": 292, + "z": 5, + "name": "Precompute Slide Viewer", + "expanded": true, + "pluginId": "polusai/PrecomputeSlideViewer@1.7.0-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "filePattern": "", + "pyramidType": "Neuroglancer", + "imageType": "image" + }, + "outputs": { + "outDir": "precomputeslideviewer_6-outDir" + } + }, + "internal": false + } + ], + "links": [ + { + "sourceId": 2, + "outletIndex": 0, + "targetId": 4, + "inletIndex": 0, + "id": 1, + "x1": 699.25, + "y1": 345.25, + "x2": 790.5754637299812, + "y2": 582.0603498684344 + }, + { + "sourceId": 3, + "outletIndex": 0, + "targetId": 2, + "inletIndex": 0, + "id": 2, + "x1": 334.25, + "y1": 301.25, + "x2": 489.25, + "y2": 345.25 + }, + { + "sourceId": 1, + "outletIndex": 0, + "targetId": 3, + "inletIndex": 0, + "id": 3, + "x1": 397.75, + "y1": 686.75, + "x2": 124.25, + "y2": 301.25 + }, + { + "sourceId": 2, + "outletIndex": 0, + "targetId": 5, + "inletIndex": 0, + "id": 4, + "x1": 699.25, + "y1": 345.25, + "x2": 1075.25, + "y2": 362.25 + }, + { + "sourceId": 4, + "outletIndex": 0, + "targetId": 5, + "inletIndex": 1, + "id": 5, + "x1": 1000.5754637299812, + "y1": 582.0603498684344, + "x2": 1075.25, + "y2": 387.25 + }, + { + "sourceId": 5, + "outletIndex": 0, + "targetId": 6, + "inletIndex": 0, + "id": 6, + "x1": 1285.25, + "y1": 362.25, + "x2": 1363.75, + "y2": 359.5 + } + ], + "selection": [] + }, + "plugins": [ + { + "name": "polusai/FileRenaming", + "version": "0.2.4-dev0", + "title": "File Renaming", + "description": "Rename and store image collection files in a new image collection", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Melanie Parham", + "Hamdah Shafqat" + ], + "contact": "melanie.parham@axleinfo.com", + "container": "polusai/file-renaming-tool:0.2.4-dev0", + "entrypoint": "python3 -m polus.images.formats.file_renaming", + "inputs": [ + { + "description": "Filename pattern used to separate data", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Desired filename pattern used to rename and separate data", + "format": [ + "string" + ], + "name": "outFilePattern", + "required": true, + "type": "string" + }, + { + "description": "Get directory name incorporated in renamed files", + "format": [ + "enum" + ], + "name": "mapDirectory", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "collection" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Filename pattern used to separate data", + "key": "inputs.filePattern", + "title": "Filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input collection", + "type": "path" + }, + { + "description": "Desired filename pattern used to rename and separate data", + "key": "inputs.outFilePattern", + "title": "Output filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Get directory name incorporated in renamed files", + "fields": [ + "raw", + "map", + "default" + ], + "key": "inputs.mapDirectory", + "title": "mapDirectory", + "type": "select", + "required": false, + "format": [ + "enum" + ] + } + ], + "path": "formats", + "tags": [ + "file-renaming-tool" + ], + "createdAt": "2024-07-10T17:11:42.680Z", + "updatedAt": "2024-07-10T17:11:42.680Z", + "id": "668ec0ceb57adb6813c44eb4", + "pid": "polusai/FileRenaming@0.2.4-dev0" + }, + { + "name": "polusai/ImageAssembler", + "version": "1.4.1-dev0", + "title": "Image Assembler", + "description": "A scalable image assembling plugin.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Antoine Gerardin" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/image-assembler-tool:1.4.1-dev0", + "entrypoint": "python3 -m polus.images.transforms.images.image_assembler", + "inputs": [ + { + "description": "Stitching vector for data", + "format": [ + "stitchingVector" + ], + "name": "stitchPath", + "required": true, + "type": "path" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "imgPath", + "required": true, + "type": "path" + }, + { + "description": "Label images by timeslice rather than analyzing input image names", + "format": [ + "boolean" + ], + "name": "timesliceNaming", + "required": false, + "type": "boolean" + }, + { + "description": "Generate preview of outputs.", + "format": [ + "boolean" + ], + "name": "preview", + "required": false, + "type": "boolean" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "collection" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/labshare/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.imgPath", + "title": "Input collection", + "type": "path" + }, + { + "description": "Stitching vectors to use", + "key": "inputs.stitchPath", + "title": "Stitching Vector", + "type": "path" + }, + { + "description": "Use stitching vector timeslice number as the image name", + "key": "inputs.timesliceNaming", + "title": "Timeslice numbers for image names:", + "type": "checkbox", + "required": false, + "format": [ + "boolean" + ] + } + ], + "path": "transforms/images", + "tags": [ + "image-assembler-tool" + ], + "createdAt": "2024-07-10T17:11:42.681Z", + "updatedAt": "2024-07-10T17:11:42.681Z", + "id": "668ec0ceb57adb6813c44ec9", + "pid": "polusai/ImageAssembler@1.4.1-dev0" + }, + { + "name": "Input Data Directory", + "title": "Input Data Directory", + "path": "data_source", + "pid": "core.input-path", + "id": "core.input-path", + "internal": true, + "version": "0.0.1", + "description": "Set workflow variable", + "inputs": [], + "outputs": [ + { + "name": "inputPath", + "type": "text" + } + ], + "ui": [ + { + "required": true, + "key": "outputs.inputPath", + "description": "Path", + "title": "Path", + "type": "path" + } + ] + }, + { + "name": "polusai/Montage", + "version": "0.5.1-dev0", + "title": "Montage", + "description": "Advanced montaging plugin.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Benjamin Houghton" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/montage-tool:0.5.1-dev0", + "entrypoint": "python3 -m polus.images.transforms.images.montage", + "inputs": [ + { + "description": "Filename pattern used to parse data", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Specify montage organization", + "format": [ + "array" + ], + "name": "layout", + "required": false, + "type": "array" + }, + { + "description": "Spacing between images at the lowest subgrid", + "format": [ + "integer" + ], + "name": "imageSpacing", + "required": false, + "type": "number" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "integer" + ], + "name": "gridSpacing", + "required": false, + "type": "number" + }, + { + "description": "Axes to flip when creating the montage", + "format": [ + "string" + ], + "name": "flipAxis", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "stitchingVector" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Filename pattern used to parse data", + "key": "inputs.filePattern", + "title": "Filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input collection", + "type": "path" + }, + { + "description": "Specify montage organization", + "key": "inputs.layout", + "title": "Grid layout", + "type": "text", + "required": false, + "format": [ + "array" + ] + }, + { + "description": "Space between images", + "key": "inputs.imageSpacing", + "title": "Image spacing", + "type": "number", + "required": false, + "format": [ + "integer" + ] + }, + { + "description": "Spacing between subgrids", + "key": "inputs.gridSpacing", + "title": "Grid spacing multiplier", + "type": "number", + "required": false, + "format": [ + "integer" + ] + }, + { + "description": "Axes to flip when laying out images.", + "key": "inputs.flipAxis", + "title": "Flip Axis", + "type": "text", + "required": false, + "format": [ + "string" + ] + } + ], + "path": "transforms/images", + "tags": [ + "montage-tool" + ], + "createdAt": "2024-07-10T17:11:42.681Z", + "updatedAt": "2024-07-10T17:11:42.681Z", + "id": "668ec0ceb57adb6813c44ecc", + "pid": "polusai/Montage@0.5.1-dev0" + }, + { + "name": "polusai/OMEConverter", + "version": "0.3.2-dev0", + "title": "OME Converter", + "description": "Convert Bioformats supported format to OME Zarr or OME TIF", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Hamdah Shafqat" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/ome-converter-tool:0.3.2-dev0", + "entrypoint": "python3 -m polus.images.formats.ome_converter", + "inputs": [ + { + "description": "Input generic data collection to be processed by this plugin", + "format": [ + "genericData" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "A filepattern, used to select data to be converted", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Type of data conversion", + "format": [ + "enum" + ], + "name": "fileExtension", + "required": true, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "genericData" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Input generic data collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input generic collection", + "type": "path" + }, + { + "description": "A filepattern, used to select data for conversion", + "key": "inputs.filePattern", + "title": "Filepattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Type of data conversion", + "fields": [ + ".ome.tif", + ".ome.zarr", + "default" + ], + "key": "inputs.fileExtension", + "title": "fileExtension", + "type": "select", + "required": true, + "format": [ + "enum" + ] + } + ], + "path": "formats", + "tags": [ + "ome-converter-tool" + ], + "createdAt": "2024-07-10T17:11:42.680Z", + "updatedAt": "2024-07-10T17:11:42.680Z", + "id": "668ec0ceb57adb6813c44eb6", + "pid": "polusai/OMEConverter@0.3.2-dev0" + }, + { + "name": "polusai/PrecomputeSlideViewer", + "version": "1.7.0-dev0", + "title": "Precompute Slide Viewer", + "description": "Precomputes a plane series in DeepZoom, Neuroglancer, or OME Zarr format.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Madhuri Vihani", + "Nick Schaub", + "Antoine Gerardin", + "Najib Ishaq" + ], + "contact": "Madhuri.Vihani@nih.gov", + "container": "polusai/precompute-slide-plugin:1.7.0-dev0", + "entrypoint": "python3 -m polus.images.visualization.precompute_slide", + "inputs": [ + { + "description": "Input collection", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Build a DeepZoom, Neuroglancer, Zarr pyramid", + "format": [ + "enum" + ], + "name": "pyramidType", + "required": true, + "type": "string" + }, + { + "description": "Image is either Segmentation or Image", + "format": [ + "enum" + ], + "name": "imageType", + "required": false, + "type": "string" + }, + { + "description": "Pattern of the images in Input", + "format": [ + "string" + ], + "name": "filePattern", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Precomputed output", + "format": [ + "pyramid" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/LabShare/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Collection name...", + "key": "inputs.inpDir", + "title": "Input collection: ", + "type": "path" + }, + { + "description": "Build a DeepZoom, Neuroglancer, or Zarr pyramid?", + "fields": [ + "DeepZoom", + "Neuroglancer", + "Zarr" + ], + "key": "inputs.pyramidType", + "title": "Pyramid Type: ", + "type": "select", + "required": true, + "format": [ + "enum" + ] + }, + { + "condition": "inputs.pyramidType=='Neuroglancer'", + "description": "Image or Segmentation?", + "fields": [ + "image", + "segmentation" + ], + "key": "inputs.imageType", + "title": "Image Type: ", + "type": "select", + "required": false, + "format": [ + "enum" + ] + }, + { + "description": "Pattern of images in input collection (image_r{rrr}_c{ccc}_z{zzz}.ome.tif). ", + "key": "inputs.filePattern", + "title": "Image Pattern: ", + "type": "text", + "required": false, + "format": [ + "string" + ] + } + ], + "path": "visualization", + "tags": [ + "precompute-slide-tool" + ], + "createdAt": "2024-07-22T20:04:05.108Z", + "updatedAt": "2024-07-22T20:15:02.835Z", + "id": "669ebb35936ebc63b94e928a", + "pid": "polusai/PrecomputeSlideViewer@1.7.0-dev0" + } + ] +} \ No newline at end of file diff --git a/tests/test_rest_core.py b/tests/test_rest_core.py index 54639d02..59003d8e 100644 --- a/tests/test_rest_core.py +++ b/tests/test_rest_core.py @@ -1,15 +1,120 @@ import json +import copy from pathlib import Path import asyncio +import subprocess as sub +import sys +import traceback +import yaml + from fastapi import Request import pytest -from sophios.wic_types import Json +from sophios.wic_types import Json, List from sophios.api.http import restapi +try: + import cwltool.main + import toil.cwl.cwltoil # transitively imports cwltool +except ImportError as exc: + print('Could not import cwltool.main and/or toil.cwl.cwltoil') + # (pwd is imported transitively in cwltool.provenance) + print(exc) + if exc.msg == "No module named 'pwd'": + print('Windows does not have a pwd module') + print('If you want to run on windows, you need to install') + print('Windows Subsystem for Linux') + print('See https://pypi.org/project/cwltool/#ms-windows-users') + else: + raise exc + + +def run_cwl_local(workflow_name: str, cwl_runner: str, docker_cmd: str, use_subprocess: bool) -> int: + """A helper function to run the compiled cwl output""" + quiet = ['--quiet'] + skip_schemas = ['--skip-schemas'] + provenance = ['--provenance', f'provenance/{workflow_name}'] + docker_cmd_: List[str] = [] + if docker_cmd == 'docker': + docker_cmd_ = [] + elif docker_cmd == 'singularity': + docker_cmd_ = ['--singularity'] + else: + docker_cmd_ = ['--user-space-docker-cmd', docker_cmd] + write_summary = ['--write-summary', f'output_{workflow_name}.json'] + path_check = ['--relax-path-checks'] + # See https://github.com/common-workflow-language/cwltool/blob/5a645dfd4b00e0a704b928cc0bae135b0591cc1a/cwltool/command_line_tool.py#L94 + # NOTE: Using --leave-outputs to disable --outdir + # See https://github.com/dnanexus/dx-cwl/issues/20 + # --outdir has one or more bugs which will cause workflows to fail!!! + docker_pull = ['--disable-pull'] # Use cwl-docker-extract to pull images + script = 'cwltool_filterlog' if cwl_runner == 'cwltool' else cwl_runner + cmd = [script] + docker_pull + quiet + provenance + \ + docker_cmd_ + write_summary + skip_schemas + path_check + if cwl_runner == 'cwltool': + cmd += ['--leave-outputs', + f'autogenerated/{workflow_name}.cwl', f'autogenerated/{workflow_name}_inputs.yml'] + elif cwl_runner == 'toil-cwl-runner': + cmd += ['--outdir', 'outdir_toil', + '--jobStore', f'file:./jobStore_{workflow_name}', # NOTE: This is the equivalent of --cachedir + '--clean', 'always', # This effectively disables caching, but is reproducible + f'autogenerated/{workflow_name}.cwl', f'autogenerated/{workflow_name}_inputs.yml'] + else: + pass + cmdline = ' '.join(cmd) + + retval = 1 # overwrite on success + print('Running ' + cmdline) + if use_subprocess: + # To run in parallel (i.e. pytest ... --workers 8 ...), we need to + # use separate processes. Otherwise: + # "signal only works in main thread or with __pypy__.thread.enable_signals()" + proc = sub.run(cmd, check=False) + retval = proc.returncode + else: + print('via cwltool.main.main python API') + try: + if cwl_runner == 'cwltool': + retval = cwltool.main.main(cmd[1:]) + elif cwl_runner == 'toil-cwl-runner': + retval = toil.cwl.cwltoil.main(cmd[1:]) + else: + raise Exception("Invalid cwl_runner!") + + print(f'Final output json metadata blob is in output_{workflow_name}.json') + except Exception as e: + print('Failed to execute', workflow_name) + print(f'See error_{workflow_name}.txt for detailed technical information.') + # Do not display a nasty stack trace to the user; hide it in a file. + with open(f'error_{workflow_name}.txt', mode='w', encoding='utf-8') as f: + # https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks + if sys.version_info >= (3, 10): + traceback.print_exception(type(e), value=e, tb=None, file=f) + print(e) # we are always running this on CI + return retval + + +def write_out_to_disk(res: Json, workflow_name: str) -> None: + "write compiled output to before running through cwl_runner entrypoints" + res_cwl = copy.deepcopy(res) + res_cwl.pop('retval', None) + res_cwl.pop('cwlJobInputs', None) + res_cwl.pop('name', None) + # Add back the dollar for tags like 'namespaces' and 'schemas' + res_cwl['$namespaces'] = res_cwl.pop('namespaces', None) + res_cwl['$schemas'] = res_cwl.pop('schemas', None) + compiled_cwl = workflow_name + '.cwl' + inputs_yml = workflow_name + '_inputs.yml' + # write compiled .cwl file + with open(Path.cwd() / 'autogenerated' / compiled_cwl, 'w', encoding='utf-8') as f: + yaml.dump(res_cwl, f) + # write _input.yml file + with open(Path.cwd() / 'autogenerated' / inputs_yml, 'w', encoding='utf-8') as f: + yaml.dump(res['cwlJobInputs'], f) + @pytest.mark.fast def test_rest_core_single_node() -> None: @@ -31,11 +136,15 @@ async def receive() -> Json: req: Request = Request(scope) req._receive = receive res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api - assert int(res['retval']) == 0 + workflow_name = inp_file.split('.', maxsplit=1)[0] + # write compiled_cwl and inputs_yml + write_out_to_disk(res, workflow_name) + retval = run_cwl_local(workflow_name, 'cwltool', 'docker', False) + assert retval == 0 @pytest.mark.fast -def test_rest_core_multi_node() -> None: +def test_rest_core_multi_node_file() -> None: """A simple multi node sophios/restapi test""" inp_file = "multi_node.json" inp: Json = {} @@ -54,7 +163,11 @@ async def receive() -> Json: req: Request = Request(scope) req._receive = receive res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api - assert int(res['retval']) == 0 + workflow_name = inp_file.split('.', maxsplit=1)[0] + # write compiled_cwl and inputs_yml + write_out_to_disk(res, workflow_name) + retval = run_cwl_local(workflow_name, 'cwltool', 'docker', False) + assert retval == 0 @pytest.mark.fast @@ -77,4 +190,8 @@ async def receive() -> Json: req: Request = Request(scope) req._receive = receive res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api - assert int(res['retval']) == 0 + workflow_name = inp_file.split('.', maxsplit=1)[0] + # write compiled_cwl and inputs_yml + write_out_to_disk(res, workflow_name) + retval = run_cwl_local(workflow_name, 'cwltool', 'docker', False) + assert retval == 0 diff --git a/tests/test_rest_wfb.py b/tests/test_rest_wfb.py new file mode 100644 index 00000000..9537b920 --- /dev/null +++ b/tests/test_rest_wfb.py @@ -0,0 +1,34 @@ +import copy +import json +from pathlib import Path +import asyncio + +from fastapi import Request + +import pytest +from sophios.wic_types import Json + + +from sophios.api.http import restapi + + +def test_rest_multinode_wfb() -> None: + """A multi node (with plugins) wfb -> sophios/restapi test""" + inp_file = "multi_node_wfb.json" + inp: Json = {} + inp_path = Path(__file__).parent / 'rest_wfb_objects' / inp_file + with open(inp_path, 'r', encoding='utf-8') as f: + inp = json.load(f) + print('----------- from rest api ----------- \n\n') + scope = {} + scope['type'] = 'http' + + async def receive() -> Json: + inp_byte = json.dumps(inp).encode('utf-8') + return {"type": "http.request", "body": inp_byte} + + # create a request object and pack it with our json payload + req: Request = Request(scope) + req._receive = receive + res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api + assert int(res['retval']) == 0 From 11705e183686fccd2ae935ef2156b47edc2c3f32 Mon Sep 17 00:00:00 2001 From: JesseMckinzie <72471813+JesseMckinzie@users.noreply.github.com> Date: Wed, 9 Oct 2024 13:13:32 -0600 Subject: [PATCH 11/15] ICT spec updates and WFB payload fix (#282) * Remove error when looking for output in inputs * Fix bug when parsing outputs * Remove incorrect keys from ui in ICT dict * Add function to add missing inputs to nodes in wfb * Make UI optional in ict spec and remove ui if found * Add test for updating nodes in wfb payload * Update ui validation * Update wfb type to Json * Update wfb nodes before raw to lean * Remove update to ui before clt conversion * Move is_inlet inside of wfb fixing function * Check that plugins are in wfb data * Move schema validation to payload update function * Check if plugins are present in WFB data --- .github/workflows/run_workflows.yml | 5 + src/sophios/api/http/restapi.py | 1 + src/sophios/api/utils/converter.py | 74 +- src/sophios/api/utils/ict/ict_spec/cast.py | 5 + .../api/utils/ict/ict_spec/io/objects.py | 1 - src/sophios/api/utils/ict/ict_spec/model.py | 38 +- .../api/utils/ict/ict_spec/tools/cwl_ict.py | 2 +- .../wfb_data/multi_node/multi_node_wfb.json | 815 +++++++++++++++++ .../multi_node/multi_node_wfb_truth.json | 821 ++++++++++++++++++ tests/test_fix_payload.py | 25 + 10 files changed, 1757 insertions(+), 30 deletions(-) create mode 100644 tests/data/wfb_data/multi_node/multi_node_wfb.json create mode 100644 tests/data/wfb_data/multi_node/multi_node_wfb_truth.json create mode 100644 tests/test_fix_payload.py diff --git a/.github/workflows/run_workflows.yml b/.github/workflows/run_workflows.yml index 4bf1d0ec..6ffc87f2 100644 --- a/.github/workflows/run_workflows.yml +++ b/.github/workflows/run_workflows.yml @@ -191,6 +191,11 @@ jobs: # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 run: cd workflow-inference-compiler/ && pytest tests/test_ict_to_clt_conversion.py -k test_ict_to_clt + - name: PyTest Run update wfb payload Tests + if: always() + # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 + run: cd workflow-inference-compiler/ && pytest tests/test_fix_payload.py -k test_fix + # NOTE: The steps below are for repository_dispatch only. For all other steps, please insert above # this comment. diff --git a/src/sophios/api/http/restapi.py b/src/sophios/api/http/restapi.py index 01c337e0..ef66e4dc 100644 --- a/src/sophios/api/http/restapi.py +++ b/src/sophios/api/http/restapi.py @@ -95,6 +95,7 @@ async def compile_wf(request: Request) -> Json: req: Json = await request.json() # clean up and convert the incoming object # schema preserving + req = converter.update_payload_missing_inputs_outputs(req) wfb_payload = converter.raw_wfb_to_lean_wfb(req) # schema non-preserving workflow_temp = converter.wfb_to_wic(wfb_payload) diff --git a/src/sophios/api/utils/converter.py b/src/sophios/api/utils/converter.py index c9df678b..ba00ac99 100644 --- a/src/sophios/api/utils/converter.py +++ b/src/sophios/api/utils/converter.py @@ -55,14 +55,7 @@ def extract_state(inp: Json) -> Json: inp_inter['state']['links'] = step_edges # massage the plugins plugins = inp_inter['plugins'] - # drop incorrect/superfluous UI fields from plugins - # 'required' and 'format' - for ict_plugin in plugins: - for ui_elem in ict_plugin['ui']: - _, _ = ui_elem.pop('required', None), ui_elem.pop('format', None) - for out in ict_plugin['outputs']: - if out['name'] == 'outDir': - ict_plugin['inputs'].append(out) + # Here goes the ICT to CLT extraction logic for node in inp_inter['state']['nodes']: node_pid = node["pluginId"] @@ -78,8 +71,6 @@ def extract_state(inp: Json) -> Json: def raw_wfb_to_lean_wfb(inp: Json) -> Json: """Drop all the unnecessary info from incoming wfb object""" - if validate_schema_and_object(SCHEMA, inp): - print('incoming object is valid against input object schema') inp_restrict = extract_state(inp) keys = list(inp_restrict.keys()) # To avoid deserialization @@ -193,3 +184,66 @@ def ict_to_clt(ict: Union[ICT, Path, str, dict], network_access: bool = False) - ict_local = ict if isinstance(ict, ICT) else cast_to_ict(ict) return ict_local.to_clt(network_access=network_access) + + +def update_payload_missing_inputs_outputs(wfb_data: Json) -> Json: + """Update payload with missing inputs and outputs using links""" + + # ensure the incoming wfb data is valid + if validate_schema_and_object(SCHEMA, wfb_data): + print('incoming object is valid against input object schema') + + # return if no plugins are found in data + if not wfb_data['plugins']: + return wfb_data + + wfb_data_copy = copy.deepcopy(wfb_data) + + links = wfb_data_copy["state"]["links"] + nodes = wfb_data_copy["state"]["nodes"] + plugins = wfb_data_copy["plugins"] + + # hashmap of node id to nodes for fast node lookup + nodes_dict = {node['id']: node for node in nodes} + + # hashmap of plugins id to nodes for fast plugin lookup + plugins_dict = {plugin['pid']: plugin for plugin in plugins} + + # find links corresponding to the node + for link in links: + + # link ids + target_id: int = link["targetId"] + source_id: int = link["sourceId"] + + target_node = nodes_dict[target_id] + source_node = nodes_dict[source_id] + + # plugins corresponding to the nodes + target_plugin = plugins_dict[target_node["pluginId"]] + source_plugin = plugins_dict[source_node["pluginId"]] + + def is_inlet(binding: Json) -> bool: + """Check if a wfb input is an inlet (directory)""" + + return ( + binding['type'] in ['directory', 'file', 'path', 'collection', 'csvCollection'] or + binding['name'].lower() == 'inpdir' or + binding['name'].lower().endswith('path') or + binding['name'].lower().endswith('dir') + ) + + # filter inputs by to only be inlets (directories) + input_directories = [binding for binding in target_plugin["inputs"] if is_inlet(binding)] + output_directories = [binding for binding in source_plugin["outputs"] if is_inlet(binding)] + + missing_input_key = input_directories[link["inletIndex"]]["name"] + missing_output_key = output_directories[link["outletIndex"]]["name"] + + # add the missing input value to the node if needed + target_node["settings"]["inputs"][missing_input_key] = source_node["settings"]["outputs"][missing_output_key] + + if validate_schema_and_object(SCHEMA, wfb_data_copy): + print('Updated object is valid against input object schema') + + return wfb_data_copy diff --git a/src/sophios/api/utils/ict/ict_spec/cast.py b/src/sophios/api/utils/ict/ict_spec/cast.py index ea84394c..89b2f687 100644 --- a/src/sophios/api/utils/ict/ict_spec/cast.py +++ b/src/sophios/api/utils/ict/ict_spec/cast.py @@ -13,6 +13,7 @@ def cast_to_ict(ict: Union[Path, str, dict]) -> ICT: ict = Path(ict) if isinstance(ict, Path): + if str(ict).endswith(".yaml") or str(ict).endswith(".yml"): with open(ict, "r", encoding="utf-8") as f_o: data = safe_load(f_o) @@ -22,6 +23,10 @@ def cast_to_ict(ict: Union[Path, str, dict]) -> ICT: else: raise ValueError(f"File extension not supported: {ict}") + data.pop("ui", None) + return ICT(**data) + ict.pop("ui", None) + return ICT(**ict) diff --git a/src/sophios/api/utils/ict/ict_spec/io/objects.py b/src/sophios/api/utils/ict/ict_spec/io/objects.py index 3cbee409..4f4f92d8 100644 --- a/src/sophios/api/utils/ict/ict_spec/io/objects.py +++ b/src/sophios/api/utils/ict/ict_spec/io/objects.py @@ -144,5 +144,4 @@ def _output_to_cwl(self, inputs: Any) -> dict: cwl_dict_["format"] = self.convert_uri_format(self.io_format["uri"]) return cwl_dict_ - raise ValueError(f"Output {self.name} not found in inputs") raise NotImplementedError(f"Output not supported {self.name}") diff --git a/src/sophios/api/utils/ict/ict_spec/model.py b/src/sophios/api/utils/ict/ict_spec/model.py index c542e2d0..7c38e9cd 100644 --- a/src/sophios/api/utils/ict/ict_spec/model.py +++ b/src/sophios/api/utils/ict/ict_spec/model.py @@ -24,29 +24,31 @@ class ICT(Metadata): inputs: list[IO] outputs: list[IO] - ui: list[UIItem] + ui: Optional[list[UIItem]] = None hardware: Optional[HardwareRequirements] = None @model_validator(mode="after") def validate_ui(self) -> "ICT": """Validate that the ui matches the inputs and outputs.""" - io_dict = {"inputs": [], "outputs": []} # type: ignore - ui_keys = [ui.key.root.split(".") for ui in self.ui] - for ui_ in ui_keys: - io_dict[ui_[0]].append(ui_[1]) - input_names = [io.name for io in self.inputs] - output_names = [io.name for io in self.outputs] - inp_bool = [x in input_names for x in io_dict["inputs"]] - out_bool = [x in output_names for x in io_dict["outputs"]] - - # if not all(inp_bool): - # raise ValueError( - # f"The ui keys must match the inputs and outputs keys. Unmatched: inputs.{set(io_dict['inputs'])-set(input_names)}" - # ) - # if not all(out_bool): - # raise ValueError( - # f"The ui keys must match the inputs and outputs keys. Unmatched: outputs.{set(io_dict['outputs'])-set(output_names)}" - # ) + if self.ui is not None: + io_dict = {"inputs": [], "outputs": []} # type: ignore + ui_keys = [ui.key.root.split(".") for ui in self.ui] + for ui_ in ui_keys: + io_dict[ui_[0]].append(ui_[1]) + input_names = [io.name for io in self.inputs] + output_names = [io.name for io in self.outputs] + inp_bool = [x in input_names for x in io_dict["inputs"]] + out_bool = [x in output_names for x in io_dict["outputs"]] + + # if not all(inp_bool): + # raise ValueError( + # f"The ui keys must match the inputs and outputs keys. Unmatched: inputs.{set(io_dict['inputs'])-set(input_names)}" + # ) + # if not all(out_bool): + # raise ValueError( + # f"The ui keys must match the inputs and outputs keys. Unmatched: outputs.{set(io_dict['outputs'])-set(output_names)}" + # ) + return self def to_clt(self, network_access: bool = False) -> dict: diff --git a/src/sophios/api/utils/ict/ict_spec/tools/cwl_ict.py b/src/sophios/api/utils/ict/ict_spec/tools/cwl_ict.py index 1d87fcdf..577b1c9b 100644 --- a/src/sophios/api/utils/ict/ict_spec/tools/cwl_ict.py +++ b/src/sophios/api/utils/ict/ict_spec/tools/cwl_ict.py @@ -32,7 +32,7 @@ def clt_dict(ict_: "ICT", network_access: bool) -> dict: }, "outputs": { io.name: io._output_to_cwl( - [io.name for io in ict_.inputs] + [io.name for io in ict_.outputs] ) # pylint: disable=W0212 for io in ict_.outputs }, diff --git a/tests/data/wfb_data/multi_node/multi_node_wfb.json b/tests/data/wfb_data/multi_node/multi_node_wfb.json new file mode 100644 index 00000000..eee562b5 --- /dev/null +++ b/tests/data/wfb_data/multi_node/multi_node_wfb.json @@ -0,0 +1,815 @@ +{ + "state": { + "nodes": [ + { + "id": 1, + "x": 167.75, + "y": 619.25, + "name": "Input Data Directory", + "expanded": true, + "pluginId": "core.input-path", + "height": 160, + "width": 250, + "settings": { + "outputs": { + "inputPath": "/viz_workflow_BBBC001__step__1__BbbcDownload/InpDirFileRenaming/BBBC/BBBC001/raw/Images/human_ht29_colon_cancer_1_images" + }, + "inputs": {} + }, + "internal": true + }, + { + "id": 2, + "x": 469.25, + "y": 277.75, + "z": 1, + "name": "OME Converter", + "expanded": true, + "pluginId": "polusai/OMEConverter@0.3.2-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "fileExtension": ".ome.tif", + "filePattern": ".*.tif" + }, + "outputs": { + "outDir": "omeconverter_2-outDir" + } + }, + "internal": false + }, + { + "id": 3, + "x": 104.25, + "y": 233.75, + "z": 4, + "name": "File Renaming", + "expanded": true, + "pluginId": "polusai/FileRenaming@0.2.4-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "mapDirectory": "", + "outFilePattern": "x{row:dd}_y{col:dd}_p{f:dd}_c{channel:d}.tif", + "filePattern": ".*_{row:c}{col:dd}f{f:dd}d{channel:d}.tif", + "inpDir": "/viz_workflow_BBBC001__step__1__BbbcDownload/InpDirFileRenaming/BBBC/BBBC001/raw/Images/human_ht29_colon_cancer_1_images" + }, + "outputs": { + "outDir": "filerenaming_3-outDir" + } + }, + "internal": false + }, + { + "id": 4, + "x": 770.5754637299812, + "y": 514.5603498684344, + "z": 2, + "name": "Montage", + "expanded": true, + "pluginId": "polusai/Montage@0.5.1-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "flipAxis": "", + "gridSpacing": 0, + "imageSpacing": 0, + "layout": "p", + "filePattern": "x00_y03_p{p:dd}_c0.ome.tif" + }, + "outputs": { + "outDir": "montage_4-outDir" + } + }, + "internal": false + }, + { + "id": 5, + "x": 1055.25, + "y": 294.75, + "z": 3, + "name": "Image Assembler", + "expanded": true, + "pluginId": "polusai/ImageAssembler@1.4.1-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": {}, + "outputs": { + "outDir": "imageassembler_5-outDir" + } + }, + "internal": false + }, + { + "id": 6, + "x": 1343.75, + "y": 292, + "z": 5, + "name": "Precompute Slide Viewer", + "expanded": true, + "pluginId": "polusai/PrecomputeSlideViewer@1.7.0-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "filePattern": "", + "pyramidType": "Neuroglancer", + "imageType": "image" + }, + "outputs": { + "outDir": "precomputeslideviewer_6-outDir" + } + }, + "internal": false + } + ], + "links": [ + { + "sourceId": 2, + "outletIndex": 0, + "targetId": 4, + "inletIndex": 0, + "id": 1, + "x1": 699.25, + "y1": 345.25, + "x2": 790.5754637299812, + "y2": 582.0603498684344 + }, + { + "sourceId": 3, + "outletIndex": 0, + "targetId": 2, + "inletIndex": 0, + "id": 2, + "x1": 334.25, + "y1": 301.25, + "x2": 489.25, + "y2": 345.25 + }, + { + "sourceId": 1, + "outletIndex": 0, + "targetId": 3, + "inletIndex": 0, + "id": 3, + "x1": 397.75, + "y1": 686.75, + "x2": 124.25, + "y2": 301.25 + }, + { + "sourceId": 2, + "outletIndex": 0, + "targetId": 5, + "inletIndex": 0, + "id": 4, + "x1": 699.25, + "y1": 345.25, + "x2": 1075.25, + "y2": 362.25 + }, + { + "sourceId": 4, + "outletIndex": 0, + "targetId": 5, + "inletIndex": 1, + "id": 5, + "x1": 1000.5754637299812, + "y1": 582.0603498684344, + "x2": 1075.25, + "y2": 387.25 + }, + { + "sourceId": 5, + "outletIndex": 0, + "targetId": 6, + "inletIndex": 0, + "id": 6, + "x1": 1285.25, + "y1": 362.25, + "x2": 1363.75, + "y2": 359.5 + } + ], + "selection": [] + }, + "plugins": [ + { + "name": "polusai/FileRenaming", + "version": "0.2.4-dev0", + "title": "File Renaming", + "description": "Rename and store image collection files in a new image collection", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Melanie Parham", + "Hamdah Shafqat" + ], + "contact": "melanie.parham@axleinfo.com", + "container": "polusai/file-renaming-tool:0.2.4-dev0", + "entrypoint": "python3 -m polus.images.formats.file_renaming", + "inputs": [ + { + "description": "Filename pattern used to separate data", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Desired filename pattern used to rename and separate data", + "format": [ + "string" + ], + "name": "outFilePattern", + "required": true, + "type": "string" + }, + { + "description": "Get directory name incorporated in renamed files", + "format": [ + "enum" + ], + "name": "mapDirectory", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "collection" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Filename pattern used to separate data", + "key": "inputs.filePattern", + "title": "Filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input collection", + "type": "path" + }, + { + "description": "Desired filename pattern used to rename and separate data", + "key": "inputs.outFilePattern", + "title": "Output filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Get directory name incorporated in renamed files", + "fields": [ + "raw", + "map", + "default" + ], + "key": "inputs.mapDirectory", + "title": "mapDirectory", + "type": "select", + "required": false, + "format": [ + "enum" + ] + } + ], + "path": "formats", + "tags": [ + "file-renaming-tool" + ], + "createdAt": "2024-07-10T17:11:42.680Z", + "updatedAt": "2024-07-10T17:11:42.680Z", + "id": "668ec0ceb57adb6813c44eb4", + "pid": "polusai/FileRenaming@0.2.4-dev0" + }, + { + "name": "polusai/ImageAssembler", + "version": "1.4.1-dev0", + "title": "Image Assembler", + "description": "A scalable image assembling plugin.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Antoine Gerardin" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/image-assembler-tool:1.4.1-dev0", + "entrypoint": "python3 -m polus.images.transforms.images.image_assembler", + "inputs": [ + { + "description": "Stitching vector for data", + "format": [ + "stitchingVector" + ], + "name": "stitchPath", + "required": true, + "type": "path" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "imgPath", + "required": true, + "type": "path" + }, + { + "description": "Label images by timeslice rather than analyzing input image names", + "format": [ + "boolean" + ], + "name": "timesliceNaming", + "required": false, + "type": "boolean" + }, + { + "description": "Generate preview of outputs.", + "format": [ + "boolean" + ], + "name": "preview", + "required": false, + "type": "boolean" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "collection" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/labshare/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.imgPath", + "title": "Input collection", + "type": "path" + }, + { + "description": "Stitching vectors to use", + "key": "inputs.stitchPath", + "title": "Stitching Vector", + "type": "path" + }, + { + "description": "Use stitching vector timeslice number as the image name", + "key": "inputs.timesliceNaming", + "title": "Timeslice numbers for image names:", + "type": "checkbox", + "required": false, + "format": [ + "boolean" + ] + } + ], + "path": "transforms/images", + "tags": [ + "image-assembler-tool" + ], + "createdAt": "2024-07-10T17:11:42.681Z", + "updatedAt": "2024-07-10T17:11:42.681Z", + "id": "668ec0ceb57adb6813c44ec9", + "pid": "polusai/ImageAssembler@1.4.1-dev0" + }, + { + "name": "Input Data Directory", + "title": "Input Data Directory", + "path": "data_source", + "pid": "core.input-path", + "id": "core.input-path", + "internal": true, + "version": "0.0.1", + "description": "Set workflow variable", + "inputs": [], + "outputs": [ + { + "name": "inputPath", + "type": "text" + } + ], + "ui": [ + { + "required": true, + "key": "outputs.inputPath", + "description": "Path", + "title": "Path", + "type": "path" + } + ] + }, + { + "name": "polusai/Montage", + "version": "0.5.1-dev0", + "title": "Montage", + "description": "Advanced montaging plugin.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Benjamin Houghton" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/montage-tool:0.5.1-dev0", + "entrypoint": "python3 -m polus.images.transforms.images.montage", + "inputs": [ + { + "description": "Filename pattern used to parse data", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Specify montage organization", + "format": [ + "array" + ], + "name": "layout", + "required": false, + "type": "array" + }, + { + "description": "Spacing between images at the lowest subgrid", + "format": [ + "integer" + ], + "name": "imageSpacing", + "required": false, + "type": "number" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "integer" + ], + "name": "gridSpacing", + "required": false, + "type": "number" + }, + { + "description": "Axes to flip when creating the montage", + "format": [ + "string" + ], + "name": "flipAxis", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "stitchingVector" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Filename pattern used to parse data", + "key": "inputs.filePattern", + "title": "Filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input collection", + "type": "path" + }, + { + "description": "Specify montage organization", + "key": "inputs.layout", + "title": "Grid layout", + "type": "text", + "required": false, + "format": [ + "array" + ] + }, + { + "description": "Space between images", + "key": "inputs.imageSpacing", + "title": "Image spacing", + "type": "number", + "required": false, + "format": [ + "integer" + ] + }, + { + "description": "Spacing between subgrids", + "key": "inputs.gridSpacing", + "title": "Grid spacing multiplier", + "type": "number", + "required": false, + "format": [ + "integer" + ] + }, + { + "description": "Axes to flip when laying out images.", + "key": "inputs.flipAxis", + "title": "Flip Axis", + "type": "text", + "required": false, + "format": [ + "string" + ] + } + ], + "path": "transforms/images", + "tags": [ + "montage-tool" + ], + "createdAt": "2024-07-10T17:11:42.681Z", + "updatedAt": "2024-07-10T17:11:42.681Z", + "id": "668ec0ceb57adb6813c44ecc", + "pid": "polusai/Montage@0.5.1-dev0" + }, + { + "name": "polusai/OMEConverter", + "version": "0.3.2-dev0", + "title": "OME Converter", + "description": "Convert Bioformats supported format to OME Zarr or OME TIF", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Hamdah Shafqat" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/ome-converter-tool:0.3.2-dev0", + "entrypoint": "python3 -m polus.images.formats.ome_converter", + "inputs": [ + { + "description": "Input generic data collection to be processed by this plugin", + "format": [ + "genericData" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "A filepattern, used to select data to be converted", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Type of data conversion", + "format": [ + "enum" + ], + "name": "fileExtension", + "required": true, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "genericData" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Input generic data collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input generic collection", + "type": "path" + }, + { + "description": "A filepattern, used to select data for conversion", + "key": "inputs.filePattern", + "title": "Filepattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Type of data conversion", + "fields": [ + ".ome.tif", + ".ome.zarr", + "default" + ], + "key": "inputs.fileExtension", + "title": "fileExtension", + "type": "select", + "required": true, + "format": [ + "enum" + ] + } + ], + "path": "formats", + "tags": [ + "ome-converter-tool" + ], + "createdAt": "2024-07-10T17:11:42.680Z", + "updatedAt": "2024-07-10T17:11:42.680Z", + "id": "668ec0ceb57adb6813c44eb6", + "pid": "polusai/OMEConverter@0.3.2-dev0" + }, + { + "name": "polusai/PrecomputeSlideViewer", + "version": "1.7.0-dev0", + "title": "Precompute Slide Viewer", + "description": "Precomputes a plane series in DeepZoom, Neuroglancer, or OME Zarr format.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Madhuri Vihani", + "Nick Schaub", + "Antoine Gerardin", + "Najib Ishaq" + ], + "contact": "Madhuri.Vihani@nih.gov", + "container": "polusai/precompute-slide-plugin:1.7.0-dev0", + "entrypoint": "python3 -m polus.images.visualization.precompute_slide", + "inputs": [ + { + "description": "Input collection", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Build a DeepZoom, Neuroglancer, Zarr pyramid", + "format": [ + "enum" + ], + "name": "pyramidType", + "required": true, + "type": "string" + }, + { + "description": "Image is either Segmentation or Image", + "format": [ + "enum" + ], + "name": "imageType", + "required": false, + "type": "string" + }, + { + "description": "Pattern of the images in Input", + "format": [ + "string" + ], + "name": "filePattern", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Precomputed output", + "format": [ + "pyramid" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/LabShare/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Collection name...", + "key": "inputs.inpDir", + "title": "Input collection: ", + "type": "path" + }, + { + "description": "Build a DeepZoom, Neuroglancer, or Zarr pyramid?", + "fields": [ + "DeepZoom", + "Neuroglancer", + "Zarr" + ], + "key": "inputs.pyramidType", + "title": "Pyramid Type: ", + "type": "select", + "required": true, + "format": [ + "enum" + ] + }, + { + "condition": "inputs.pyramidType=='Neuroglancer'", + "description": "Image or Segmentation?", + "fields": [ + "image", + "segmentation" + ], + "key": "inputs.imageType", + "title": "Image Type: ", + "type": "select", + "required": false, + "format": [ + "enum" + ] + }, + { + "description": "Pattern of images in input collection (image_r{rrr}_c{ccc}_z{zzz}.ome.tif). ", + "key": "inputs.filePattern", + "title": "Image Pattern: ", + "type": "text", + "required": false, + "format": [ + "string" + ] + } + ], + "path": "visualization", + "tags": [ + "precompute-slide-tool" + ], + "createdAt": "2024-07-22T20:04:05.108Z", + "updatedAt": "2024-07-22T20:15:02.835Z", + "id": "669ebb35936ebc63b94e928a", + "pid": "polusai/PrecomputeSlideViewer@1.7.0-dev0" + } + ] +} \ No newline at end of file diff --git a/tests/data/wfb_data/multi_node/multi_node_wfb_truth.json b/tests/data/wfb_data/multi_node/multi_node_wfb_truth.json new file mode 100644 index 00000000..3c0d5ace --- /dev/null +++ b/tests/data/wfb_data/multi_node/multi_node_wfb_truth.json @@ -0,0 +1,821 @@ +{ + "state": { + "nodes": [ + { + "id": 1, + "x": 167.75, + "y": 619.25, + "name": "Input Data Directory", + "expanded": true, + "pluginId": "core.input-path", + "height": 160, + "width": 250, + "settings": { + "outputs": { + "inputPath": "/viz_workflow_BBBC001__step__1__BbbcDownload/InpDirFileRenaming/BBBC/BBBC001/raw/Images/human_ht29_colon_cancer_1_images" + }, + "inputs": {} + }, + "internal": true + }, + { + "id": 2, + "x": 469.25, + "y": 277.75, + "z": 1, + "name": "OME Converter", + "expanded": true, + "pluginId": "polusai/OMEConverter@0.3.2-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "fileExtension": ".ome.tif", + "filePattern": ".*.tif", + "inpDir": "filerenaming_3-outDir" + }, + "outputs": { + "outDir": "omeconverter_2-outDir" + } + }, + "internal": false + }, + { + "id": 3, + "x": 104.25, + "y": 233.75, + "z": 4, + "name": "File Renaming", + "expanded": true, + "pluginId": "polusai/FileRenaming@0.2.4-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "mapDirectory": "", + "outFilePattern": "x{row:dd}_y{col:dd}_p{f:dd}_c{channel:d}.tif", + "filePattern": ".*_{row:c}{col:dd}f{f:dd}d{channel:d}.tif", + "inpDir": "/viz_workflow_BBBC001__step__1__BbbcDownload/InpDirFileRenaming/BBBC/BBBC001/raw/Images/human_ht29_colon_cancer_1_images" + }, + "outputs": { + "outDir": "filerenaming_3-outDir" + } + }, + "internal": false + }, + { + "id": 4, + "x": 770.5754637299812, + "y": 514.5603498684344, + "z": 2, + "name": "Montage", + "expanded": true, + "pluginId": "polusai/Montage@0.5.1-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "flipAxis": "", + "gridSpacing": 0, + "imageSpacing": 0, + "layout": "p", + "filePattern": "x00_y03_p{p:dd}_c0.ome.tif", + "inpDir": "omeconverter_2-outDir" + }, + "outputs": { + "outDir": "montage_4-outDir" + } + }, + "internal": false + }, + { + "id": 5, + "x": 1055.25, + "y": 294.75, + "z": 3, + "name": "Image Assembler", + "expanded": true, + "pluginId": "polusai/ImageAssembler@1.4.1-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "stitchPath": "omeconverter_2-outDir", + "imgPath": "montage_4-outDir" + }, + "outputs": { + "outDir": "imageassembler_5-outDir" + } + }, + "internal": false + }, + { + "id": 6, + "x": 1343.75, + "y": 292, + "z": 5, + "name": "Precompute Slide Viewer", + "expanded": true, + "pluginId": "polusai/PrecomputeSlideViewer@1.7.0-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "filePattern": "", + "pyramidType": "Neuroglancer", + "imageType": "image", + "inpDir": "imageassembler_5-outDir" + }, + "outputs": { + "outDir": "precomputeslideviewer_6-outDir" + } + }, + "internal": false + } + ], + "links": [ + { + "sourceId": 2, + "outletIndex": 0, + "targetId": 4, + "inletIndex": 0, + "id": 1, + "x1": 699.25, + "y1": 345.25, + "x2": 790.5754637299812, + "y2": 582.0603498684344 + }, + { + "sourceId": 3, + "outletIndex": 0, + "targetId": 2, + "inletIndex": 0, + "id": 2, + "x1": 334.25, + "y1": 301.25, + "x2": 489.25, + "y2": 345.25 + }, + { + "sourceId": 1, + "outletIndex": 0, + "targetId": 3, + "inletIndex": 0, + "id": 3, + "x1": 397.75, + "y1": 686.75, + "x2": 124.25, + "y2": 301.25 + }, + { + "sourceId": 2, + "outletIndex": 0, + "targetId": 5, + "inletIndex": 0, + "id": 4, + "x1": 699.25, + "y1": 345.25, + "x2": 1075.25, + "y2": 362.25 + }, + { + "sourceId": 4, + "outletIndex": 0, + "targetId": 5, + "inletIndex": 1, + "id": 5, + "x1": 1000.5754637299812, + "y1": 582.0603498684344, + "x2": 1075.25, + "y2": 387.25 + }, + { + "sourceId": 5, + "outletIndex": 0, + "targetId": 6, + "inletIndex": 0, + "id": 6, + "x1": 1285.25, + "y1": 362.25, + "x2": 1363.75, + "y2": 359.5 + } + ], + "selection": [] + }, + "plugins": [ + { + "name": "polusai/FileRenaming", + "version": "0.2.4-dev0", + "title": "File Renaming", + "description": "Rename and store image collection files in a new image collection", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Melanie Parham", + "Hamdah Shafqat" + ], + "contact": "melanie.parham@axleinfo.com", + "container": "polusai/file-renaming-tool:0.2.4-dev0", + "entrypoint": "python3 -m polus.images.formats.file_renaming", + "inputs": [ + { + "description": "Filename pattern used to separate data", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Desired filename pattern used to rename and separate data", + "format": [ + "string" + ], + "name": "outFilePattern", + "required": true, + "type": "string" + }, + { + "description": "Get directory name incorporated in renamed files", + "format": [ + "enum" + ], + "name": "mapDirectory", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "collection" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Filename pattern used to separate data", + "key": "inputs.filePattern", + "title": "Filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input collection", + "type": "path" + }, + { + "description": "Desired filename pattern used to rename and separate data", + "key": "inputs.outFilePattern", + "title": "Output filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Get directory name incorporated in renamed files", + "fields": [ + "raw", + "map", + "default" + ], + "key": "inputs.mapDirectory", + "title": "mapDirectory", + "type": "select", + "required": false, + "format": [ + "enum" + ] + } + ], + "path": "formats", + "tags": [ + "file-renaming-tool" + ], + "createdAt": "2024-07-10T17:11:42.680Z", + "updatedAt": "2024-07-10T17:11:42.680Z", + "id": "668ec0ceb57adb6813c44eb4", + "pid": "polusai/FileRenaming@0.2.4-dev0" + }, + { + "name": "polusai/ImageAssembler", + "version": "1.4.1-dev0", + "title": "Image Assembler", + "description": "A scalable image assembling plugin.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Antoine Gerardin" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/image-assembler-tool:1.4.1-dev0", + "entrypoint": "python3 -m polus.images.transforms.images.image_assembler", + "inputs": [ + { + "description": "Stitching vector for data", + "format": [ + "stitchingVector" + ], + "name": "stitchPath", + "required": true, + "type": "path" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "imgPath", + "required": true, + "type": "path" + }, + { + "description": "Label images by timeslice rather than analyzing input image names", + "format": [ + "boolean" + ], + "name": "timesliceNaming", + "required": false, + "type": "boolean" + }, + { + "description": "Generate preview of outputs.", + "format": [ + "boolean" + ], + "name": "preview", + "required": false, + "type": "boolean" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "collection" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/labshare/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.imgPath", + "title": "Input collection", + "type": "path" + }, + { + "description": "Stitching vectors to use", + "key": "inputs.stitchPath", + "title": "Stitching Vector", + "type": "path" + }, + { + "description": "Use stitching vector timeslice number as the image name", + "key": "inputs.timesliceNaming", + "title": "Timeslice numbers for image names:", + "type": "checkbox", + "required": false, + "format": [ + "boolean" + ] + } + ], + "path": "transforms/images", + "tags": [ + "image-assembler-tool" + ], + "createdAt": "2024-07-10T17:11:42.681Z", + "updatedAt": "2024-07-10T17:11:42.681Z", + "id": "668ec0ceb57adb6813c44ec9", + "pid": "polusai/ImageAssembler@1.4.1-dev0" + }, + { + "name": "Input Data Directory", + "title": "Input Data Directory", + "path": "data_source", + "pid": "core.input-path", + "id": "core.input-path", + "internal": true, + "version": "0.0.1", + "description": "Set workflow variable", + "inputs": [], + "outputs": [ + { + "name": "inputPath", + "type": "text" + } + ], + "ui": [ + { + "required": true, + "key": "outputs.inputPath", + "description": "Path", + "title": "Path", + "type": "path" + } + ] + }, + { + "name": "polusai/Montage", + "version": "0.5.1-dev0", + "title": "Montage", + "description": "Advanced montaging plugin.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Benjamin Houghton" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/montage-tool:0.5.1-dev0", + "entrypoint": "python3 -m polus.images.transforms.images.montage", + "inputs": [ + { + "description": "Filename pattern used to parse data", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Specify montage organization", + "format": [ + "array" + ], + "name": "layout", + "required": false, + "type": "array" + }, + { + "description": "Spacing between images at the lowest subgrid", + "format": [ + "integer" + ], + "name": "imageSpacing", + "required": false, + "type": "number" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "integer" + ], + "name": "gridSpacing", + "required": false, + "type": "number" + }, + { + "description": "Axes to flip when creating the montage", + "format": [ + "string" + ], + "name": "flipAxis", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "stitchingVector" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Filename pattern used to parse data", + "key": "inputs.filePattern", + "title": "Filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input collection", + "type": "path" + }, + { + "description": "Specify montage organization", + "key": "inputs.layout", + "title": "Grid layout", + "type": "text", + "required": false, + "format": [ + "array" + ] + }, + { + "description": "Space between images", + "key": "inputs.imageSpacing", + "title": "Image spacing", + "type": "number", + "required": false, + "format": [ + "integer" + ] + }, + { + "description": "Spacing between subgrids", + "key": "inputs.gridSpacing", + "title": "Grid spacing multiplier", + "type": "number", + "required": false, + "format": [ + "integer" + ] + }, + { + "description": "Axes to flip when laying out images.", + "key": "inputs.flipAxis", + "title": "Flip Axis", + "type": "text", + "required": false, + "format": [ + "string" + ] + } + ], + "path": "transforms/images", + "tags": [ + "montage-tool" + ], + "createdAt": "2024-07-10T17:11:42.681Z", + "updatedAt": "2024-07-10T17:11:42.681Z", + "id": "668ec0ceb57adb6813c44ecc", + "pid": "polusai/Montage@0.5.1-dev0" + }, + { + "name": "polusai/OMEConverter", + "version": "0.3.2-dev0", + "title": "OME Converter", + "description": "Convert Bioformats supported format to OME Zarr or OME TIF", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Hamdah Shafqat" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/ome-converter-tool:0.3.2-dev0", + "entrypoint": "python3 -m polus.images.formats.ome_converter", + "inputs": [ + { + "description": "Input generic data collection to be processed by this plugin", + "format": [ + "genericData" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "A filepattern, used to select data to be converted", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Type of data conversion", + "format": [ + "enum" + ], + "name": "fileExtension", + "required": true, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "genericData" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Input generic data collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input generic collection", + "type": "path" + }, + { + "description": "A filepattern, used to select data for conversion", + "key": "inputs.filePattern", + "title": "Filepattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Type of data conversion", + "fields": [ + ".ome.tif", + ".ome.zarr", + "default" + ], + "key": "inputs.fileExtension", + "title": "fileExtension", + "type": "select", + "required": true, + "format": [ + "enum" + ] + } + ], + "path": "formats", + "tags": [ + "ome-converter-tool" + ], + "createdAt": "2024-07-10T17:11:42.680Z", + "updatedAt": "2024-07-10T17:11:42.680Z", + "id": "668ec0ceb57adb6813c44eb6", + "pid": "polusai/OMEConverter@0.3.2-dev0" + }, + { + "name": "polusai/PrecomputeSlideViewer", + "version": "1.7.0-dev0", + "title": "Precompute Slide Viewer", + "description": "Precomputes a plane series in DeepZoom, Neuroglancer, or OME Zarr format.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Madhuri Vihani", + "Nick Schaub", + "Antoine Gerardin", + "Najib Ishaq" + ], + "contact": "Madhuri.Vihani@nih.gov", + "container": "polusai/precompute-slide-plugin:1.7.0-dev0", + "entrypoint": "python3 -m polus.images.visualization.precompute_slide", + "inputs": [ + { + "description": "Input collection", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Build a DeepZoom, Neuroglancer, Zarr pyramid", + "format": [ + "enum" + ], + "name": "pyramidType", + "required": true, + "type": "string" + }, + { + "description": "Image is either Segmentation or Image", + "format": [ + "enum" + ], + "name": "imageType", + "required": false, + "type": "string" + }, + { + "description": "Pattern of the images in Input", + "format": [ + "string" + ], + "name": "filePattern", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Precomputed output", + "format": [ + "pyramid" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/LabShare/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Collection name...", + "key": "inputs.inpDir", + "title": "Input collection: ", + "type": "path" + }, + { + "description": "Build a DeepZoom, Neuroglancer, or Zarr pyramid?", + "fields": [ + "DeepZoom", + "Neuroglancer", + "Zarr" + ], + "key": "inputs.pyramidType", + "title": "Pyramid Type: ", + "type": "select", + "required": true, + "format": [ + "enum" + ] + }, + { + "condition": "inputs.pyramidType=='Neuroglancer'", + "description": "Image or Segmentation?", + "fields": [ + "image", + "segmentation" + ], + "key": "inputs.imageType", + "title": "Image Type: ", + "type": "select", + "required": false, + "format": [ + "enum" + ] + }, + { + "description": "Pattern of images in input collection (image_r{rrr}_c{ccc}_z{zzz}.ome.tif). ", + "key": "inputs.filePattern", + "title": "Image Pattern: ", + "type": "text", + "required": false, + "format": [ + "string" + ] + } + ], + "path": "visualization", + "tags": [ + "precompute-slide-tool" + ], + "createdAt": "2024-07-22T20:04:05.108Z", + "updatedAt": "2024-07-22T20:15:02.835Z", + "id": "669ebb35936ebc63b94e928a", + "pid": "polusai/PrecomputeSlideViewer@1.7.0-dev0" + } + ] +} \ No newline at end of file diff --git a/tests/test_fix_payload.py b/tests/test_fix_payload.py new file mode 100644 index 00000000..85304eb8 --- /dev/null +++ b/tests/test_fix_payload.py @@ -0,0 +1,25 @@ +import pytest +import json +import pathlib + +from sophios.api.utils.converter import update_payload_missing_inputs_outputs + + +@pytest.mark.fast +def test_fix_multi_node_payload() -> None: + + path = pathlib.Path(__file__).parent.resolve() + + with open( + path / "data/wfb_data/multi_node/multi_node_wfb.json", "r" + ) as file: + wfb = json.load(file) + + updated_payload = update_payload_missing_inputs_outputs(wfb) + + with open( + path / "data/wfb_data/multi_node/multi_node_wfb_truth.json", "r" + ) as file: + truth = json.load(file) + + assert updated_payload == truth From 7719dafde5fb8593651bfd86c5633d700830f89c Mon Sep 17 00:00:00 2001 From: VasuJ <145879890+vjaganat90@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:21:41 -0400 Subject: [PATCH 12/15] add user args for workflows built with python api (#283) Co-authored-by: Vasu Jaganath --- src/sophios/api/pythonapi.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/sophios/api/pythonapi.py b/src/sophios/api/pythonapi.py index a5984c5f..195ba7e1 100644 --- a/src/sophios/api/pythonapi.py +++ b/src/sophios/api/pythonapi.py @@ -489,6 +489,7 @@ class Workflow(BaseModel): steps: list # list[Process] # and cannot use Process defined after Workflow within a Workflow process_name: str + user_args: list[str] inputs: list[ProcessInput] = [] outputs: list[ProcessOutput] = [] _input_names: list[str] = PrivateAttr(default_factory=list) @@ -499,10 +500,11 @@ class Workflow(BaseModel): # field(default=None, init=False, repr=False) # TypeError: 'ModelPrivateAttr' object is not iterable - def __init__(self, steps: list, workflow_name: str): + def __init__(self, steps: list, workflow_name: str, user_args: list[str] = []): data = { "process_name": workflow_name, "steps": steps, + "user_args": user_args } super().__init__(**data) @@ -725,7 +727,7 @@ def compile(self, write_to_disk: bool = False) -> CompilerInfo: """ global global_config self._validate() - args = get_args(self.process_name) # Use mock CLI args + args = get_args(self.process_name, self.user_args) # Use mock CLI args graph = get_graph_reps(self.process_name) yaml_tree = YamlTree(StepId(self.process_name, 'global'), self.yaml) @@ -751,7 +753,7 @@ def run(self) -> None: plugins.logging_filters() compiler_info = self.compile(write_to_disk=True) - args = get_args(self.process_name) # Use mock CLI args + args = get_args(self.process_name, self.user_args) # Use mock CLI args rose_tree: RoseTree = compiler_info.rose # cwl-docker-extract recursively `docker pull`s all images in all subworkflows. From edbe28e46824ea57a04630ec32095d070f30c2f3 Mon Sep 17 00:00:00 2001 From: VasuJ <145879890+vjaganat90@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:23:35 -0400 Subject: [PATCH 13/15] remove redundant io in main & remove irrelevant warning/exit in compiler (#284) * add user args for workflows built with python api * remove redundant io in main & remove irrelevant warning/exit in compiler --------- Co-authored-by: Vasu Jaganath --- src/sophios/compiler.py | 4 ---- src/sophios/main.py | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/sophios/compiler.py b/src/sophios/compiler.py index 353c7837..e81e843b 100644 --- a/src/sophios/compiler.py +++ b/src/sophios/compiler.py @@ -881,10 +881,6 @@ def compile_workflow_once(yaml_tree_ast: YamlTree, new_keyval = {key: newval} elif 'Directory' == in_dict['type']: if not args.ignore_dir_path: - if in_dict['value'].startswith('/'): - print("Warning! directory can not start with '/'") - print("It is most likely an incorrect path! Can't create directories!") - sys.exit(1) ldir = Path(in_dict['value']) if not ldir.is_absolute(): ldir = Path('autogenerated') / ldir diff --git a/src/sophios/main.py b/src/sophios/main.py index 24a6dbdd..73838fc2 100644 --- a/src/sophios/main.py +++ b/src/sophios/main.py @@ -194,9 +194,6 @@ def main() -> None: print("(This may happen if you installed the graphviz python package") print("but not the graphviz system package.)") - if args.generate_cwl_workflow: - io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) - if args.run_local or args.generate_run_script: # cwl-docker-extract recursively `docker pull`s all images in all subworkflows. # This is important because cwltool only uses `docker run` when executing From 5d64de38b049394c4c79d633308eb2b75cd003e9 Mon Sep 17 00:00:00 2001 From: VasuJ <145879890+vjaganat90@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:24:04 -0400 Subject: [PATCH 14/15] Refactor some common transformations (#285) * add user args for workflows built with python api * remove redundant io in main & remove irrelevant warning/exit in compiler * Refactor some common transformations into post_compile --------- Co-authored-by: Vasu Jaganath --- src/sophios/api/http/restapi.py | 6 ++--- src/sophios/api/pythonapi.py | 29 +++------------------- src/sophios/main.py | 31 ++++-------------------- src/sophios/post_compile.py | 43 +++++++++++++++++++++++++++++++++ tests/test_examples.py | 14 ++--------- 5 files changed, 57 insertions(+), 66 deletions(-) create mode 100644 src/sophios/post_compile.py diff --git a/src/sophios/api/http/restapi.py b/src/sophios/api/http/restapi.py index ef66e4dc..9bdb6c8e 100644 --- a/src/sophios/api/http/restapi.py +++ b/src/sophios/api/http/restapi.py @@ -13,6 +13,7 @@ from sophios.utils_graphs import get_graph_reps from sophios.utils_yaml import wic_loader from sophios import utils_cwl +from sophios.post_compile import cwl_inline_runtag from sophios.cli import get_args from sophios.wic_types import CompilerInfo, Json, Tool, Tools, StepId, YamlTree, Cwl, NodeData from sophios.api.utils import converter @@ -132,9 +133,8 @@ async def compile_wf(request: Request) -> Json: tools_cwl, True, relative_run_path=True, testing=False) rose_tree = compiler_info.rose - if args.inline_cwl_runtag: - input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) - rose_tree = plugins.cwl_update_inline_runtag_rosetree(rose_tree, Path('autogenerated/'), True) + input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) + cwl_inline_runtag(args, rose_tree) # ======== OUTPUT PROCESSING ================ # ========= PROCESS COMPILED OBJECT ========= sub_node_data: NodeData = rose_tree.data diff --git a/src/sophios/api/pythonapi.py b/src/sophios/api/pythonapi.py index 195ba7e1..a81a8bd9 100644 --- a/src/sophios/api/pythonapi.py +++ b/src/sophios/api/pythonapi.py @@ -12,7 +12,7 @@ from cwl_utils.parser import load_document_by_uri, load_document_by_yaml from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator -from sophios import compiler, input_output, plugins, utils_cwl +from sophios import compiler, input_output, plugins, utils_cwl, post_compile from sophios import run_local as run_local_module from sophios.cli import get_args from sophios.utils_graphs import get_graph_reps @@ -756,32 +756,11 @@ def run(self) -> None: args = get_args(self.process_name, self.user_args) # Use mock CLI args rose_tree: RoseTree = compiler_info.rose - # cwl-docker-extract recursively `docker pull`s all images in all subworkflows. - # This is important because cwltool only uses `docker run` when executing - # workflows, and if there is a local image available, - # `docker run` will NOT query the remote repository for the latest image! - # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. - if args.container_engine == 'singularity': - cmd = ['cwl-docker-extract', '-s', '--dir', - f'{args.singularity_pull_dir}', f'autogenerated/{self.process_name}.cwl'] - else: - cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{self.process_name}.cwl'] - sub.run(cmd, check=True) - - # If you don't like it, you can programmatically overwrite anything in args - # args.docker_remove_entrypoints = True - if args.docker_remove_entrypoints: - # Requires root, so guard behind CLI option - if args.container_engine == 'docker': - plugins.remove_entrypoints_docker() - if args.container_engine == 'podman': - plugins.remove_entrypoints_podman() - - rose_tree = plugins.dockerPull_append_noentrypoint_rosetree(rose_tree) - input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) + post_compile.cwl_docker_extract(args, self.process_name) + post_compile.remove_entrypoints(args, rose_tree) # Do NOT capture stdout and/or stderr and pipe warnings and errors into a black hole. - retval = run_local_module.run_local(args, rose_tree, args.cachedir, 'cwltool', True) + retval = run_local_module.run_local(args, rose_tree, args.cachedir, args.cwl_runner, True) # Finally, since there is an output file copying bug in cwltool, # we need to copy the output files manually. See comment above. diff --git a/src/sophios/main.py b/src/sophios/main.py index 73838fc2..63a17b69 100644 --- a/src/sophios/main.py +++ b/src/sophios/main.py @@ -11,6 +11,7 @@ from sophios.utils_yaml import wic_loader from . import input_output as io +from . import post_compile as pc from . import ast, cli, compiler, inference, inlineing, plugins, run_local, utils # , utils_graphs from .schemas import wic_schema from .wic_types import GraphData, GraphReps, Json, StepId, Yaml, YamlTree @@ -170,11 +171,8 @@ def main() -> None: io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) - # this has to happen after at least one write - # so we can copy from local cwl_dapters in autogenerated/ - if args.inline_cwl_runtag: - rose_tree = plugins.cwl_update_inline_runtag_rosetree(rose_tree, Path('autogenerated/'), True) - io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) + pc.cwl_inline_runtag(args, rose_tree) + io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) if args.graphviz: if shutil.which('dot'): @@ -195,27 +193,8 @@ def main() -> None: print("but not the graphviz system package.)") if args.run_local or args.generate_run_script: - # cwl-docker-extract recursively `docker pull`s all images in all subworkflows. - # This is important because cwltool only uses `docker run` when executing - # workflows, and if there is a local image available, - # `docker run` will NOT query the remote repository for the latest image! - # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. - if args.container_engine == 'singularity': - cmd = ['cwl-docker-extract', '-s', '--dir', - f'{args.singularity_pull_dir}', f'autogenerated/{yaml_stem}.cwl'] - else: - cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{yaml_stem}.cwl'] - sub.run(cmd, check=True) - - if args.docker_remove_entrypoints: - # Requires root, so guard behind CLI option - if args.container_engine == 'docker': - plugins.remove_entrypoints_docker() - if args.container_engine == 'podman': - plugins.remove_entrypoints_podman() - - rose_tree = plugins.dockerPull_append_noentrypoint_rosetree(rose_tree) - io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) + pc.cwl_docker_extract(args, yaml_stem) + pc.remove_entrypoints(args, rose_tree) run_local.run_local(args, rose_tree, args.cachedir, args.cwl_runner, False) diff --git a/src/sophios/post_compile.py b/src/sophios/post_compile.py new file mode 100644 index 00000000..40d61c44 --- /dev/null +++ b/src/sophios/post_compile.py @@ -0,0 +1,43 @@ +import argparse +from pathlib import Path +import subprocess as sub + +from . import plugins +from . import input_output as io +from .wic_types import RoseTree + + +def cwl_docker_extract(args: argparse.Namespace, file_name: str) -> None: + """Helper function to do the cwl_docker_extract""" + # cwl-docker-extract recursively `docker pull`s all images in all subworkflows. + # This is important because cwltool only uses `docker run` when executing + # workflows, and if there is a local image available, + # `docker run` will NOT query the remote repository for the latest image! + # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. + if args.container_engine == 'singularity': + cmd = ['cwl-docker-extract', '-s', '--dir', + f'{args.singularity_pull_dir}', f'autogenerated/{file_name}.cwl'] + else: + cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{file_name}.cwl'] + sub.run(cmd, check=True) + + +def cwl_inline_runtag(args: argparse.Namespace, rose_tree: RoseTree) -> None: + """Transform with cwl inline runtag""" + # this has to happen after at least one write + # so we can copy from local cwl_dapters in autogenerated/ + if args.inline_cwl_runtag: + rose_tree = plugins.cwl_update_inline_runtag_rosetree(rose_tree, Path('autogenerated/'), True) + + +def remove_entrypoints(args: argparse.Namespace, rose_tree: RoseTree) -> None: + """Remove entry points""" + if args.docker_remove_entrypoints: + # Requires root, so guard behind CLI option + if args.container_engine == 'docker': + plugins.remove_entrypoints_docker() + if args.container_engine == 'podman': + plugins.remove_entrypoints_podman() + + rose_tree = plugins.dockerPull_append_noentrypoint_rosetree(rose_tree) + io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) diff --git a/tests/test_examples.py b/tests/test_examples.py index 153fdedb..a4322ac8 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -22,6 +22,7 @@ from sophios import auto_gen_header from sophios.cli import get_args from sophios.utils_yaml import wic_loader +from sophios.post_compile import cwl_docker_extract from sophios.wic_types import NodeData, StepId, Yaml, YamlTree, Json from sophios.utils_graphs import get_graph_reps @@ -212,18 +213,7 @@ def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str, args: argp sophios.input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) if docker_pull_only: - # cwl-docker-extract recursively `docker pull`s all images in all subworkflows. - # This is important because cwltool only uses `docker run` when executing - # workflows, and if there is a local image available, - # `docker run` will NOT query the remote repository for the latest image! - # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. - if args.container_engine == 'singularity': - cmd = ['cwl-docker-extract', '-s', '--dir', - f'{args.singularity_pull_dir}', f'autogenerated/{Path(yml_path).stem}.cwl'] - else: - cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{Path(yml_path).stem}.cwl'] - sub.run(cmd, check=True) - + cwl_docker_extract(args, Path(yml_path).stem) return if args.docker_remove_entrypoints: From 2e19fc85b000746ea2d4c8e51d17651d04330108 Mon Sep 17 00:00:00 2001 From: Vasu Jaganath Date: Wed, 16 Oct 2024 12:29:59 -0400 Subject: [PATCH 15/15] pypi release prep 015 --- pyproject.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d9df3db9..07b41b91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,11 +105,6 @@ runners = [ "toil[cwl]", "cwl-utils", ] -runners-src = [ - "toil[cwl] @ git+https://github.com/sameeul/toil.git", - "cwltool @ git+https://github.com/sameeul/cwltool.git", - "cwl-utils @ git+https://github.com/sameeul/cwl-utils.git", -] # See docs/requirements.txt doc = [ "sphinx", @@ -119,7 +114,6 @@ doc = [ plots = ["matplotlib"] cyto = ["ipycytoscape"] # only for DAG visualization all_except_runner_src = ["sophios[test,doc,plots,cyto,mypy-types]"] -all = ["sophios[test,doc,plots,cyto,runners-src,mypy-types]"] [project.scripts] sophios = "sophios.main:main"