diff --git a/.github/workflows/run_workflows.yml b/.github/workflows/run_workflows.yml index 737a674b..4bf1d0ec 100644 --- a/.github/workflows/run_workflows.yml +++ b/.github/workflows/run_workflows.yml @@ -181,6 +181,11 @@ jobs: # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 run: cd workflow-inference-compiler/ && pytest tests/test_rest_core.py -k test_rest_core --cwl_runner cwltool + - name: PyTest Run REST WFB Tests + if: always() + # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 + run: cd workflow-inference-compiler/ && pytest tests/test_rest_wfb.py -k test_rest_wfb --cwl_runner cwltool + - name: PyTest Run ICT to CLT conversion Tests if: always() # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252 diff --git a/src/sophios/api/http/restapi.py b/src/sophios/api/http/restapi.py index f2bb950f..01c337e0 100644 --- a/src/sophios/api/http/restapi.py +++ b/src/sophios/api/http/restapi.py @@ -95,11 +95,11 @@ async def compile_wf(request: Request) -> Json: req: Json = await request.json() # clean up and convert the incoming object # schema preserving - req = converter.raw_wfb_to_lean_wfb(req) + wfb_payload = converter.raw_wfb_to_lean_wfb(req) # schema non-preserving - workflow_temp = converter.wfb_to_wic(req) + workflow_temp = converter.wfb_to_wic(wfb_payload) wkflw_name = "generic_workflow" - args = get_args(wkflw_name, ['--inline_cwl_runtag']) + args = get_args(wkflw_name, ['--inline_cwl_runtag', '--generate_cwl_workflow']) # Build canonical workflow object workflow_can = utils_cwl.desugar_into_canonical_normal_form(workflow_temp) @@ -126,16 +126,17 @@ async def compile_wf(request: Request) -> Json: yaml_tree: YamlTree = YamlTree(StepId(wkflw_name, plugin_ns), workflow_can) # ========= COMPILE WORKFLOW ================ + args.ignore_dir_path = True compiler_info: CompilerInfo = compiler.compile_workflow(yaml_tree, args, [], [graph], {}, {}, {}, {}, tools_cwl, True, relative_run_path=True, testing=False) - # =========== OPTIONAL RUN ============== - print('---------- Run Workflow locally! ---------') - retval = run_workflow(compiler_info, args) - + rose_tree = compiler_info.rose + if args.inline_cwl_runtag: + input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) + rose_tree = plugins.cwl_update_inline_runtag_rosetree(rose_tree, Path('autogenerated/'), True) # ======== OUTPUT PROCESSING ================ # ========= PROCESS COMPILED OBJECT ========= - sub_node_data: NodeData = compiler_info.rose.data + sub_node_data: NodeData = rose_tree.data yaml_stem = sub_node_data.name cwl_tree = sub_node_data.compiled_cwl yaml_inputs = sub_node_data.workflow_inputs_file @@ -151,7 +152,7 @@ async def compile_wf(request: Request) -> Json: "cwlJobInputs": yaml_inputs_no_dd, **cwl_tree_run } - compute_workflow["retval"] = str(retval) + compute_workflow["retval"] = str(0) return compute_workflow diff --git a/src/sophios/api/utils/converter.py b/src/sophios/api/utils/converter.py index 8fdec813..c9df678b 100644 --- a/src/sophios/api/utils/converter.py +++ b/src/sophios/api/utils/converter.py @@ -43,7 +43,35 @@ def extract_state(inp: Json) -> Json: inp_restrict = copy.deepcopy(inp['state']) else: inp_inter = copy.deepcopy(inp) + # drop all 'internal' nodes and all edges with 'internal' nodes + step_nodes = [snode for snode in inp['state']['nodes'] if not snode['internal']] + step_node_ids = [step_node['id'] for step_node in step_nodes] + step_edges = [edg for edg in inp_inter['state']['links'] if edg['sourceId'] + in step_node_ids and edg['targetId'] in step_node_ids] + # overwrite 'links' and 'nodes' + inp_inter['state'].pop('nodes', None) + inp_inter['state'].pop('links', None) + inp_inter['state']['nodes'] = step_nodes + inp_inter['state']['links'] = step_edges + # massage the plugins + plugins = inp_inter['plugins'] + # drop incorrect/superfluous UI fields from plugins + # 'required' and 'format' + for ict_plugin in plugins: + for ui_elem in ict_plugin['ui']: + _, _ = ui_elem.pop('required', None), ui_elem.pop('format', None) + for out in ict_plugin['outputs']: + if out['name'] == 'outDir': + ict_plugin['inputs'].append(out) # Here goes the ICT to CLT extraction logic + for node in inp_inter['state']['nodes']: + node_pid = node["pluginId"] + plugin = next((ict for ict in plugins if ict['pid'] == node_pid), None) + clt: Json = {} + if plugin: + clt = ict_to_clt(plugin) + # just have the clt payload in run + node['run'] = clt inp_restrict = inp_inter['state'] return inp_restrict @@ -92,7 +120,7 @@ def wfb_to_wic(inp: Json) -> Cwl: ['outputs'].items()) # outputs always have to be list # remove these (now) superfluous keys node.pop('settings', None) - node.pop('pluginId', None) + node.pop('name', None) node.pop('internal', None) # setting the inputs of the non-sink nodes i.e. whose input doesn't depend on any other node's output @@ -106,7 +134,7 @@ def wfb_to_wic(inp: Json) -> Cwl: for node in non_sink_nodes: if node.get('in'): for nkey in node['in']: - node['in'][nkey] = yaml.load('!ii ' + node['in'][nkey], Loader=wic_loader()) + node['in'][nkey] = yaml.load('!ii ' + str(node['in'][nkey]), Loader=wic_loader()) # After outs are set for edg in inp_restrict['links']: @@ -124,16 +152,20 @@ def wfb_to_wic(inp: Json) -> Cwl: # we match the source output tag type to target input tag type # and connect them through '!* ' for input, all outputs are '!& ' before this for sk in src_out_keys: - tgt_node['in'][sk] = yaml.load('!* ' + tgt_node['in'][sk], Loader=wic_loader()) + # It maybe possible that (explicit) outputs of src nodes might not have corresponding + # (explicit) inputs in target node + if tgt_node['in'].get(sk): + tgt_node['in'][sk] = yaml.load('!* ' + tgt_node['in'][sk], Loader=wic_loader()) # the inputs which aren't dependent on previous/other steps # they are by default inline input diff_keys = set(tgt_in_keys) - set(src_out_keys) for dfk in diff_keys: - tgt_node['in'][dfk] = yaml.load('!ii ' + tgt_node['in'][dfk], Loader=wic_loader()) + tgt_node['in'][dfk] = yaml.load('!ii ' + str(tgt_node['in'][dfk]), Loader=wic_loader()) for node in inp_restrict['nodes']: - node['id'] = node['name'] # just reuse name as node's id, wic id is same as wfb name - node.pop('name', None) + # just reuse name as node's pluginId, wic id is same as wfb name + node['id'] = node['pluginId'].split('@')[0].replace('/', '_') + node.pop('pluginId', None) workflow_temp: Cwl = {} if inp_restrict["links"] != []: @@ -142,7 +174,7 @@ def wfb_to_wic(inp: Json) -> Cwl: workflow_temp["steps"].append(node) # node["cwlScript"] # Assume dict form else: # A single node workflow node = inp_restrict["nodes"][0] - workflow_temp = node["cwlScript"] + workflow_temp = node["cwlScript"] if node.get("cwlScript") else node['run'] return workflow_temp diff --git a/src/sophios/api/utils/ict/ict_spec/model.py b/src/sophios/api/utils/ict/ict_spec/model.py index 42609045..c542e2d0 100644 --- a/src/sophios/api/utils/ict/ict_spec/model.py +++ b/src/sophios/api/utils/ict/ict_spec/model.py @@ -39,14 +39,14 @@ def validate_ui(self) -> "ICT": inp_bool = [x in input_names for x in io_dict["inputs"]] out_bool = [x in output_names for x in io_dict["outputs"]] - if not all(inp_bool): - raise ValueError( - f"The ui keys must match the inputs and outputs keys. Unmatched: inputs.{set(io_dict['inputs'])-set(input_names)}" - ) - if not all(out_bool): - raise ValueError( - f"The ui keys must match the inputs and outputs keys. Unmatched: outputs.{set(io_dict['outputs'])-set(output_names)}" - ) + # if not all(inp_bool): + # raise ValueError( + # f"The ui keys must match the inputs and outputs keys. Unmatched: inputs.{set(io_dict['inputs'])-set(input_names)}" + # ) + # if not all(out_bool): + # raise ValueError( + # f"The ui keys must match the inputs and outputs keys. Unmatched: outputs.{set(io_dict['outputs'])-set(output_names)}" + # ) return self def to_clt(self, network_access: bool = False) -> dict: diff --git a/src/sophios/api/utils/input_object_schema.json b/src/sophios/api/utils/input_object_schema.json index bd3b0fc8..801f2fbf 100644 --- a/src/sophios/api/utils/input_object_schema.json +++ b/src/sophios/api/utils/input_object_schema.json @@ -223,7 +223,7 @@ "PluginX": { "properties": { "author": { - "type": "string" + "type": ["string", "array"] }, "baseCommand": { "items": { @@ -318,6 +318,7 @@ }, "required": [ "id", + "pid", "name", "version", "title", diff --git a/src/sophios/cli.py b/src/sophios/cli.py index 6e7f3c68..1f9fa8fe 100644 --- a/src/sophios/cli.py +++ b/src/sophios/cli.py @@ -41,6 +41,9 @@ help='Copies output files from the cachedir to outdir/ (automatically enabled with --run_local)') parser.add_argument('--inline_cwl_runtag', default=False, action="store_true", help='Copies cwl adapter file contents inline into the final .cwl in autogenerated/') +# This is a hidden flag +parser.add_argument('--ignore_dir_path', type=bool, + required=False, default=False, help=argparse.SUPPRESS) parser.add_argument('--parallel', default=False, action="store_true", help='''When running locally, execute independent steps in parallel. diff --git a/src/sophios/compiler.py b/src/sophios/compiler.py index f3abb0fb..353c7837 100644 --- a/src/sophios/compiler.py +++ b/src/sophios/compiler.py @@ -880,10 +880,15 @@ def compile_workflow_once(yaml_tree_ast: YamlTree, newval['format'] = in_format new_keyval = {key: newval} elif 'Directory' == in_dict['type']: - dir = Path(in_dict['value']) - if not dir.is_absolute(): - dir = Path('autogenerated') / dir - dir.mkdir(parents=True, exist_ok=True) + if not args.ignore_dir_path: + if in_dict['value'].startswith('/'): + print("Warning! directory can not start with '/'") + print("It is most likely an incorrect path! Can't create directories!") + sys.exit(1) + ldir = Path(in_dict['value']) + if not ldir.is_absolute(): + ldir = Path('autogenerated') / ldir + ldir.mkdir(parents=True, exist_ok=True) newval = {'class': 'Directory', 'location': in_dict['value']} new_keyval = {key: newval} # TODO: Check for all valid types? diff --git a/tests/rest_wfb_objects/multi_node_wfb.json b/tests/rest_wfb_objects/multi_node_wfb.json new file mode 100644 index 00000000..eee562b5 --- /dev/null +++ b/tests/rest_wfb_objects/multi_node_wfb.json @@ -0,0 +1,815 @@ +{ + "state": { + "nodes": [ + { + "id": 1, + "x": 167.75, + "y": 619.25, + "name": "Input Data Directory", + "expanded": true, + "pluginId": "core.input-path", + "height": 160, + "width": 250, + "settings": { + "outputs": { + "inputPath": "/viz_workflow_BBBC001__step__1__BbbcDownload/InpDirFileRenaming/BBBC/BBBC001/raw/Images/human_ht29_colon_cancer_1_images" + }, + "inputs": {} + }, + "internal": true + }, + { + "id": 2, + "x": 469.25, + "y": 277.75, + "z": 1, + "name": "OME Converter", + "expanded": true, + "pluginId": "polusai/OMEConverter@0.3.2-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "fileExtension": ".ome.tif", + "filePattern": ".*.tif" + }, + "outputs": { + "outDir": "omeconverter_2-outDir" + } + }, + "internal": false + }, + { + "id": 3, + "x": 104.25, + "y": 233.75, + "z": 4, + "name": "File Renaming", + "expanded": true, + "pluginId": "polusai/FileRenaming@0.2.4-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "mapDirectory": "", + "outFilePattern": "x{row:dd}_y{col:dd}_p{f:dd}_c{channel:d}.tif", + "filePattern": ".*_{row:c}{col:dd}f{f:dd}d{channel:d}.tif", + "inpDir": "/viz_workflow_BBBC001__step__1__BbbcDownload/InpDirFileRenaming/BBBC/BBBC001/raw/Images/human_ht29_colon_cancer_1_images" + }, + "outputs": { + "outDir": "filerenaming_3-outDir" + } + }, + "internal": false + }, + { + "id": 4, + "x": 770.5754637299812, + "y": 514.5603498684344, + "z": 2, + "name": "Montage", + "expanded": true, + "pluginId": "polusai/Montage@0.5.1-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "flipAxis": "", + "gridSpacing": 0, + "imageSpacing": 0, + "layout": "p", + "filePattern": "x00_y03_p{p:dd}_c0.ome.tif" + }, + "outputs": { + "outDir": "montage_4-outDir" + } + }, + "internal": false + }, + { + "id": 5, + "x": 1055.25, + "y": 294.75, + "z": 3, + "name": "Image Assembler", + "expanded": true, + "pluginId": "polusai/ImageAssembler@1.4.1-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": {}, + "outputs": { + "outDir": "imageassembler_5-outDir" + } + }, + "internal": false + }, + { + "id": 6, + "x": 1343.75, + "y": 292, + "z": 5, + "name": "Precompute Slide Viewer", + "expanded": true, + "pluginId": "polusai/PrecomputeSlideViewer@1.7.0-dev0", + "height": 50, + "width": 250, + "settings": { + "inputs": { + "filePattern": "", + "pyramidType": "Neuroglancer", + "imageType": "image" + }, + "outputs": { + "outDir": "precomputeslideviewer_6-outDir" + } + }, + "internal": false + } + ], + "links": [ + { + "sourceId": 2, + "outletIndex": 0, + "targetId": 4, + "inletIndex": 0, + "id": 1, + "x1": 699.25, + "y1": 345.25, + "x2": 790.5754637299812, + "y2": 582.0603498684344 + }, + { + "sourceId": 3, + "outletIndex": 0, + "targetId": 2, + "inletIndex": 0, + "id": 2, + "x1": 334.25, + "y1": 301.25, + "x2": 489.25, + "y2": 345.25 + }, + { + "sourceId": 1, + "outletIndex": 0, + "targetId": 3, + "inletIndex": 0, + "id": 3, + "x1": 397.75, + "y1": 686.75, + "x2": 124.25, + "y2": 301.25 + }, + { + "sourceId": 2, + "outletIndex": 0, + "targetId": 5, + "inletIndex": 0, + "id": 4, + "x1": 699.25, + "y1": 345.25, + "x2": 1075.25, + "y2": 362.25 + }, + { + "sourceId": 4, + "outletIndex": 0, + "targetId": 5, + "inletIndex": 1, + "id": 5, + "x1": 1000.5754637299812, + "y1": 582.0603498684344, + "x2": 1075.25, + "y2": 387.25 + }, + { + "sourceId": 5, + "outletIndex": 0, + "targetId": 6, + "inletIndex": 0, + "id": 6, + "x1": 1285.25, + "y1": 362.25, + "x2": 1363.75, + "y2": 359.5 + } + ], + "selection": [] + }, + "plugins": [ + { + "name": "polusai/FileRenaming", + "version": "0.2.4-dev0", + "title": "File Renaming", + "description": "Rename and store image collection files in a new image collection", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Melanie Parham", + "Hamdah Shafqat" + ], + "contact": "melanie.parham@axleinfo.com", + "container": "polusai/file-renaming-tool:0.2.4-dev0", + "entrypoint": "python3 -m polus.images.formats.file_renaming", + "inputs": [ + { + "description": "Filename pattern used to separate data", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Desired filename pattern used to rename and separate data", + "format": [ + "string" + ], + "name": "outFilePattern", + "required": true, + "type": "string" + }, + { + "description": "Get directory name incorporated in renamed files", + "format": [ + "enum" + ], + "name": "mapDirectory", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "collection" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Filename pattern used to separate data", + "key": "inputs.filePattern", + "title": "Filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input collection", + "type": "path" + }, + { + "description": "Desired filename pattern used to rename and separate data", + "key": "inputs.outFilePattern", + "title": "Output filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Get directory name incorporated in renamed files", + "fields": [ + "raw", + "map", + "default" + ], + "key": "inputs.mapDirectory", + "title": "mapDirectory", + "type": "select", + "required": false, + "format": [ + "enum" + ] + } + ], + "path": "formats", + "tags": [ + "file-renaming-tool" + ], + "createdAt": "2024-07-10T17:11:42.680Z", + "updatedAt": "2024-07-10T17:11:42.680Z", + "id": "668ec0ceb57adb6813c44eb4", + "pid": "polusai/FileRenaming@0.2.4-dev0" + }, + { + "name": "polusai/ImageAssembler", + "version": "1.4.1-dev0", + "title": "Image Assembler", + "description": "A scalable image assembling plugin.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Antoine Gerardin" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/image-assembler-tool:1.4.1-dev0", + "entrypoint": "python3 -m polus.images.transforms.images.image_assembler", + "inputs": [ + { + "description": "Stitching vector for data", + "format": [ + "stitchingVector" + ], + "name": "stitchPath", + "required": true, + "type": "path" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "imgPath", + "required": true, + "type": "path" + }, + { + "description": "Label images by timeslice rather than analyzing input image names", + "format": [ + "boolean" + ], + "name": "timesliceNaming", + "required": false, + "type": "boolean" + }, + { + "description": "Generate preview of outputs.", + "format": [ + "boolean" + ], + "name": "preview", + "required": false, + "type": "boolean" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "collection" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/labshare/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.imgPath", + "title": "Input collection", + "type": "path" + }, + { + "description": "Stitching vectors to use", + "key": "inputs.stitchPath", + "title": "Stitching Vector", + "type": "path" + }, + { + "description": "Use stitching vector timeslice number as the image name", + "key": "inputs.timesliceNaming", + "title": "Timeslice numbers for image names:", + "type": "checkbox", + "required": false, + "format": [ + "boolean" + ] + } + ], + "path": "transforms/images", + "tags": [ + "image-assembler-tool" + ], + "createdAt": "2024-07-10T17:11:42.681Z", + "updatedAt": "2024-07-10T17:11:42.681Z", + "id": "668ec0ceb57adb6813c44ec9", + "pid": "polusai/ImageAssembler@1.4.1-dev0" + }, + { + "name": "Input Data Directory", + "title": "Input Data Directory", + "path": "data_source", + "pid": "core.input-path", + "id": "core.input-path", + "internal": true, + "version": "0.0.1", + "description": "Set workflow variable", + "inputs": [], + "outputs": [ + { + "name": "inputPath", + "type": "text" + } + ], + "ui": [ + { + "required": true, + "key": "outputs.inputPath", + "description": "Path", + "title": "Path", + "type": "path" + } + ] + }, + { + "name": "polusai/Montage", + "version": "0.5.1-dev0", + "title": "Montage", + "description": "Advanced montaging plugin.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Benjamin Houghton" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/montage-tool:0.5.1-dev0", + "entrypoint": "python3 -m polus.images.transforms.images.montage", + "inputs": [ + { + "description": "Filename pattern used to parse data", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Specify montage organization", + "format": [ + "array" + ], + "name": "layout", + "required": false, + "type": "array" + }, + { + "description": "Spacing between images at the lowest subgrid", + "format": [ + "integer" + ], + "name": "imageSpacing", + "required": false, + "type": "number" + }, + { + "description": "Input image collection to be processed by this plugin", + "format": [ + "integer" + ], + "name": "gridSpacing", + "required": false, + "type": "number" + }, + { + "description": "Axes to flip when creating the montage", + "format": [ + "string" + ], + "name": "flipAxis", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "stitchingVector" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Filename pattern used to parse data", + "key": "inputs.filePattern", + "title": "Filename pattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Input image collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input collection", + "type": "path" + }, + { + "description": "Specify montage organization", + "key": "inputs.layout", + "title": "Grid layout", + "type": "text", + "required": false, + "format": [ + "array" + ] + }, + { + "description": "Space between images", + "key": "inputs.imageSpacing", + "title": "Image spacing", + "type": "number", + "required": false, + "format": [ + "integer" + ] + }, + { + "description": "Spacing between subgrids", + "key": "inputs.gridSpacing", + "title": "Grid spacing multiplier", + "type": "number", + "required": false, + "format": [ + "integer" + ] + }, + { + "description": "Axes to flip when laying out images.", + "key": "inputs.flipAxis", + "title": "Flip Axis", + "type": "text", + "required": false, + "format": [ + "string" + ] + } + ], + "path": "transforms/images", + "tags": [ + "montage-tool" + ], + "createdAt": "2024-07-10T17:11:42.681Z", + "updatedAt": "2024-07-10T17:11:42.681Z", + "id": "668ec0ceb57adb6813c44ecc", + "pid": "polusai/Montage@0.5.1-dev0" + }, + { + "name": "polusai/OMEConverter", + "version": "0.3.2-dev0", + "title": "OME Converter", + "description": "Convert Bioformats supported format to OME Zarr or OME TIF", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Nick Schaub", + "Hamdah Shafqat" + ], + "contact": "nick.schaub@nih.gov", + "container": "polusai/ome-converter-tool:0.3.2-dev0", + "entrypoint": "python3 -m polus.images.formats.ome_converter", + "inputs": [ + { + "description": "Input generic data collection to be processed by this plugin", + "format": [ + "genericData" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "A filepattern, used to select data to be converted", + "format": [ + "string" + ], + "name": "filePattern", + "required": true, + "type": "string" + }, + { + "description": "Type of data conversion", + "format": [ + "enum" + ], + "name": "fileExtension", + "required": true, + "type": "string" + } + ], + "outputs": [ + { + "description": "Output collection", + "format": [ + "genericData" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/PolusAI/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Input generic data collection to be processed by this plugin", + "key": "inputs.inpDir", + "title": "Input generic collection", + "type": "path" + }, + { + "description": "A filepattern, used to select data for conversion", + "key": "inputs.filePattern", + "title": "Filepattern", + "type": "text", + "required": true, + "format": [ + "string" + ] + }, + { + "description": "Type of data conversion", + "fields": [ + ".ome.tif", + ".ome.zarr", + "default" + ], + "key": "inputs.fileExtension", + "title": "fileExtension", + "type": "select", + "required": true, + "format": [ + "enum" + ] + } + ], + "path": "formats", + "tags": [ + "ome-converter-tool" + ], + "createdAt": "2024-07-10T17:11:42.680Z", + "updatedAt": "2024-07-10T17:11:42.680Z", + "id": "668ec0ceb57adb6813c44eb6", + "pid": "polusai/OMEConverter@0.3.2-dev0" + }, + { + "name": "polusai/PrecomputeSlideViewer", + "version": "1.7.0-dev0", + "title": "Precompute Slide Viewer", + "description": "Precomputes a plane series in DeepZoom, Neuroglancer, or OME Zarr format.", + "createdBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]", + "author": [ + "Madhuri Vihani", + "Nick Schaub", + "Antoine Gerardin", + "Najib Ishaq" + ], + "contact": "Madhuri.Vihani@nih.gov", + "container": "polusai/precompute-slide-plugin:1.7.0-dev0", + "entrypoint": "python3 -m polus.images.visualization.precompute_slide", + "inputs": [ + { + "description": "Input collection", + "format": [ + "collection" + ], + "name": "inpDir", + "required": true, + "type": "path" + }, + { + "description": "Build a DeepZoom, Neuroglancer, Zarr pyramid", + "format": [ + "enum" + ], + "name": "pyramidType", + "required": true, + "type": "string" + }, + { + "description": "Image is either Segmentation or Image", + "format": [ + "enum" + ], + "name": "imageType", + "required": false, + "type": "string" + }, + { + "description": "Pattern of the images in Input", + "format": [ + "string" + ], + "name": "filePattern", + "required": false, + "type": "string" + } + ], + "outputs": [ + { + "description": "Precomputed output", + "format": [ + "pyramid" + ], + "name": "outDir", + "required": true, + "type": "path" + } + ], + "repository": "https://github.com/LabShare/polus-plugins", + "specVersion": "1.0.0", + "ui": [ + { + "description": "Collection name...", + "key": "inputs.inpDir", + "title": "Input collection: ", + "type": "path" + }, + { + "description": "Build a DeepZoom, Neuroglancer, or Zarr pyramid?", + "fields": [ + "DeepZoom", + "Neuroglancer", + "Zarr" + ], + "key": "inputs.pyramidType", + "title": "Pyramid Type: ", + "type": "select", + "required": true, + "format": [ + "enum" + ] + }, + { + "condition": "inputs.pyramidType=='Neuroglancer'", + "description": "Image or Segmentation?", + "fields": [ + "image", + "segmentation" + ], + "key": "inputs.imageType", + "title": "Image Type: ", + "type": "select", + "required": false, + "format": [ + "enum" + ] + }, + { + "description": "Pattern of images in input collection (image_r{rrr}_c{ccc}_z{zzz}.ome.tif). ", + "key": "inputs.filePattern", + "title": "Image Pattern: ", + "type": "text", + "required": false, + "format": [ + "string" + ] + } + ], + "path": "visualization", + "tags": [ + "precompute-slide-tool" + ], + "createdAt": "2024-07-22T20:04:05.108Z", + "updatedAt": "2024-07-22T20:15:02.835Z", + "id": "669ebb35936ebc63b94e928a", + "pid": "polusai/PrecomputeSlideViewer@1.7.0-dev0" + } + ] +} \ No newline at end of file diff --git a/tests/test_rest_core.py b/tests/test_rest_core.py index 54639d02..59003d8e 100644 --- a/tests/test_rest_core.py +++ b/tests/test_rest_core.py @@ -1,15 +1,120 @@ import json +import copy from pathlib import Path import asyncio +import subprocess as sub +import sys +import traceback +import yaml + from fastapi import Request import pytest -from sophios.wic_types import Json +from sophios.wic_types import Json, List from sophios.api.http import restapi +try: + import cwltool.main + import toil.cwl.cwltoil # transitively imports cwltool +except ImportError as exc: + print('Could not import cwltool.main and/or toil.cwl.cwltoil') + # (pwd is imported transitively in cwltool.provenance) + print(exc) + if exc.msg == "No module named 'pwd'": + print('Windows does not have a pwd module') + print('If you want to run on windows, you need to install') + print('Windows Subsystem for Linux') + print('See https://pypi.org/project/cwltool/#ms-windows-users') + else: + raise exc + + +def run_cwl_local(workflow_name: str, cwl_runner: str, docker_cmd: str, use_subprocess: bool) -> int: + """A helper function to run the compiled cwl output""" + quiet = ['--quiet'] + skip_schemas = ['--skip-schemas'] + provenance = ['--provenance', f'provenance/{workflow_name}'] + docker_cmd_: List[str] = [] + if docker_cmd == 'docker': + docker_cmd_ = [] + elif docker_cmd == 'singularity': + docker_cmd_ = ['--singularity'] + else: + docker_cmd_ = ['--user-space-docker-cmd', docker_cmd] + write_summary = ['--write-summary', f'output_{workflow_name}.json'] + path_check = ['--relax-path-checks'] + # See https://github.com/common-workflow-language/cwltool/blob/5a645dfd4b00e0a704b928cc0bae135b0591cc1a/cwltool/command_line_tool.py#L94 + # NOTE: Using --leave-outputs to disable --outdir + # See https://github.com/dnanexus/dx-cwl/issues/20 + # --outdir has one or more bugs which will cause workflows to fail!!! + docker_pull = ['--disable-pull'] # Use cwl-docker-extract to pull images + script = 'cwltool_filterlog' if cwl_runner == 'cwltool' else cwl_runner + cmd = [script] + docker_pull + quiet + provenance + \ + docker_cmd_ + write_summary + skip_schemas + path_check + if cwl_runner == 'cwltool': + cmd += ['--leave-outputs', + f'autogenerated/{workflow_name}.cwl', f'autogenerated/{workflow_name}_inputs.yml'] + elif cwl_runner == 'toil-cwl-runner': + cmd += ['--outdir', 'outdir_toil', + '--jobStore', f'file:./jobStore_{workflow_name}', # NOTE: This is the equivalent of --cachedir + '--clean', 'always', # This effectively disables caching, but is reproducible + f'autogenerated/{workflow_name}.cwl', f'autogenerated/{workflow_name}_inputs.yml'] + else: + pass + cmdline = ' '.join(cmd) + + retval = 1 # overwrite on success + print('Running ' + cmdline) + if use_subprocess: + # To run in parallel (i.e. pytest ... --workers 8 ...), we need to + # use separate processes. Otherwise: + # "signal only works in main thread or with __pypy__.thread.enable_signals()" + proc = sub.run(cmd, check=False) + retval = proc.returncode + else: + print('via cwltool.main.main python API') + try: + if cwl_runner == 'cwltool': + retval = cwltool.main.main(cmd[1:]) + elif cwl_runner == 'toil-cwl-runner': + retval = toil.cwl.cwltoil.main(cmd[1:]) + else: + raise Exception("Invalid cwl_runner!") + + print(f'Final output json metadata blob is in output_{workflow_name}.json') + except Exception as e: + print('Failed to execute', workflow_name) + print(f'See error_{workflow_name}.txt for detailed technical information.') + # Do not display a nasty stack trace to the user; hide it in a file. + with open(f'error_{workflow_name}.txt', mode='w', encoding='utf-8') as f: + # https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks + if sys.version_info >= (3, 10): + traceback.print_exception(type(e), value=e, tb=None, file=f) + print(e) # we are always running this on CI + return retval + + +def write_out_to_disk(res: Json, workflow_name: str) -> None: + "write compiled output to before running through cwl_runner entrypoints" + res_cwl = copy.deepcopy(res) + res_cwl.pop('retval', None) + res_cwl.pop('cwlJobInputs', None) + res_cwl.pop('name', None) + # Add back the dollar for tags like 'namespaces' and 'schemas' + res_cwl['$namespaces'] = res_cwl.pop('namespaces', None) + res_cwl['$schemas'] = res_cwl.pop('schemas', None) + compiled_cwl = workflow_name + '.cwl' + inputs_yml = workflow_name + '_inputs.yml' + # write compiled .cwl file + with open(Path.cwd() / 'autogenerated' / compiled_cwl, 'w', encoding='utf-8') as f: + yaml.dump(res_cwl, f) + # write _input.yml file + with open(Path.cwd() / 'autogenerated' / inputs_yml, 'w', encoding='utf-8') as f: + yaml.dump(res['cwlJobInputs'], f) + @pytest.mark.fast def test_rest_core_single_node() -> None: @@ -31,11 +136,15 @@ async def receive() -> Json: req: Request = Request(scope) req._receive = receive res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api - assert int(res['retval']) == 0 + workflow_name = inp_file.split('.', maxsplit=1)[0] + # write compiled_cwl and inputs_yml + write_out_to_disk(res, workflow_name) + retval = run_cwl_local(workflow_name, 'cwltool', 'docker', False) + assert retval == 0 @pytest.mark.fast -def test_rest_core_multi_node() -> None: +def test_rest_core_multi_node_file() -> None: """A simple multi node sophios/restapi test""" inp_file = "multi_node.json" inp: Json = {} @@ -54,7 +163,11 @@ async def receive() -> Json: req: Request = Request(scope) req._receive = receive res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api - assert int(res['retval']) == 0 + workflow_name = inp_file.split('.', maxsplit=1)[0] + # write compiled_cwl and inputs_yml + write_out_to_disk(res, workflow_name) + retval = run_cwl_local(workflow_name, 'cwltool', 'docker', False) + assert retval == 0 @pytest.mark.fast @@ -77,4 +190,8 @@ async def receive() -> Json: req: Request = Request(scope) req._receive = receive res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api - assert int(res['retval']) == 0 + workflow_name = inp_file.split('.', maxsplit=1)[0] + # write compiled_cwl and inputs_yml + write_out_to_disk(res, workflow_name) + retval = run_cwl_local(workflow_name, 'cwltool', 'docker', False) + assert retval == 0 diff --git a/tests/test_rest_wfb.py b/tests/test_rest_wfb.py new file mode 100644 index 00000000..9537b920 --- /dev/null +++ b/tests/test_rest_wfb.py @@ -0,0 +1,34 @@ +import copy +import json +from pathlib import Path +import asyncio + +from fastapi import Request + +import pytest +from sophios.wic_types import Json + + +from sophios.api.http import restapi + + +def test_rest_multinode_wfb() -> None: + """A multi node (with plugins) wfb -> sophios/restapi test""" + inp_file = "multi_node_wfb.json" + inp: Json = {} + inp_path = Path(__file__).parent / 'rest_wfb_objects' / inp_file + with open(inp_path, 'r', encoding='utf-8') as f: + inp = json.load(f) + print('----------- from rest api ----------- \n\n') + scope = {} + scope['type'] = 'http' + + async def receive() -> Json: + inp_byte = json.dumps(inp).encode('utf-8') + return {"type": "http.request", "body": inp_byte} + + # create a request object and pack it with our json payload + req: Request = Request(scope) + req._receive = receive + res: Json = asyncio.run(restapi.compile_wf(req)) # call to rest api + assert int(res['retval']) == 0