Skip to content

Commit

Permalink
one step wfb, cwl dict format and uuid (#292)
Browse files Browse the repository at this point in the history
Co-authored-by: Vasu Jaganath <vasu.jaganath@axleinfo.com>
  • Loading branch information
vjaganat90 and Vasu Jaganath authored Nov 12, 2024
1 parent 9797013 commit 61abb94
Show file tree
Hide file tree
Showing 6 changed files with 240 additions and 48 deletions.
30 changes: 22 additions & 8 deletions src/sophios/api/http/restapi.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from pathlib import Path
import argparse
import copy
import uuid
import yaml


Expand All @@ -13,7 +14,7 @@
from sophios.utils_graphs import get_graph_reps
from sophios.utils_yaml import wic_loader
from sophios import utils_cwl
from sophios.post_compile import cwl_inline_runtag
from sophios.post_compile import cwl_inline_runtag, remove_entrypoints
from sophios.cli import get_args
from sophios.wic_types import CompilerInfo, Json, Tool, Tools, StepId, YamlTree, Cwl, NodeData
from sophios.api.utils import converter
Expand Down Expand Up @@ -94,14 +95,15 @@ async def compile_wf(request: Request) -> Json:
print('---------- Compile Workflow! ---------')
# ========= PROCESS REQUEST OBJECT ==========
req: Json = await request.json()
suppliedargs = ['--cwl_inline_runtag', '--generate_cwl_workflow']
# clean up and convert the incoming object
# schema preserving
req = converter.update_payload_missing_inputs_outputs(req)
wfb_payload = converter.raw_wfb_to_lean_wfb(req)
# schema non-preserving
workflow_temp = converter.wfb_to_wic(wfb_payload)
wkflw_name = "generic_workflow"
args = get_args(wkflw_name, ['--cwl_inline_runtag', '--generate_cwl_workflow'])
wkflw_name = "generic_workflow_" + str(uuid.uuid4())
args = get_args(wkflw_name, suppliedargs)

# Build canonical workflow object
workflow_can = utils_cwl.desugar_into_canonical_normal_form(workflow_temp)
Expand All @@ -128,29 +130,41 @@ async def compile_wf(request: Request) -> Json:
yaml_tree: YamlTree = YamlTree(StepId(wkflw_name, plugin_ns), workflow_can)

# ========= COMPILE WORKFLOW ================
args.ignore_dir_path = True
if req.get('run_local_env') == 'true':
args.ignore_dir_path = False
else:
args.ignore_dir_path = True
compiler_info: CompilerInfo = compiler.compile_workflow(yaml_tree, args, [], [graph], {}, {}, {}, {},
tools_cwl, True, relative_run_path=True, testing=False)

rose_tree = compiler_info.rose
input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file)
rose_tree = cwl_inline_runtag(args, rose_tree)
rose_tree = remove_entrypoints(args, rose_tree)
# ======== OUTPUT PROCESSING ================
# ========= PROCESS COMPILED OBJECT =========
sub_node_data: NodeData = rose_tree.data
yaml_stem = sub_node_data.name
cwl_tree = sub_node_data.compiled_cwl
yaml_inputs = sub_node_data.workflow_inputs_file
cwl_tree_no_dd = remove_dot_dollar(cwl_tree)
yaml_inputs_no_dd = remove_dot_dollar(yaml_inputs)

# Convert the compiled yaml file to json for labshare Compute.
cwl_tree_run = copy.deepcopy(cwl_tree_no_dd)
cwl_tree_run = copy.deepcopy(cwl_tree)
cwl_tree_run['steps_dict'] = {}
for step in cwl_tree_run['steps']:
node_name = step['id']
step.pop('id', None)
step = {node_name: step}
step_copy = copy.deepcopy(step)
cwl_tree_run['steps_dict'].update(step_copy)

cwl_tree_run.pop('steps', None)
cwl_tree_run['steps'] = cwl_tree_run.pop('steps_dict', None)

compute_workflow: Json = {}
compute_workflow = {
"name": yaml_stem,
"cwlJobInputs": yaml_inputs_no_dd,
"cwlJobInputs": yaml_inputs,
**cwl_tree_run
}
compute_workflow["retval"] = str(0)
Expand Down
1 change: 0 additions & 1 deletion src/sophios/api/pythonapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
"""CLT utilities."""
import logging
from pathlib import Path
import subprocess as sub
from typing import Any, ClassVar, Optional, TypeVar, Union

import cwl_utils.parser as cu_parser
Expand Down
6 changes: 6 additions & 0 deletions src/sophios/api/utils/ict/ict_spec/tools/cwl_ict.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ def requirements(ict_: "ICT", network_access: bool) -> dict:
return reqs


def split_entrypoint_string(enrtypoint: str) -> list[str]:
"""Fix str to list of str for entrypoint/baseCommand"""
list_of_str_entry = enrtypoint.split(' ')
return list_of_str_entry


def clt_dict(ict_: "ICT", network_access: bool) -> dict:
"""Return a dict of a CommandLineTool from an ICT object."""

Expand Down
85 changes: 85 additions & 0 deletions tests/rest_wfb_objects/bbbc_download_wfb.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
{
"state": {
"nodes": [
{
"id": 1,
"x": 259,
"y": 209,
"z": 1,
"name": "BBBC Download",
"expanded": true,
"pluginId": "polusai/bbbc-download-plugin@0.1.0-dev1",
"height": 50,
"width": 250,
"settings": {
"inputs": {
"name": "BBBC001",
"outDir": "bbbcdownload.outDir"
}
},
"internal": false
}
],
"links": [],
"selection": []
},
"plugins": [
{
"name": "polusai/bbbc-download-plugin:0.1.0-dev1",
"version": "0.1.0-dev1",
"title": "BBBC Download",
"description": "Downloads the datasets on the Broad Bioimage Benchmark Collection website",
"createdBy": "Serebryakov, Artem (NIH/NCATS) [C]",
"updatedBy": "Serebryakov, Artem (NIH/NCATS) [C]",
"author": [
"Hamdah Abbasi"
],
"contact": "hamdahshafqat.abbasi@nih.gov",
"container": "polusai/bbbc-download-plugin:0.1.0-dev1",
"entrypoint": "",
"inputs": [
{
"description": "The name of the dataset(s) to be downloaded (separate the datasets with a comma. eg BBBC001,BBBC002,BBBC003)",
"format": [
"collection"
],
"name": "name",
"required": true,
"type": "string"
}
],
"outputs": [
{
"description": "Output collection",
"name": "outDir",
"format": [
"directory"
],
"required": true,
"type": "path"
}
],
"repository": "https://github.com/LabShare/polus-plugins",
"specVersion": "1.0.0",
"ui": [
{
"description": "The name of the dataset(s) to be downloaded (separate the datasets with a comma. eg BBBC001,BBBC002,BBBC003)",
"format": [
"collection"
],
"name": "name",
"required": true,
"type": "string"
}
],
"path": "visualization",
"tags": [
"bbbc-download"
],
"createdAt": "2024-10-29T18:59:55.843Z",
"updatedAt": "2024-10-29T20:01:42.463Z",
"id": "672130abad801ccee7f5eaad",
"pid": "polusai/bbbc-download-plugin@0.1.0-dev1"
}
]
}
74 changes: 74 additions & 0 deletions tests/rest_wfb_objects/single_node_bbbc_download.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
{
"state": {
"nodes": [
{
"id": 1,
"name": "bbbcdownload",
"pluginId": "",
"run": {
"baseCommand": [],
"class": "CommandLineTool",
"cwlVersion": "v1.2",
"inputs": {
"name": {
"label": "The name of the dataset(s) to be downloaded (separate the datasets with a comma. eg BBBC001,BBBC002,BBBC003)",
"doc": "The name of the dataset(s) to be downloaded (separate the datasets with a comma. eg BBBC001,BBBC002,BBBC003)",
"inputBinding": {
"prefix": "--name"
},
"type": "string"
},
"outDir": {
"label": "Output collection",
"doc": "Output collection",
"inputBinding": {
"prefix": "--outDir"
},
"type": "Directory"
}
},
"outputs": {
"outDir": {
"label": "Output collection",
"doc": "Output collection",
"type": "Directory",
"outputBinding": {
"glob": "$(inputs.outDir.basename)"
}
}
},
"stdout": "output",
"requirements": {
"DockerRequirement": {
"dockerPull": "polusai/bbbc-download-plugin:0.1.0-dev1"
},
"InitialWorkDirRequirement": {
"listing": [
{
"entry": "$(inputs.outDir)",
"writable": true
}
]
},
"InlineJavascriptRequirement": {},
"NetworkAccess": {
"networkAccess": true
}
}
},
"settings": {
"inputs": {
"name": "BBBC001",
"outDir": "bbbcdownload.outDir"
},
"outputs": {
"outDir": "bbbcdownload.outDir"
}
},
"internal": false
}
],
"links": []
},
"plugins": []
}
Loading

0 comments on commit 61abb94

Please sign in to comment.