From 04e5fa8911dea574c6431d180d292a88048bac34 Mon Sep 17 00:00:00 2001 From: Vasu Jaganath Date: Wed, 30 Oct 2024 18:01:35 -0400 Subject: [PATCH 1/2] fix cwl_inline_runtag(name) and post_compile functions --- src/sophios/api/http/restapi.py | 4 ++-- src/sophios/api/pythonapi.py | 2 +- src/sophios/cli.py | 2 +- src/sophios/main.py | 5 +++-- src/sophios/plugins.py | 6 +++--- src/sophios/post_compile.py | 10 +++++----- tests/test_examples.py | 13 +++---------- 7 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/sophios/api/http/restapi.py b/src/sophios/api/http/restapi.py index 9bdb6c8e..5adba7a1 100644 --- a/src/sophios/api/http/restapi.py +++ b/src/sophios/api/http/restapi.py @@ -101,7 +101,7 @@ async def compile_wf(request: Request) -> Json: # schema non-preserving workflow_temp = converter.wfb_to_wic(wfb_payload) wkflw_name = "generic_workflow" - args = get_args(wkflw_name, ['--inline_cwl_runtag', '--generate_cwl_workflow']) + args = get_args(wkflw_name, ['--cwl_inline_runtag', '--generate_cwl_workflow']) # Build canonical workflow object workflow_can = utils_cwl.desugar_into_canonical_normal_form(workflow_temp) @@ -134,7 +134,7 @@ async def compile_wf(request: Request) -> Json: rose_tree = compiler_info.rose input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) - cwl_inline_runtag(args, rose_tree) + rose_tree = cwl_inline_runtag(args, rose_tree) # ======== OUTPUT PROCESSING ================ # ========= PROCESS COMPILED OBJECT ========= sub_node_data: NodeData = rose_tree.data diff --git a/src/sophios/api/pythonapi.py b/src/sophios/api/pythonapi.py index a81a8bd9..2d0f8665 100644 --- a/src/sophios/api/pythonapi.py +++ b/src/sophios/api/pythonapi.py @@ -757,7 +757,7 @@ def run(self) -> None: rose_tree: RoseTree = compiler_info.rose post_compile.cwl_docker_extract(args, self.process_name) - post_compile.remove_entrypoints(args, rose_tree) + rose_tree = post_compile.remove_entrypoints(args, rose_tree) # Do NOT capture stdout and/or stderr and pipe warnings and errors into a black hole. retval = run_local_module.run_local(args, rose_tree, args.cachedir, args.cwl_runner, True) diff --git a/src/sophios/cli.py b/src/sophios/cli.py index 1f9fa8fe..45a3bb81 100644 --- a/src/sophios/cli.py +++ b/src/sophios/cli.py @@ -39,7 +39,7 @@ (You should only disable provenance if absolutely necessary.)''') parser.add_argument('--copy_output_files', default=False, action="store_true", help='Copies output files from the cachedir to outdir/ (automatically enabled with --run_local)') -parser.add_argument('--inline_cwl_runtag', default=False, action="store_true", +parser.add_argument('--cwl_inline_runtag', default=False, action="store_true", help='Copies cwl adapter file contents inline into the final .cwl in autogenerated/') # This is a hidden flag parser.add_argument('--ignore_dir_path', type=bool, diff --git a/src/sophios/main.py b/src/sophios/main.py index 63a17b69..14ec0707 100644 --- a/src/sophios/main.py +++ b/src/sophios/main.py @@ -171,7 +171,7 @@ def main() -> None: io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) - pc.cwl_inline_runtag(args, rose_tree) + rose_tree = pc.cwl_inline_runtag(args, rose_tree) io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) if args.graphviz: @@ -194,7 +194,8 @@ def main() -> None: if args.run_local or args.generate_run_script: pc.cwl_docker_extract(args, yaml_stem) - pc.remove_entrypoints(args, rose_tree) + rose_tree = pc.remove_entrypoints(args, rose_tree) + io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) run_local.run_local(args, rose_tree, args.cachedir, args.cwl_runner, False) diff --git a/src/sophios/plugins.py b/src/sophios/plugins.py index 91a33c46..a5015368 100644 --- a/src/sophios/plugins.py +++ b/src/sophios/plugins.py @@ -279,13 +279,13 @@ def cwl_update_inline_runtag_rosetree(rose_tree: RoseTree, path: Path, relative_ """ n_d: NodeData = rose_tree.data if n_d.compiled_cwl['class'] == 'Workflow': - outputs_inline_cwl_runtag = cwl_update_inline_runtag(n_d.compiled_cwl, path, relative_run_path) + outputs_cwl_inline_runtag = cwl_update_inline_runtag(n_d.compiled_cwl, path, relative_run_path) else: - outputs_inline_cwl_runtag = n_d.compiled_cwl + outputs_cwl_inline_runtag = n_d.compiled_cwl sub_trees_path = [cwl_update_inline_runtag_rosetree(sub_rose_tree, path, relative_run_path) for sub_rose_tree in rose_tree.sub_trees] - node_data_path = NodeData(n_d.namespaces, n_d.name, n_d.yml, outputs_inline_cwl_runtag, n_d.tool, + node_data_path = NodeData(n_d.namespaces, n_d.name, n_d.yml, outputs_cwl_inline_runtag, n_d.tool, n_d.workflow_inputs_file, n_d.explicit_edge_defs, n_d.explicit_edge_calls, n_d.graph, n_d.inputs_workflow, n_d.step_name_1) return RoseTree(node_data_path, sub_trees_path) diff --git a/src/sophios/post_compile.py b/src/sophios/post_compile.py index 40d61c44..e5bb126d 100644 --- a/src/sophios/post_compile.py +++ b/src/sophios/post_compile.py @@ -3,7 +3,6 @@ import subprocess as sub from . import plugins -from . import input_output as io from .wic_types import RoseTree @@ -22,15 +21,16 @@ def cwl_docker_extract(args: argparse.Namespace, file_name: str) -> None: sub.run(cmd, check=True) -def cwl_inline_runtag(args: argparse.Namespace, rose_tree: RoseTree) -> None: +def cwl_inline_runtag(args: argparse.Namespace, rose_tree: RoseTree) -> RoseTree: """Transform with cwl inline runtag""" # this has to happen after at least one write # so we can copy from local cwl_dapters in autogenerated/ - if args.inline_cwl_runtag: + if args.cwl_inline_runtag: rose_tree = plugins.cwl_update_inline_runtag_rosetree(rose_tree, Path('autogenerated/'), True) + return rose_tree -def remove_entrypoints(args: argparse.Namespace, rose_tree: RoseTree) -> None: +def remove_entrypoints(args: argparse.Namespace, rose_tree: RoseTree) -> RoseTree: """Remove entry points""" if args.docker_remove_entrypoints: # Requires root, so guard behind CLI option @@ -40,4 +40,4 @@ def remove_entrypoints(args: argparse.Namespace, rose_tree: RoseTree) -> None: plugins.remove_entrypoints_podman() rose_tree = plugins.dockerPull_append_noentrypoint_rosetree(rose_tree) - io.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) + return rose_tree diff --git a/tests/test_examples.py b/tests/test_examples.py index a4322ac8..7603a2ca 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -22,7 +22,7 @@ from sophios import auto_gen_header from sophios.cli import get_args from sophios.utils_yaml import wic_loader -from sophios.post_compile import cwl_docker_extract +from sophios.post_compile import cwl_docker_extract, remove_entrypoints from sophios.wic_types import NodeData, StepId, Yaml, YamlTree, Json from sophios.utils_graphs import get_graph_reps @@ -216,15 +216,8 @@ def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str, args: argp cwl_docker_extract(args, Path(yml_path).stem) return - if args.docker_remove_entrypoints: - # Requires root, so guard behind CLI option - if args.container_engine == 'docker': - sophios.plugins.remove_entrypoints_docker() - if args.container_engine == 'podman': - sophios.plugins.remove_entrypoints_podman() - - rose_tree = sophios.plugins.dockerPull_append_noentrypoint_rosetree(rose_tree) - sophios.input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) + rose_tree = remove_entrypoints(args, rose_tree) + sophios.input_output.write_to_disk(rose_tree, Path('autogenerated/'), True, args.inputs_file) if args.partial_failure_enable: rose_tree = sophios.plugins.cwl_update_outputs_optional_rosetree(rose_tree) From 4110ae81c28f9ec4d3d178d1620a3a4b8aa81ae1 Mon Sep 17 00:00:00 2001 From: Vasu Jaganath Date: Wed, 30 Oct 2024 22:11:00 -0400 Subject: [PATCH 2/2] fix ict->clt io_type when specified directory --- src/sophios/api/utils/ict/ict_spec/io/objects.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sophios/api/utils/ict/ict_spec/io/objects.py b/src/sophios/api/utils/ict/ict_spec/io/objects.py index 4f4f92d8..5dd8e1fd 100644 --- a/src/sophios/api/utils/ict/ict_spec/io/objects.py +++ b/src/sophios/api/utils/ict/ict_spec/io/objects.py @@ -128,6 +128,12 @@ def _output_to_cwl(self, inputs: Any) -> dict: == "file" # pylint: disable=unsubscriptable-object ): cwl_type = "File" + elif ( + isinstance(self.io_format, list) + and len(self.io_format) == 1 + and self.io_format[0].lower() == 'directory' + ): + cwl_type = "Directory" else: cwl_type = "File"