Merge pull request #40 from ResearchObject/cwl_utils_newer

Unpin & upgrade cwl-utils version

.github/dependabot.yml

@@ -0,0 +1,16 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

CITATION.cff

@@ -7,6 +7,9 @@ authors:
   - family-names: Soiland-Reyes
     given-names: Stian
     orcid: https://orcid.org/0000-0001-9842-9718
+  - family-names: Crusoe
+    given-names: Michael R.
+    orcid: https://orcid.org/0000-0002-2961-9670
 title: "runcrate"
 version: 0.1.2
 doi: 10.5281/zenodo.7762627

cwl/grepsort/sorttool.cwl

@@ -1,6 +1,10 @@
 class: CommandLineTool
 cwlVersion: v1.2
 
+requirements:
+  - class: ResourceRequirement
+    ramMin: 16
+
 baseCommand: sort
 
 inputs:

requirements.txt

@@ -1,6 +1,6 @@
 bdbag>=1.4.1
 click~=8.1
-cwl-utils==0.13
+cwl-utils>=0.27
 cwlprov==0.1.1
 networkx==2.8
 prov>=1.5.1

setup.cfg

@@ -32,7 +32,7 @@ python_requires=>=3.8, <4
 install_requires=
     bdbag>=1.4.1
     click~=8.1
-    cwl-utils==0.13
+    cwl-utils>=0.27
     cwlprov==0.1.1
     networkx==2.8
     prov>=1.5.1

src/runcrate/convert.py

@@ -1,4 +1,5 @@
 # Copyright 2022-2023 CRS4.
+# Copyright 2023 Michael R. Crusoe
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -95,16 +96,12 @@ def is_structured(cwl_type):
     if is_structured(cwl_p.type):
         properties["multipleValues"] = "True"
     if hasattr(cwl_p, "default"):
-        try:
-            default_type = cwl_p.default["class"]
-        except (TypeError, KeyError):
-            if not is_structured(cwl_p.type) and cwl_p.default is not None:
-                properties["defaultValue"] = str(cwl_p.default)
-        else:
-            if default_type in ("File", "Directory"):
-                default = cwl_p.default.get("location", cwl_p.default.get("path"))
-                if default:
-                    properties["defaultValue"] = default
+        if hasattr(cwl_p.default, "class_") and cwl_p.default.class_ in ("File", "Directory"):
+            default = cwl_p.default.location or cwl_p.default.path
+            if default:
+                properties["defaultValue"] = default
+        elif not is_structured(cwl_p.type) and cwl_p.default is not None:
+            properties["defaultValue"] = str(cwl_p.default)
         # TODO: support more cases
     if getattr(cwl_p.type, "type", None) == "enum":
         properties["valuePattern"] = "|".join(_.rsplit("/", 1)[-1] for _ in cwl_p.type.symbols)
@@ -163,7 +160,7 @@ def get_workflow(wf_path):
         ns = n.pop("$namespaces", {})
         if ns:
             json_wf.setdefault("$namespaces", {}).update(ns)
-    defs = load_document_by_yaml(json_wf, wf_path.absolute().as_uri())
+    defs = load_document_by_yaml(json_wf, wf_path.absolute().as_uri(), load_all=True)
     if not isinstance(defs, list):
         defs = [defs]
     def_map = {}
@@ -362,13 +359,18 @@ def add_tool(self, crate, workflow, cwl_tool):
             properties["@type"] = "SoftwareApplication"
         if hasattr(cwl_tool, "intent") and cwl_tool.intent:
             properties["featureList"] = cwl_tool.intent
+        if hasattr(cwl_tool, "requirements") and cwl_tool.requirements:
+            for req in cwl_tool.requirements:
+                if req.class_ == "ResourceRequirement":
+                    ramMin = req.ramMin
+                    if ramMin:
+                        properties["memoryRequirements"] = f"{int(ramMin)} MiB"
         if hasattr(cwl_tool, "hints") and cwl_tool.hints:
-            hints_map = {_["class"]: _ for _ in cwl_tool.hints}
-            rreq = hints_map.get("ResourceRequirement")
-            if rreq:
-                ramMin = rreq.get("ramMin")
-                if ramMin:
-                    properties["memoryRequirements"] = f"{int(ramMin)} MiB"
+            for req in cwl_tool.hints:
+                if hasattr(req, "class_") and req.class_ == "ResourceRequirement":
+                    ramMin = req.ramMin
+                    if ramMin:
+                        properties["memoryRequirements"] = f"{int(ramMin)} MiB"
         tool = crate.add(ContextEntity(crate, tool_id, properties=properties))
         tool["input"] = self.add_params(crate, cwl_tool.inputs)
         tool["output"] = self.add_params(crate, cwl_tool.outputs)

tests/data/grepsort-run-1/bag-info.txt

@@ -1,6 +1,6 @@
-Bag-Software-Agent: cwltool 3.1.20230425144158
+Bag-Software-Agent: cwltool 3.1.20230513155734
 BagIt-Profile-Identifier: https://w3id.org/ro/bagit/profile
-Bagging-Date: 2023-05-12
+Bagging-Date: 2023-05-26
 External-Description: Research Object of CWL workflow run
-External-Identifier: arcp://uuid,c81049f5-bc5f-4c3a-a62e-75cc370736f7/
+External-Identifier: arcp://uuid,c54a7289-731a-498b-9f08-3cc2bb801ba1/
 Payload-Oxum: 192.4

tests/data/grepsort-run-1/metadata/logs/engine.fb8f0a7a-121b-47e1-b955-75d213864598.txt

@@ -0,0 +1,20 @@
+[2023-05-26T13:48:22,761.874199Z] [cwltool] /home/simleo/git/runcrate/venv/bin/cwltool --provenance grepsort-run-1 grepsort.cwl params.yml
+[2023-05-26T13:48:22,763.775349Z] Resolved 'grepsort.cwl' to 'file:///home/simleo/git/runcrate/cwl/grepsort/grepsort.cwl'
+[2023-05-26T13:48:23,857.358217Z] [provenance] Adding to RO file:///home/simleo/git/runcrate/cwl/grepsort/lines.txt
+[2023-05-26T13:48:23,858.058929Z] [provenance] Adding to RO file:///home/simleo/git/runcrate/cwl/grepsort/lines.aux
+[2023-05-26T13:48:23,860.283136Z] [workflow ] start
+[2023-05-26T13:48:23,860.562325Z] [workflow ] starting step grep
+[2023-05-26T13:48:23,860.731125Z] [step grep] start
+[2023-05-26T13:48:23,862.983942Z] [job grep] /tmp/f9r41on_$ bash \
+    -c \
+    'grep -f /tmp/iueonoei/stga889db5c-8b30-45b5-9826-287bfb970c74/lines.aux /tmp/iueonoei/stga889db5c-8b30-45b5-9826-287bfb970c74/lines.txt >grep_out.txt'
+[2023-05-26T13:48:23,871.237516Z] [job grep] completed success
+[2023-05-26T13:48:23,871.550560Z] [step grep] completed success
+[2023-05-26T13:48:23,872.055531Z] [workflow ] starting step sorted
+[2023-05-26T13:48:23,872.438669Z] [step sorted] start
+[2023-05-26T13:48:23,874.803782Z] [job sorted] /tmp/7lf3aa84$ sort \
+    /tmp/6h7ltdar/stg5551dd85-4192-4d88-a91f-7b2dfbff9740/grep_out.txt > /tmp/7lf3aa84/sort_out.txt
+[2023-05-26T13:48:23,880.517006Z] [job sorted] completed success
+[2023-05-26T13:48:23,880.779028Z] [step sorted] completed success
+[2023-05-26T13:48:23,881.006241Z] [workflow ] completed success
+[2023-05-26T13:48:23,940.695763Z] Final process status is success