me-model sub-workflow

doc/source/registry.rst

@@ -4,6 +4,12 @@
 Registry
 ========
 
+memodel|neurons_memodel|v3
+**************************
+
+.. image:: generated/memodel__neurons_memodel__v3.svg
+
+
 connectome_filtering|synapses|v2
 ********************************
 

src/blue_cwl/core/exceptions.py

@@ -13,3 +13,7 @@ class ReferenceResolutionError(CWLError):
 
 class CWLValidationError(CWLError):
     """CWL Validation error."""
+
+
+class InputConcretizationError(CWLError):
+    """Input concretization error."""

src/blue_cwl/core/process.py

@@ -8,7 +8,7 @@
 import blue_cwl.core.cwl
 from blue_cwl.core.common import CustomBaseModel
 from blue_cwl.core.cwl_types import Directory, File, NexusResource
-from blue_cwl.core.exceptions import CWLError
+from blue_cwl.core.exceptions import CWLError, InputConcretizationError, ReferenceResolutionError
 from blue_cwl.core.executor import Executor, LocalExecutor
 from blue_cwl.core.resolve import resolve_parameter_references
 from blue_cwl.core.types import EnvVarDict, InputValue, InputValueObject, OutputValueObject
@@ -113,7 +113,12 @@ def build_command_line_tool_process(
     """
     L.debug("CommandLineTool input values: %s", input_values)
 
-    concretized_inputs: dict[str, InputValueObject] = _concretize_inputs(tool.inputs, input_values)
+    try:
+        concretized_inputs: dict[str, InputValueObject] = _concretize_inputs(
+            tool.inputs, input_values
+        )
+    except Exception as e:
+        raise CWLError(f"CommandLineTool'{tool.id}' inputs concretization failed.\n") from e
     L.debug("Concretized CommandLineTool inputs: %s", concretized_inputs)
 
     concretized_outputs: dict[str, OutputValueObject] = _concretize_tool_outputs(
@@ -153,7 +158,7 @@ def _concretize_inputs(
             concretized_inputs[name] = _input_value_to_object(inp.type, value)
 
     if not set(inputs).issubset(set(concretized_inputs)):
-        raise CWLError(
+        raise InputConcretizationError(
             "Concretized input values are not consistent with the input template.\n"
             f"Expected: {sorted(inputs.keys())}.\n"
             f"Got     : {sorted(concretized_inputs.keys())}"
@@ -321,14 +326,32 @@ def _get_source_value(source, input_objects, sources):
     # The valueFrom fields are evaluated after the the source fields.
     for name, inp in step.inputs.items():
         if inp.valueFrom:
-            step_input_values[name] = resolve_parameter_references(
-                expression=inp.valueFrom,
-                inputs=step_input_values,
-                context=step_sources[name],
-                runtime={},
-            )
-
-    step_process = build_command_line_tool_process(step.run, step_input_values)
+            try:
+                step_input_values[name] = resolve_parameter_references(
+                    expression=inp.valueFrom,
+                    inputs=step_input_values,
+                    context=step_sources[name],
+                    runtime={},
+                )
+            except ReferenceResolutionError as e:
+                raise CWLError(
+                    f"Workflow step '{step_name}' parameter reference resolution failed.\n"
+                    f"Input    : {name}\n"
+                    f"valueFrom: {inp.valueFrom}\n"
+                    f"self type: {type(step_sources[name])}"
+                ) from e
+
+    try:
+        step_process = build_command_line_tool_process(step.run, step_input_values)
+    except InputConcretizationError as e:
+        raise CWLError(
+            f"Workflow step '{step_name}' process build failed.\n"
+            f"Workflow step input values     : {sorted(step_input_values.keys())}\n"
+            f"CommandLineTool expected inputs: {sorted(step.run.inputs.keys())}"
+        ) from e
+    except Exception as e:
+        raise CWLError(f"Workflow step '{step_name}' process build failed.\n") from e
+
     return step_process
 
 

src/blue_cwl/generators/memodel/neurons_memodel/v3/definition.cwl

@@ -0,0 +1,134 @@
+cwlVersion: v1.2
+class: Workflow
+
+id: workflow_me_model
+label: workflow-me-model
+
+inputs:
+
+  - id: configuration_id
+    type: NexusType
+
+  - id: circuit_id
+    type: NexusType
+
+  - id: output_dir
+    type: Directory
+
+outputs:
+
+  - id: circuit
+    type: NexusType
+    outputSource: register/circuit
+
+steps:
+
+  - id: setup
+    in:
+      output_dir: output_dir
+    out:
+      - stage_dir
+      - build_dir
+    run: ./setup.cwl
+
+  - id: stage
+    in:
+      stage_dir: setup/stage_dir
+      circuit_id: circuit_id
+      configuration_id: configuration_id
+    out:
+      - config_file
+      - circuit_file
+      - nodes_file
+      - morphologies_dir
+    run: ./stage.cwl
+
+  - id: recipe
+    run: ./recipe.cwl
+    in:
+      config_file: stage/config_file
+      output_file:
+        source: setup/build_dir
+        valueFrom: $(self.path)/recipe.json
+    out:
+      - recipe_file
+
+  - id: emodel_prepare
+    run: ./emodel_prepare.cwl
+    in:
+      config_file: recipe/recipe_file
+      work_dir:
+        source: setup/build_dir
+        valueFrom: $(self.path)/configs
+      out_mechanisms_dir:
+        source: setup/build_dir
+        valueFrom: $(self.path)/mechanisms
+    out:
+      - work_dir
+      - mechanisms_dir
+
+  - id: emodel_assign
+    run: ./emodel_assign.cwl
+    in:
+      nodes_file: stage/nodes_file
+      recipe_file: recipe/recipe_file
+      out_nodes_file:
+        source: setup/build_dir
+        valueFrom: $(self.path)/assign_nodes.h5
+      out_configs_dir:
+        source: setup/build_dir
+        valueFrom: $(self.path)/configs
+    out:
+      - nodes_file
+      - configs_dir
+
+  - id: emodel_adapt
+    run: ./emodel_adapt.cwl
+    in:
+      nodes_file: emodel_assign/nodes_file
+      recipe_file: recipe/recipe_file
+      configs_dir: emodel_assign/configs_dir
+      mechanisms_dir: emodel_prepare/mechanisms_dir
+      morphologies_dir: stage/morphologies_dir
+      work_dir:
+        source: setup/build_dir
+        valueFrom: $(self.path)/adapt_workdir
+      out_nodes_file:
+        source: setup/build_dir
+        valueFrom: $(self.path)/adapt_nodes.h5
+      out_hoc_dir:
+        source: setup/build_dir
+        valueFrom: $(self.path)/hoc
+      parallel_lib:
+        valueFrom: multiprocessing
+    out:
+      - hoc_dir
+      - nodes_file
+
+  - id: emodel_currents
+    run: ./emodel_currents.cwl
+    in:
+      hoc_dir: emodel_adapt/hoc_dir
+      recipe_file: recipe/recipe_file
+      circuit_file: stage/circuit_file
+      nodes_file: emodel_adapt/nodes_file
+      morphologies_dir: stage/morphologies_dir
+      mechanisms_dir: emodel_prepare/mechanisms_dir
+      out_nodes_file:
+        source: setup/build_dir
+        valueFrom: $(self.path)/nodes.h5
+      parallel_lib:
+        valueFrom: multiprocessing
+    out:
+      - nodes_file
+
+  - id: register
+    run: ./register.cwl
+    in:
+      circuit_id: circuit_id
+      circuit_file: stage/circuit_file
+      hoc_dir: emodel_adapt/hoc_dir
+      nodes_file: emodel_currents/nodes_file
+      output_dir: output_dir
+    out:
+      - circuit

src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_adapt.cwl

@@ -0,0 +1,87 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+id: emodel_adapt
+label: emodel-adapt
+
+environment:
+  env_type: MODULE
+  modules:
+    - unstable
+    - py-emodel-generalisation
+
+executor:
+  type: slurm
+  slurm_config:
+    partition: prod
+    account: proj134
+    exclusive: true
+    time: '11:00:00'
+    constraint: nvme
+    nodes: 1
+    mem: 0
+  remote_config:
+    host: bbpv1.epfl.ch
+  env_vars:
+    NEURON_MODULE_OPTIONS: --nogui
+
+baseCommand: ["emodel-generalisation", "-v", "--no-progress", "adapt", "--no-reuse"]
+
+inputs:
+
+  - id: nodes_file
+    type: File
+    inputBinding:
+      prefix: --input-node-path
+
+  - id: out_nodes_file
+    type: string
+    inputBinding:
+      prefix: --output-node-path
+
+  - id: morphologies_dir
+    type: Directory
+    inputBinding:
+      prefix: --morphology-path
+
+  - id: mechanisms_dir
+    type: Directory
+    inputBinding:
+      prefix: --mech-path
+
+  - id: recipe_file
+    type: File
+    inputBinding:
+      prefix: --config-path
+
+  - id: out_hoc_dir
+    type: string
+    inputBinding:
+      prefix: --output-hoc-path
+
+  - id: configs_dir
+    type: Directory
+    inputBinding:
+      prefix: --local-config-path
+
+  - id: work_dir
+    type: string
+    inputBinding:
+      prefix: --local-dir
+
+  - id: parallel_lib
+    type: string
+    inputBinding:
+      prefix: --parallel-lib
+
+outputs:
+
+  - id: nodes_file
+    type: File
+    outputBinding:
+      glob: $(inputs.out_nodes_file)
+
+  - id: hoc_dir
+    type: Directory
+    outputBinding:
+      glob: $(inputs.out_hoc_dir)

src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_assign.cwl

@@ -0,0 +1,61 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+id: emodel_assign
+label: emodel-assign
+
+environment:
+  env_type: MODULE
+  modules:
+    - unstable
+    - py-emodel-generalisation
+
+executor:
+  type: slurm
+  slurm_config:
+    partition: prod
+    account: proj134
+    exclusive: true
+    time: '01:00:00'
+    nodes: 1
+    mem: 0
+  remote_config:
+    host: bbpv1.epfl.ch
+  env_vars:
+    NEURON_MODULE_OPTIONS: --nogui
+
+baseCommand: ["emodel-generalisation", "-v", "--no-progress", "assign"]
+
+inputs:
+
+  - id: nodes_file
+    type: File
+    inputBinding:
+      prefix: --input-node-path
+
+  - id: recipe_file
+    type: File
+    inputBinding:
+      prefix: --config-path
+
+  - id: out_nodes_file
+    type: string
+    inputBinding:
+      prefix: --output-node-path
+
+  - id: out_configs_dir
+    type: string
+    inputBinding:
+      prefix: --local-config-path
+
+outputs:
+
+  - id: nodes_file
+    type: File
+    outputBinding:
+      glob: $(inputs.out_nodes_file)
+
+  - id: configs_dir
+    type: Directory
+    outputBinding:
+      glob: $(inputs.out_configs_dir)