From 2d4e2ba4d65ad148a854559cb59c0622debc2f10 Mon Sep 17 00:00:00 2001 From: Zisis Eleftherios Date: Thu, 30 May 2024 17:30:03 +0200 Subject: [PATCH 1/3] me-model sub-workflow --- src/blue_cwl/core/exceptions.py | 4 + src/blue_cwl/core/process.py | 45 ++- .../memodel/neurons_memodel/v3/definition.cwl | 134 +++++++ .../neurons_memodel/v3/emodel_adapt.cwl | 85 +++++ .../neurons_memodel/v3/emodel_assign.cwl | 59 +++ .../neurons_memodel/v3/emodel_currents.cwl | 65 ++++ .../neurons_memodel/v3/emodel_prepare.cwl | 54 +++ .../memodel/neurons_memodel/v3/recipe.cwl | 42 +++ .../memodel/neurons_memodel/v3/register.cwl | 57 +++ .../memodel/neurons_memodel/v3/setup.cwl | 42 +++ .../memodel/neurons_memodel/v3/stage.cwl | 62 ++++ src/blue_cwl/wrappers/common.py | 14 + src/blue_cwl/wrappers/memodel.py | 348 +++++------------- tests/unit/me-model/test_wrapper.py | 153 +------- 14 files changed, 751 insertions(+), 413 deletions(-) create mode 100644 src/blue_cwl/generators/memodel/neurons_memodel/v3/definition.cwl create mode 100644 src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_adapt.cwl create mode 100644 src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_assign.cwl create mode 100644 src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_currents.cwl create mode 100644 src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_prepare.cwl create mode 100644 src/blue_cwl/generators/memodel/neurons_memodel/v3/recipe.cwl create mode 100644 src/blue_cwl/generators/memodel/neurons_memodel/v3/register.cwl create mode 100644 src/blue_cwl/generators/memodel/neurons_memodel/v3/setup.cwl create mode 100644 src/blue_cwl/generators/memodel/neurons_memodel/v3/stage.cwl create mode 100644 src/blue_cwl/wrappers/common.py diff --git a/src/blue_cwl/core/exceptions.py b/src/blue_cwl/core/exceptions.py index 3d0ecc7..958f239 100644 --- a/src/blue_cwl/core/exceptions.py +++ b/src/blue_cwl/core/exceptions.py @@ -13,3 +13,7 @@ class ReferenceResolutionError(CWLError): class CWLValidationError(CWLError): """CWL Validation error.""" + + +class InputConcretizationError(CWLError): + """Input concretization error.""" diff --git a/src/blue_cwl/core/process.py b/src/blue_cwl/core/process.py index e915a3b..bea994a 100644 --- a/src/blue_cwl/core/process.py +++ b/src/blue_cwl/core/process.py @@ -8,7 +8,7 @@ import blue_cwl.core.cwl from blue_cwl.core.common import CustomBaseModel from blue_cwl.core.cwl_types import Directory, File, NexusResource -from blue_cwl.core.exceptions import CWLError +from blue_cwl.core.exceptions import CWLError, InputConcretizationError, ReferenceResolutionError from blue_cwl.core.executor import Executor, LocalExecutor from blue_cwl.core.resolve import resolve_parameter_references from blue_cwl.core.types import EnvVarDict, InputValue, InputValueObject, OutputValueObject @@ -113,7 +113,12 @@ def build_command_line_tool_process( """ L.debug("CommandLineTool input values: %s", input_values) - concretized_inputs: dict[str, InputValueObject] = _concretize_inputs(tool.inputs, input_values) + try: + concretized_inputs: dict[str, InputValueObject] = _concretize_inputs( + tool.inputs, input_values + ) + except Exception as e: + raise CWLError(f"CommandLineTool'{tool.id}' inputs concretization failed.\n") from e L.debug("Concretized CommandLineTool inputs: %s", concretized_inputs) concretized_outputs: dict[str, OutputValueObject] = _concretize_tool_outputs( @@ -153,7 +158,7 @@ def _concretize_inputs( concretized_inputs[name] = _input_value_to_object(inp.type, value) if not set(inputs).issubset(set(concretized_inputs)): - raise CWLError( + raise InputConcretizationError( "Concretized input values are not consistent with the input template.\n" f"Expected: {sorted(inputs.keys())}.\n" f"Got : {sorted(concretized_inputs.keys())}" @@ -321,14 +326,32 @@ def _get_source_value(source, input_objects, sources): # The valueFrom fields are evaluated after the the source fields. for name, inp in step.inputs.items(): if inp.valueFrom: - step_input_values[name] = resolve_parameter_references( - expression=inp.valueFrom, - inputs=step_input_values, - context=step_sources[name], - runtime={}, - ) - - step_process = build_command_line_tool_process(step.run, step_input_values) + try: + step_input_values[name] = resolve_parameter_references( + expression=inp.valueFrom, + inputs=step_input_values, + context=step_sources[name], + runtime={}, + ) + except ReferenceResolutionError as e: + raise CWLError( + f"Workflow step '{step_name}' parameter reference resolution failed.\n" + f"Input : {name}\n" + f"valueFrom: {inp.valueFrom}\n" + f"self type: {type(step_sources[name])}" + ) from e + + try: + step_process = build_command_line_tool_process(step.run, step_input_values) + except InputConcretizationError as e: + raise CWLError( + f"Workflow step '{step_name}' process build failed.\n" + f"Workflow step input values : {sorted(step_input_values.keys())}\n" + f"CommandLineTool expected inputs: {sorted(step.run.inputs.keys())}" + ) from e + except Exception as e: + raise CWLError(f"Workflow step '{step_name}' process build failed.\n") from e + return step_process diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/definition.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/definition.cwl new file mode 100644 index 0000000..e17eaa6 --- /dev/null +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/definition.cwl @@ -0,0 +1,134 @@ +cwlVersion: v1.2 +class: Workflow + +id: workflow_me_model +label: workflow-me-model + +inputs: + + - id: configuration_id + type: NexusType + + - id: circuit_id + type: NexusType + + - id: output_dir + type: Directory + +outputs: + + - id: circuit + type: NexusType + outputSource: register/circuit + +steps: + + - id: setup + in: + output_dir: output_dir + out: + - stage_dir + - build_dir + run: ./setup.cwl + + - id: stage + in: + stage_dir: setup/stage_dir + circuit_id: circuit_id + configuration_id: configuration_id + out: + - config_file + - circuit_file + - nodes_file + - morphologies_dir + run: ./stage.cwl + + - id: recipe + run: ./recipe.cwl + in: + config_file: stage/config_file + output_file: + source: setup/build_dir + valueFrom: $(self.path)/recipe.json + out: + - recipe_file + + - id: emodel_prepare + run: ./emodel_prepare.cwl + in: + config_file: recipe/recipe_file + work_dir: + source: setup/build_dir + valueFrom: $(self.path)/configs + out_mechanisms_dir: + source: setup/build_dir + valueFrom: $(self.path)/mechanisms + out: + - work_dir + - mechanisms_dir + + - id: emodel_assign + run: ./emodel_assign.cwl + in: + nodes_file: stage/nodes_file + recipe_file: recipe/recipe_file + out_nodes_file: + source: setup/build_dir + valueFrom: $(self.path)/assign_nodes.h5 + out_configs_dir: + source: setup/build_dir + valueFrom: $(self.path)/configs + out: + - nodes_file + - configs_dir + + - id: emodel_adapt + run: ./emodel_adapt.cwl + in: + nodes_file: emodel_assign/nodes_file + recipe_file: recipe/recipe_file + configs_dir: emodel_assign/configs_dir + mechanisms_dir: emodel_prepare/mechanisms_dir + morphologies_dir: stage/morphologies_dir + work_dir: + source: setup/build_dir + valueFrom: $(self.path)/adapt_workdir + out_nodes_file: + source: setup/build_dir + valueFrom: $(self.path)/adapt_nodes.h5 + out_hoc_dir: + source: setup/build_dir + valueFrom: $(self.path)/hoc + parallel_lib: + valueFrom: multiprocessing + out: + - hoc_dir + - nodes_file + + - id: emodel_currents + run: ./emodel_currents.cwl + in: + hoc_dir: emodel_adapt/hoc_dir + recipe_file: recipe/recipe_file + circuit_file: stage/circuit_file + nodes_file: emodel_adapt/nodes_file + morphologies_dir: stage/morphologies_dir + mechanisms_dir: emodel_prepare/mechanisms_dir + out_nodes_file: + source: setup/build_dir + valueFrom: $(self.path)/nodes.h5 + parallel_lib: + valueFrom: multiprocessing + out: + - nodes_file + + - id: register + run: ./register.cwl + in: + circuit_id: circuit_id + circuit_file: stage/circuit_file + hoc_dir: emodel_adapt/hoc_dir + nodes_file: emodel_currents/nodes_file + output_dir: output_dir + out: + - circuit diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_adapt.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_adapt.cwl new file mode 100644 index 0000000..5aee21f --- /dev/null +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_adapt.cwl @@ -0,0 +1,85 @@ +cwlVersion: v1.2 +class: CommandLineTool + +id: emodel_adapt +label: emodel-adapt + +environment: + env_type: VENV + path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + +executor: + type: slurm + slurm_config: + partition: prod + account: proj134 + exclusive: true + time: '11:00:00' + constraint: nvme + nodes: 1 + mem: 0 + remote_config: + host: bbpv1.epfl.ch + env_vars: + NEURON_MODULE_OPTIONS: --nogui + +baseCommand: ["emodel-generalisation", "-v", "--no-progress", "adapt", "--no-reuse"] + +inputs: + + - id: nodes_file + type: File + inputBinding: + prefix: --input-node-path + + - id: out_nodes_file + type: string + inputBinding: + prefix: --output-node-path + + - id: morphologies_dir + type: Directory + inputBinding: + prefix: --morphology-path + + - id: mechanisms_dir + type: Directory + inputBinding: + prefix: --mech-path + + - id: recipe_file + type: File + inputBinding: + prefix: --config-path + + - id: out_hoc_dir + type: string + inputBinding: + prefix: --output-hoc-path + + - id: configs_dir + type: Directory + inputBinding: + prefix: --local-config-path + + - id: work_dir + type: string + inputBinding: + prefix: --local-dir + + - id: parallel_lib + type: string + inputBinding: + prefix: --parallel-lib + +outputs: + + - id: nodes_file + type: File + outputBinding: + glob: $(inputs.out_nodes_file) + + - id: hoc_dir + type: Directory + outputBinding: + glob: $(inputs.out_hoc_dir) diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_assign.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_assign.cwl new file mode 100644 index 0000000..842ffcc --- /dev/null +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_assign.cwl @@ -0,0 +1,59 @@ +cwlVersion: v1.2 +class: CommandLineTool + +id: emodel_assign +label: emodel-assign + +environment: + env_type: VENV + path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + +executor: + type: slurm + slurm_config: + partition: prod + account: proj134 + exclusive: true + time: '01:00:00' + nodes: 1 + mem: 0 + remote_config: + host: bbpv1.epfl.ch + env_vars: + NEURON_MODULE_OPTIONS: --nogui + +baseCommand: ["emodel-generalisation", "-v", "--no-progress", "assign"] + +inputs: + + - id: nodes_file + type: File + inputBinding: + prefix: --input-node-path + + - id: recipe_file + type: File + inputBinding: + prefix: --config-path + + - id: out_nodes_file + type: string + inputBinding: + prefix: --output-node-path + + - id: out_configs_dir + type: string + inputBinding: + prefix: --local-config-path + +outputs: + + - id: nodes_file + type: File + outputBinding: + glob: $(inputs.out_nodes_file) + + - id: configs_dir + type: Directory + outputBinding: + glob: $(inputs.out_configs_dir) diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_currents.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_currents.cwl new file mode 100644 index 0000000..ca39d7e --- /dev/null +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_currents.cwl @@ -0,0 +1,65 @@ +cwlVersion: v1.2 +class: CommandLineTool + +id: emodel_currents +label: emodel-currents + +environment: + env_type: VENV + path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + +executor: + type: slurm + slurm_config: + partition: prod + account: proj134 + exclusive: true + time: '12:00:00' + constraint: nvme + nodes: 1 + mem: 0 + remote_config: + host: bbpv1.epfl.ch + env_vars: + NEURON_MODULE_OPTIONS: --nogui + +baseCommand: ["emodel-generalisation", "-v", "--no-progress", "compute_currents"] + +inputs: + + - id: nodes_file + type: File + inputBinding: + prefix: --input-path + + - id: out_nodes_file + type: string + inputBinding: + prefix: --output-path + + - id: morphologies_dir + type: Directory + inputBinding: + prefix: --morphology-path + + - id: mechanisms_dir + type: Directory + inputBinding: + prefix: --mech-path + + - id: hoc_dir + type: Directory + inputBinding: + prefix: --hoc-path + + - id: parallel_lib + type: string + inputBinding: + prefix: --parallel-lib + +outputs: + + - id: nodes_file + type: File + outputBinding: + glob: $(inputs.out_nodes_file) diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_prepare.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_prepare.cwl new file mode 100644 index 0000000..8739210 --- /dev/null +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_prepare.cwl @@ -0,0 +1,54 @@ +cwlVersion: v1.2 +class: CommandLineTool + +id: emodel_prepare +label: emodel-prepare + +environment: + env_type: VENV + path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + +executor: + type: slurm + slurm_config: + partition: prod + account: proj134 + exclusive: true + time: '00:10:00' + ntasks: 1 + mem: 0 + remote_config: + host: bbpv1.epfl.ch + env_vars: + NEURON_MODULE_OPTIONS: --nogui + +baseCommand: ["emodel-generalisation", "-v", "prepare"] + +inputs: + + - id: config_file + type: File + inputBinding: + prefix: --config-path + + - id: work_dir + type: string + inputBinding: + prefix: --local-config-path + + - id: out_mechanisms_dir + type: string + inputBinding: + prefix: --mechanisms-path + +outputs: + + - id: work_dir + type: Directory + outputBinding: + glob: $(inputs.work_dir) + + - id: mechanisms_dir + type: Directory + outputBinding: + glob: $(inputs.out_mechanisms_dir) diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/recipe.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/recipe.cwl new file mode 100644 index 0000000..6012c34 --- /dev/null +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/recipe.cwl @@ -0,0 +1,42 @@ +cwlVersion: v1.2 +class: CommandLineTool + +id: emogel_recipe +label: emodel-recipe + +environment: + env_type: VENV + path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + +executor: + type: slurm + slurm_config: + partition: prod + account: proj134 + exclusive: true + time: '01:00:00' + nodes: 1 + mem: 0 + remote_config: + host: bbpv1.epfl.ch + +baseCommand: ["blue-cwl", "execute", "me-model", "recipe"] + +inputs: + + - id: config_file + type: File + inputBinding: + prefix: --config-file + + - id: output_file + type: string + inputBinding: + prefix: --output-file + +outputs: + + - id: recipe_file + type: File + outputBinding: + glob: $(inputs.output_file) diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/register.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/register.cwl new file mode 100644 index 0000000..6628786 --- /dev/null +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/register.cwl @@ -0,0 +1,57 @@ +cwlVersion: v1.2 +class: CommandLineTool + +id: me-model-register +label: me-model-register + +environment: + env_type: VENV + path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + +executor: + type: slurm + slurm_config: + partition: prod + account: proj134 + exclusive: true + time: '1:00:00' + nodes: 1 + mem: 0 + remote_config: + host: bbpv1.epfl.ch + +baseCommand: ["blue-cwl", "execute", "me-model", "register"] + +inputs: + + - id: circuit_id + type: NexusType + inputBinding: + prefix: --circuit-id + + - id: circuit_file + type: File + inputBinding: + prefix: --circuit-file + + - id: nodes_file + type: File + inputBinding: + prefix: --nodes-file + + - id: hoc_dir + type: Directory + inputBinding: + prefix: --hoc-dir + + - id: output_dir + type: Directory + inputBinding: + prefix: --output-dir + +outputs: + + - id: circuit + type: NexusType + outputBinding: + glob: $(inputs.output_dir.path)/resource.json diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/setup.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/setup.cwl new file mode 100644 index 0000000..0e813a1 --- /dev/null +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/setup.cwl @@ -0,0 +1,42 @@ +cwlVersion: v1.2 +class: CommandLineTool + +id: create-directories +label: create-directories + +environment: + env_type: VENV + path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + +executor: + type: slurm + slurm_config: + partition: prod + account: proj134 + exclusive: true + time: '00:10:00' + ntasks: 1 + mem: 0 + remote_config: + host: bbpv1.epfl.ch + +baseCommand: ["blue-cwl", "execute", "me-model", "setup"] + +inputs: + + - id: output_dir + type: Directory + inputBinding: + prefix: --output-dir + +outputs: + + - id: stage_dir + type: Directory + outputBinding: + glob: $(inputs.output_dir.path)/stage + + - id: build_dir + type: Directory + outputBinding: + glob: $(inputs.output_dir.path)/build diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/stage.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/stage.cwl new file mode 100644 index 0000000..b92f85c --- /dev/null +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/stage.cwl @@ -0,0 +1,62 @@ +cwlVersion: v1.2 +class: CommandLineTool + +id: stage_nexus_resources +label: stage-nexus-resources + +environment: + env_type: VENV + path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + +executor: + type: slurm + slurm_config: + partition: prod + account: proj134 + exclusive: true + time: '00:10:00' + ntasks: 1 + mem: 0 + remote_config: + host: bbpv1.epfl.ch + +baseCommand: ["blue-cwl", "execute", "me-model", "stage"] + +inputs: + + - id: configuration_id + type: NexusType + inputBinding: + prefix: --configuration-id + + - id: circuit_id + type: NexusType + inputBinding: + prefix: --circuit-id + + - id: stage_dir + type: string + inputBinding: + prefix: --stage-dir + +outputs: + + - id: config_file + type: File + outputBinding: + glob: $(inputs.stage_dir)/materialized_me_model_config.json + + - id: circuit_file + type: File + outputBinding: + glob: $(inputs.stage_dir)/circuit_config.json + + - id: morphologies_dir + type: Directory + outputBinding: + glob: $(inputs.stage_dir)/morphologies + + - id: nodes_file + type: File + outputBinding: + glob: $(inputs.stage_dir)/nodes.h5 diff --git a/src/blue_cwl/wrappers/common.py b/src/blue_cwl/wrappers/common.py new file mode 100644 index 0000000..c9f6e4f --- /dev/null +++ b/src/blue_cwl/wrappers/common.py @@ -0,0 +1,14 @@ +"""Wrapper common utils.""" + +from pathlib import Path + +from blue_cwl.typing import StrOrPath +from blue_cwl.utils import create_dir + +_SUB_DIRECTORIES = ("build", "stage", "transform") + + +def setup_directories(output_dir: StrOrPath, sub_directories=_SUB_DIRECTORIES) -> dict[str, Path]: + """Setup directory hierarchy for wrapper output.""" + create_dir(output_dir) + return {dirname: create_dir(Path(output_dir, dirname)) for dirname in sub_directories} diff --git a/src/blue_cwl/wrappers/memodel.py b/src/blue_cwl/wrappers/memodel.py index 27d0448..04eb74c 100644 --- a/src/blue_cwl/wrappers/memodel.py +++ b/src/blue_cwl/wrappers/memodel.py @@ -3,9 +3,7 @@ """me-model wrapper.""" import logging -import os import shutil -import subprocess from pathlib import Path import click @@ -20,7 +18,7 @@ from blue_cwl.nexus import get_distribution_as_dict from blue_cwl.staging import stage_file from blue_cwl.utils import get_partial_circuit_region_id -from blue_cwl.variant import Variant +from blue_cwl.wrappers import common L = logging.getLogger(__name__) @@ -52,284 +50,138 @@ def app(): """me-model app.""" -@app.command() -@click.option("--configuration-id", required=True, help="Configuration id.") -@click.option("--circuit-id", required=True, help="Partial circuit id.") -@click.option("--variant-id", required=True, help="Variant id.") +@app.command(name="setup") @click.option("--output-dir", required=True, help="Output directory path") -def mono_execution(configuration_id, circuit_id, variant_id, output_dir): - """Single execution endpoint.""" - _mono_execution(configuration_id, circuit_id, variant_id, output_dir) +def setup_cli(output_dir): + """Setup wrapper output directories.""" + common.setup_directories(output_dir=output_dir) -def _mono_execution(configuration, partial_circuit, variant_config, output_dir): - output_dir = utils.create_dir(output_dir) +@app.command(name="stage") +@click.option("--configuration-id", required=True, help="me-model configuration id.") +@click.option("--circuit-id", required=True, help="Circuit id.") +@click.option("--stage-dir", required=True, help="Stagind directory") +def stage_cli(**kwargs): + """Stage cli.""" + stage(**kwargs) - variant = get_entity(resource_id=variant_config, cls=Variant) - staging_dir = utils.create_dir(output_dir / "stage") - build_dir = utils.create_dir(output_dir / "build") - configuration_file = staging_dir / "materialized_me_model_config.json" +def stage(*, configuration_id, circuit_id, stage_dir): + """Stage NEXUS resources to local files.""" + utils.create_dir(stage_dir) + config_file = Path(stage_dir, "materialized_me_model_config.json") stage_me_model_config( - dataset=get_distribution_as_dict(configuration, cls=MEModelConfig), - staging_dir=staging_dir / "me-model-config", - output_file=configuration_file, + dataset=get_distribution_as_dict(configuration_id, cls=MEModelConfig), + staging_dir=Path(stage_dir, "me-model-config"), + output_file=config_file, ) + L.debug("me-model config %s materialized at %s", configuration_id, config_file) - circuit_config_file = staging_dir / "circuit_config.json" + circuit_file = Path(stage_dir, "circuit_config.json") _stage_circuit( - partial_circuit=partial_circuit, - output_file=circuit_config_file, + partial_circuit=circuit_id, + output_file=Path(stage_dir, "circuit_config.json"), ) + L.debug("Circuit %s staged at %s", circuit_id, circuit_file) - recipe_file = build_dir / "recipe.json" - _build_recipe(configuration_file=configuration_file, output_file=recipe_file) - - mechanisms_dir = utils.create_dir(build_dir / "mechanisms") - _run_emodel_prepare( - recipe_file=recipe_file, - mechanisms_dir=mechanisms_dir, - variant=variant, - work_dir=build_dir, + nodes_file, _, morphologies_dir = _get_biophysical_population_info( + circuit_config_file=circuit_file, + ext="asc", ) - assign_nodes_file = build_dir / "assign_nodes.h5" - _run_emodel_assign( - circuit_config_file=circuit_config_file, - recipe_file=recipe_file, - output_nodes_file=assign_nodes_file, - work_dir=build_dir, - variant=variant, - ) + staged_morphologies_dir = Path(stage_dir, "morphologies") + stage_file(source=morphologies_dir, target=staged_morphologies_dir) + L.debug("Staged %s -> %s", morphologies_dir, staged_morphologies_dir) - output_biophysical_models_dir = utils.create_dir(build_dir / "hoc") + staged_nodes_file = Path(stage_dir, "nodes.h5") + stage_file(source=nodes_file, target=staged_nodes_file) + L.debug("Staged %s -> %s", nodes_file, staged_nodes_file) - adapt_nodes_file = build_dir / "adapt_nodes.h5" - _run_emodel_adapt( - circuit_config_file=circuit_config_file, - nodes_file=assign_nodes_file, - recipe_file=recipe_file, - output_nodes_file=adapt_nodes_file, - output_biophysical_models_dir=output_biophysical_models_dir, - variant=variant, - work_dir=build_dir, - mechanisms_dir=mechanisms_dir, - ) - output_nodes_file = build_dir / "nodes.h5" - _run_emodel_currents( - circuit_config_file=circuit_config_file, - nodes_file=adapt_nodes_file, - biophysical_neuron_models_dir=output_biophysical_models_dir, - output_nodes_file=output_nodes_file, - variant=variant, - mechanisms_dir=mechanisms_dir, - ) +@app.command(name="recipe") +@click.option("--config-file", required=True, help="me-model configuration file") +@click.option("--output-file", required=True, help="Output recipe file") +def recipe_cli(**kwargs): + """Generate me-model recipe.""" + recipe(**kwargs) - _register( - partial_circuit=partial_circuit, - variant=variant, - circuit_config_file=circuit_config_file, - nodes_file=output_nodes_file, - biophysical_neuron_models_dir=output_biophysical_models_dir, - output_dir=output_dir, - ) +def recipe(*, config_file, output_file): + """Generate me-model recipe.""" + configuration = utils.load_json(config_file) + recipe_config = build_me_model_recipe(me_model_config=configuration) + utils.write_json(data=recipe_config, filepath=output_file) -def _rmdir_if_exists(path: Path) -> Path: - if path.exists(): - shutil.rmtree(path) - return path +@app.command(name="register") +@click.option("--circuit-id", required=True) +@click.option("--circuit-file", required=True) +@click.option("--nodes-file", required=True) +@click.option("--hoc-dir", required=True) +@click.option("--output-dir", required=True) +def register_cli(**kwargs): + """Register cli.""" + register(**kwargs) -def _stage_circuit(partial_circuit, output_file): - entity = get_entity(resource_id=partial_circuit, cls=DetailedCircuit) - stage_file( - source=unquote_uri_path(entity.circuitConfigPath.url), - target=output_file, - symbolic=True, - ) - - -def _build_recipe(configuration_file, output_file): - configuration = utils.load_json(configuration_file) - recipe = build_me_model_recipe(me_model_config=configuration) - utils.write_json(data=recipe, filepath=output_file) - - -def _run_emodel_prepare(recipe_file, mechanisms_dir, variant, work_dir): - local_config_path = _rmdir_if_exists(Path(work_dir, "configs")) - - arglist = [ - "emodel-generalisation", - "-v", - "prepare", - "--config-path", - str(recipe_file), - "--local-config-path", - str(local_config_path), - "--mechanisms-path", - str(mechanisms_dir), - ] - cmd = " ".join(arglist) - cmd = utils.build_variant_allocation_command(cmd, variant, sub_task_index=0) - - env = os.environ | {"NEURON_MODULE_OPTIONS": "-nogui"} - - L.info("Tool command :%s", cmd) - subprocess.run(cmd, check=True, shell=True, env=env) +def register( + *, + circuit_id, + circuit_file, + nodes_file, + hoc_dir, + output_dir, +): + """Register new circuit with generated nodes and hoc directory.""" + config = utils.load_json(circuit_file) + _, population_name = utils.get_biophysical_partial_population_from_config(config) -def _run_emodel_assign(circuit_config_file, recipe_file, output_nodes_file, work_dir, variant): - nodes_file, population_name, _ = _get_biophysical_population_info( - circuit_config_file=circuit_config_file, - ext="asc", - ) - validation.check_properties_in_population(population_name, nodes_file, INPUT_POPULATION_COLUMNS) - - arglist = [ - "emodel-generalisation", - "-v", - "--no-progress", - "assign", - "--input-node-path", - str(nodes_file), - "--config-path", - str(recipe_file), - "--output-node-path", - str(output_nodes_file), - "--local-config-path", - str(Path(work_dir, "configs")), - ] - - cmd = " ".join(arglist) - cmd = utils.build_variant_allocation_command(cmd, variant, sub_task_index=0) - - env = os.environ | {"NEURON_MODULE_OPTIONS": "-nogui"} - - L.info("Tool command :%s", cmd) - subprocess.run(cmd, check=True, shell=True, env=env) - - L.info("Validating generated nodes file...") + L.info("Validating odes file...") validation.check_properties_in_population( population_name=population_name, - nodes_file=output_nodes_file, - property_names=INPUT_POPULATION_COLUMNS + ASSIGN_PROPERTIES, + nodes_file=nodes_file, + property_names=INPUT_POPULATION_COLUMNS + + ASSIGN_PROPERTIES + + ADAPT_PROPERTIES + + CURRENTS_PROPERTIES, ) - -def _run_emodel_adapt( - circuit_config_file, - nodes_file, - recipe_file, - output_nodes_file, - output_biophysical_models_dir, - variant, - work_dir, - mechanisms_dir, -): - _, population_name, morphologies_dir = _get_biophysical_population_info( - circuit_config_file=circuit_config_file, - ext="asc", + updated_config = utils.update_circuit_config_population( + config=config, + population_name=population_name, + population_data={ + "biophysical_neuron_models_dir": str(hoc_dir), + }, + filepath=str(nodes_file), ) + output_circuit_config_file = Path(output_dir, "circuit_config.json") + utils.write_json(filepath=output_circuit_config_file, data=updated_config) + L.info("Created circuit config file at %s", output_circuit_config_file) - local_dir = _rmdir_if_exists(work_dir / "local") - - arglist = [ - "emodel-generalisation", - "-v", - "--no-progress", - "adapt", - "--input-node-path", - str(nodes_file), - "--output-node-path", - str(output_nodes_file), - "--morphology-path", - str(morphologies_dir), - "--config-path", - str(recipe_file), - "--output-hoc-path", - str(output_biophysical_models_dir), - "--parallel-lib", - _parallel_mode(), - "--local-config-path", - str(work_dir / "configs"), - "--local-dir", - str(local_dir), - ] - cmd = " ".join(arglist) - cmd = utils.build_variant_allocation_command(cmd, variant, sub_task_index=1) - - L.info("Tool command :%s", cmd) - - env = os.environ.copy() - env.update( - { - "EMODEL_GENERALISATION_MOD_LIBRARY_PATH": str(mechanisms_dir), - "NEURON_MODULE_OPTIONS": "-nogui", - } - ) - subprocess.run(cmd, check=True, shell=True, env=env) + validation.check_population_name_in_config(population_name, output_circuit_config_file) - L.info("Validating generated nodes file...") - validation.check_properties_in_population( - population_name=population_name, - nodes_file=output_nodes_file, - property_names=INPUT_POPULATION_COLUMNS + ASSIGN_PROPERTIES + ADAPT_PROPERTIES, + circuit = _register_circuit(circuit_id, output_circuit_config_file) + + utils.write_json( + data=load_by_id(circuit.get_id()), + filepath=Path(output_dir, "resource.json"), ) -def _run_emodel_currents( - circuit_config_file, - nodes_file, - biophysical_neuron_models_dir, - output_nodes_file, - variant, - mechanisms_dir, -): - _, population_name, morphologies_dir = _get_biophysical_population_info( - circuit_config_file=circuit_config_file, - ext="asc", - ) - arglist = [ - "emodel-generalisation", - "-v", - "--no-progress", - "compute_currents", - "--input-path", - str(nodes_file), - "--output-path", - str(output_nodes_file), - "--morphology-path", - str(morphologies_dir), - "--hoc-path", - str(biophysical_neuron_models_dir), - "--parallel-lib", - _parallel_mode(), - ] - cmd = " ".join(arglist) - cmd = utils.build_variant_allocation_command(cmd, variant, sub_task_index=2) - - L.info("Tool command :%s", cmd) - env = os.environ.copy() - env.update( - { - "EMODEL_GENERALISATION_MOD_LIBRARY_PATH": str(mechanisms_dir), - "NEURON_MODULE_OPTIONS": "-nogui", - } - ) - subprocess.run(cmd, check=True, shell=True, env=env) +def _rmdir_if_exists(path: Path) -> Path: + if path.exists(): + shutil.rmtree(path) + return path - L.info("Validating generated nodes file...") - validation.check_properties_in_population( - population_name=population_name, - nodes_file=output_nodes_file, - property_names=INPUT_POPULATION_COLUMNS - + ASSIGN_PROPERTIES - + ADAPT_PROPERTIES - + CURRENTS_PROPERTIES, + +def _stage_circuit(partial_circuit, output_file): + entity = get_entity(resource_id=partial_circuit, cls=DetailedCircuit) + stage_file( + source=unquote_uri_path(entity.circuitConfigPath.url), + target=output_file, + symbolic=True, ) @@ -391,9 +243,3 @@ def _register_circuit(partial_circuit, output_circuit_config_file): ) return circuit - - -def _parallel_mode(): - mode = os.getenv("ME_MODEL_MULTIPROCESSING_BACKEND", "dask_dataframe") - L.debug("Multiprocessing backend: %s", mode) - return mode diff --git a/tests/unit/me-model/test_wrapper.py b/tests/unit/me-model/test_wrapper.py index ab0cdd5..9d9e498 100644 --- a/tests/unit/me-model/test_wrapper.py +++ b/tests/unit/me-model/test_wrapper.py @@ -38,10 +38,10 @@ def test_stage_circuit(tmp_path, detailed_circuit_metadata, circuit_config_file) assert res == load_json(circuit_config_file) -def test_build_recipe(tmp_path, materialized_me_model_config_file): +def test_recipe(tmp_path, materialized_me_model_config_file): output_file = tmp_path / "recipe.json" - test_module._build_recipe(materialized_me_model_config_file, output_file) + test_module.recipe(config_file=materialized_me_model_config_file, output_file=output_file) res = load_json(output_file) @@ -77,155 +77,6 @@ def test_build_recipe(tmp_path, materialized_me_model_config_file): } -@patchenv(foo="bar") -def test_run_emodel_prepare(): - with ( - patch("subprocess.run") as patched_subprocess, - patch( - "blue_cwl.utils.build_variant_allocation_command", - side_effect=lambda e, *args, **kwargs: e, - ), - ): - test_module._run_emodel_prepare( - recipe_file="recipe-file", - mechanisms_dir="mechanisms-dir", - work_dir="work-dir", - variant=None, - ) - expected_command = ( - "emodel-generalisation -v prepare " - "--config-path recipe-file " - "--local-config-path work-dir/configs " - "--mechanisms-path mechanisms-dir" - ) - patched_subprocess.assert_called_once_with( - expected_command, - check=True, - shell=True, - env={ - "foo": "bar", - "NEURON_MODULE_OPTIONS": "-nogui", - }, - ) - - -@patchenv(foo="bar") -def test_run_emodel_assign(circuit_config_file): - with ( - patch("subprocess.run") as patched_subprocess, - patch("blue_cwl.validation.check_properties_in_population"), - patch( - "blue_cwl.utils.build_variant_allocation_command", - side_effect=lambda e, *args, **kwargs: e, - ), - ): - test_module._run_emodel_assign( - circuit_config_file=circuit_config_file, - recipe_file="recipe-file", - output_nodes_file="out-file", - variant=None, - work_dir=Path("work-dir"), - ) - expected_command = ( - "emodel-generalisation -v --no-progress assign " - "--input-node-path nodes.h5 " - "--config-path recipe-file " - "--output-node-path out-file " - "--local-config-path work-dir/configs" - ) - - patched_subprocess.assert_called_once_with( - expected_command, - check=True, - shell=True, - env={ - "foo": "bar", - "NEURON_MODULE_OPTIONS": "-nogui", - }, - ) - - -@patchenv(foo="bar") -def test_run_emodel_adapt(circuit_config_file): - with ( - patch("subprocess.run") as patched_subprocess, - patch("blue_cwl.validation.check_properties_in_population"), - patch( - "blue_cwl.utils.build_variant_allocation_command", - side_effect=lambda e, *args, **kwargs: e, - ), - ): - test_module._run_emodel_adapt( - circuit_config_file=circuit_config_file, - nodes_file="nodes-file-path", - recipe_file="recipe-file", - output_nodes_file="out-file", - output_biophysical_models_dir="hoc-dir", - variant=None, - work_dir=Path("work-dir"), - mechanisms_dir="mechanisms-dir", - ) - expected_command = ( - "emodel-generalisation -v --no-progress adapt " - "--input-node-path nodes-file-path " - "--output-node-path out-file " - "--morphology-path morphologies " - "--config-path recipe-file " - "--output-hoc-path hoc-dir " - "--parallel-lib dask_dataframe " - "--local-config-path work-dir/configs " - "--local-dir work-dir/local" - ) - patched_subprocess.assert_called_once_with( - expected_command, - check=True, - shell=True, - env={ - "foo": "bar", - "EMODEL_GENERALISATION_MOD_LIBRARY_PATH": "mechanisms-dir", - "NEURON_MODULE_OPTIONS": "-nogui", - }, - ) - - -@patchenv(foo="bar") -def test_run_emodel_currents(circuit_config_file): - with ( - patch("subprocess.run") as patched_subprocess, - patch("blue_cwl.validation.check_properties_in_population"), - patch( - "blue_cwl.utils.build_variant_allocation_command", - side_effect=lambda e, *args, **kwargs: e, - ), - ): - test_module._run_emodel_currents( - circuit_config_file=circuit_config_file, - nodes_file="nodes-file-path", - biophysical_neuron_models_dir="hoc-dir", - output_nodes_file="out-file", - variant=None, - mechanisms_dir="mechanisms-dir", - ) - expected_command = ( - "emodel-generalisation -v --no-progress compute_currents " - "--input-path nodes-file-path " - "--output-path out-file " - "--morphology-path morphologies " - "--hoc-path hoc-dir " - "--parallel-lib dask_dataframe" - ) - patched_subprocess.assert_called_once_with( - expected_command, - check=True, - shell=True, - env={ - "foo": "bar", - "EMODEL_GENERALISATION_MOD_LIBRARY_PATH": "mechanisms-dir", - "NEURON_MODULE_OPTIONS": "-nogui", - }, - ) - - def test_register(tmp_path, circuit_config_file, circuit_config, detailed_circuit_metadata): output_dir = tmp_path / "out" output_dir.mkdir() From 6f5dc38c7a82de44084b4e4fe3a1e9f8afeafed0 Mon Sep 17 00:00:00 2001 From: Zisis Eleftherios Date: Fri, 31 May 2024 11:35:24 +0200 Subject: [PATCH 2/3] Use spack modules --- .../generators/memodel/neurons_memodel/v3/emodel_adapt.cwl | 6 ++++-- .../generators/memodel/neurons_memodel/v3/emodel_assign.cwl | 6 ++++-- .../memodel/neurons_memodel/v3/emodel_currents.cwl | 6 ++++-- .../memodel/neurons_memodel/v3/emodel_prepare.cwl | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_adapt.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_adapt.cwl index 5aee21f..6f95db7 100644 --- a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_adapt.cwl +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_adapt.cwl @@ -5,8 +5,10 @@ id: emodel_adapt label: emodel-adapt environment: - env_type: VENV - path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + env_type: MODULE + modules: + - unstable + - py-emodel-generalisation executor: type: slurm diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_assign.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_assign.cwl index 842ffcc..eefe197 100644 --- a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_assign.cwl +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_assign.cwl @@ -5,8 +5,10 @@ id: emodel_assign label: emodel-assign environment: - env_type: VENV - path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + env_type: MODULE + modules: + - unstable + - py-emodel-generalisation executor: type: slurm diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_currents.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_currents.cwl index ca39d7e..2657439 100644 --- a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_currents.cwl +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_currents.cwl @@ -5,8 +5,10 @@ id: emodel_currents label: emodel-currents environment: - env_type: VENV - path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + env_type: MODULE + modules: + - unstable + - py-emodel-generalisation executor: type: slurm diff --git a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_prepare.cwl b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_prepare.cwl index 8739210..9de1b37 100644 --- a/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_prepare.cwl +++ b/src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_prepare.cwl @@ -5,8 +5,10 @@ id: emodel_prepare label: emodel-prepare environment: - env_type: VENV - path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311 + env_type: MODULE + modules: + - unstable + - py-emodel-generalisation executor: type: slurm From cb0e84dfc2834c23cca25e1547546c670ec33874 Mon Sep 17 00:00:00 2001 From: Zisis Eleftherios Date: Fri, 31 May 2024 13:22:37 +0200 Subject: [PATCH 3/3] Add workflow image --- doc/source/registry.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/registry.rst b/doc/source/registry.rst index 472afa1..6f904d4 100644 --- a/doc/source/registry.rst +++ b/doc/source/registry.rst @@ -4,6 +4,12 @@ Registry ======== +memodel|neurons_memodel|v3 +************************** + +.. image:: generated/memodel__neurons_memodel__v3.svg + + connectome_filtering|synapses|v2 ********************************