Convert cell composition manipulation CommandLineTool into a Workflow

doc/source/registry.rst

@@ -4,6 +4,11 @@
 Registry
 ========
 
+cell_composition|cell_composition_manipulation|v3
+*************************************************
+
+.. image:: generated/cell_composition__cell_composition_manipulation__v3.svg
+
 mmodel|neurons_mmodel|v3
 ************************
 
@@ -16,6 +21,12 @@ memodel|neurons_memodel|v3
 .. image:: generated/memodel__neurons_memodel__v3.svg
 
 
+connectome__placeholder__v3.svg
+*******************************
+
+.. image:: generated/connectome__placeholder__v3.svg
+
+
 connectome_filtering|synapses|v2
 ********************************
 

pyproject.toml

@@ -21,9 +21,9 @@ dependencies = [
   "pandas",
   "voxcell",
   "joblib",
-  "entity_management>=1.2.45",  # latest emodel definitions
+  "entity-management>=1.3.1",  # METypeDensity entity
   "pyarrow>=3.0.0",
-  "fz_td_recipe>=0.2.0",  # support for json recipe
+  "fz-td-recipe>=0.2.0",  # support for json recipe
   "pydantic",
   "morph-tool",
   "jsonschema",

src/blue_cwl/generators/cell_composition/cell_composition_manipulation/v3/definition.cwl

@@ -0,0 +1,70 @@
+cwlVersion: v1.2
+class: Workflow
+
+id: cell_composition_manipulation_workflow
+label: CellComposition manipulation workflow
+
+
+inputs:
+
+  - id: configuration_id
+    type: NexusType
+
+  - id: base_cell_composition_id
+    type: NexusType
+
+  - id: output_dir
+    type: Directory
+
+outputs:
+
+  - id: cell_composition
+    type: NexusType
+    doc: Manipulated CellComposition
+    outputSource: register/cell_composition
+
+steps:
+
+  - id: setup
+    run: ./setup.cwl
+    in:
+      output_dir: output_dir
+    out:
+      - stage_dir
+      - build_dir
+
+  - id: stage
+    run: ./stage.cwl
+    in:
+      stage_dir: setup/stage_dir
+      configuration_id: configuration_id
+      base_cell_composition_id: base_cell_composition_id
+    out:
+      - atlas_file
+      - recipe_file
+      - region_selection_file
+      - densities_file
+      - materialized_densities_file
+
+  - id: manipulate_cell_composition
+    run: ./manipulate_cell_composition.cwl
+    in:
+      atlas_file: stage/atlas_file
+      recipe_file: stage/recipe_file
+      region_selection_file: stage/region_selection_file
+      densities_file: stage/densities_file
+      materialized_densities_file: stage/materialized_densities_file
+      output_dir: setup/build_dir
+    out:
+      - cell_composition_volume_file
+      - cell_composition_summary_file
+
+  - id: register
+    run: ./register.cwl
+    in:
+      base_cell_composition_id: base_cell_composition_id
+      cell_composition_volume_file: manipulate_cell_composition/cell_composition_volume_file
+      cell_composition_summary_file: manipulate_cell_composition/cell_composition_summary_file
+      output_dir: output_dir
+    out:
+      - cell_composition

src/blue_cwl/generators/cell_composition/cell_composition_manipulation/v3/manipulate_cell_composition.cwl

@@ -0,0 +1,67 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+id: transform
+label: transform
+
+environment:
+  env_type: VENV
+  path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311
+
+executor:
+  type: slurm
+  slurm_config:
+    partition: prod
+    account: proj134
+    exclusive: true
+    time: '1:00:00'
+    nodes: 1
+    mem: 0
+  remote_config:
+    host: bbpv1.epfl.ch
+
+baseCommand: ["blue-cwl", "execute", "cell-composition-manipulation", "manipulate-cell-composition"]
+
+inputs:
+
+  - id: atlas_file
+    type: File
+    inputBinding:
+      prefix: --atlas-file
+
+  - id: recipe_file
+    type: File
+    inputBinding:
+      prefix: --manipulation-file
+
+  - id: region_selection_file
+    type: File
+    inputBinding:
+      prefix: --region-selection-file
+
+  - id: densities_file
+    type: File
+    inputBinding:
+      prefix: --cell-composition-volume-file
+
+  - id: materialized_densities_file
+    type: File
+    inputBinding:
+      prefix: --materialized-cell-composition-volume-file
+
+  - id: output_dir
+    type: Directory
+    inputBinding:
+      prefix: --output-dir
+
+outputs:
+
+  - id: cell_composition_volume_file
+    type: File
+    outputBinding:
+      glob: $(inputs.output_dir.path)/cell_composition_volume.json
+
+  - id: cell_composition_summary_file
+    type: File
+    outputBinding:
+      glob: $(inputs.output_dir.path)/cell_composition_summary.json

src/blue_cwl/generators/cell_composition/cell_composition_manipulation/v3/register.cwl

@@ -0,0 +1,52 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+id: cell_composition_manipulation_register
+label: cell-composition-manipulation-register
+
+environment:
+  env_type: VENV
+  path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311
+
+executor:
+  type: slurm
+  slurm_config:
+    partition: prod
+    account: proj134
+    exclusive: true
+    time: '1:00:00'
+    nodes: 1
+    mem: 0
+  remote_config:
+    host: bbpv1.epfl.ch
+
+baseCommand: ["blue-cwl", "execute", "cell-composition-manipulation", "register"]
+
+inputs:
+
+  - id: base_cell_composition_id
+    type: NexusType
+    inputBinding:
+      prefix: --base-cell-composition-id
+
+  - id: cell_composition_volume_file
+    type: File
+    inputBinding:
+      prefix: --cell-composition-volume-file
+
+  - id: cell_composition_summary_file
+    type: File
+    inputBinding:
+      prefix: --cell-composition-summary-file
+
+  - id: output_dir
+    type: Directory
+    inputBinding:
+      prefix: --output-dir
+
+outputs:
+
+  - id: cell_composition
+    type: NexusType
+    outputBinding:
+      glob: $(inputs.output_dir.path)/resource.json

src/blue_cwl/generators/cell_composition/cell_composition_manipulation/v3/setup.cwl

@@ -0,0 +1,42 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+id: create-directories
+label: create-directories
+
+environment:
+  env_type: VENV
+  path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311
+
+executor:
+  type: slurm
+  slurm_config:
+    partition: prod
+    account: proj134
+    exclusive: true
+    time: '00:10:00'
+    ntasks: 1
+    mem: 0
+  remote_config:
+    host: bbpv1.epfl.ch
+
+baseCommand: ["blue-cwl", "execute", "common", "setup-directories"]
+
+inputs:
+
+  - id: output_dir
+    type: Directory
+    inputBinding:
+      prefix: --output-dir
+
+outputs:
+
+  - id: stage_dir
+    type: Directory
+    outputBinding:
+      glob: $(inputs.output_dir.path)/stage
+
+  - id: build_dir
+    type: Directory
+    outputBinding:
+      glob: $(inputs.output_dir.path)/build

src/blue_cwl/generators/cell_composition/cell_composition_manipulation/v3/stage.cwl

@@ -0,0 +1,67 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+id: stage_cell_composition_manipulation
+label: stage-cell-composition-manipulation
+
+environment:
+  env_type: VENV
+  path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311
+
+executor:
+  type: slurm
+  slurm_config:
+    partition: prod
+    account: proj134
+    exclusive: true
+    time: '01:00:00'
+    ntasks: 1
+    mem: 0
+  remote_config:
+    host: bbpv1.epfl.ch
+
+baseCommand: ["blue-cwl", "execute", "cell-composition-manipulation", "stage"]
+
+inputs:
+
+  - id: configuration_id
+    type: NexusType
+    inputBinding:
+      prefix: --configuration-id
+
+  - id: base_cell_composition_id
+    type: NexusType
+    inputBinding:
+      prefix: --base-cell-composition-id
+
+  - id: stage_dir
+    type: Directory
+    inputBinding:
+      prefix: --stage-dir
+
+outputs:
+
+  - id: atlas_file
+    type: File
+    outputBinding:
+      glob: $(inputs.stage_dir.path)/atlas.json
+
+  - id: recipe_file
+    type: File
+    outputBinding:
+      glob: $(inputs.stage_dir.path)/recipe.parquet
+
+  - id: region_selection_file
+    type: File
+    outputBinding:
+      glob: $(inputs.stage_dir.path)/region_selection.json
+
+  - id: densities_file
+    type: File
+    outputBinding:
+      glob: $(inputs.stage_dir.path)/cell_composition_volume.json
+
+  - id: materialized_densities_file
+    type: File
+    outputBinding:
+      glob: $(inputs.stage_dir.path)/cell_composition_volume.parquet