Fixes (#22)

* Add derivation to canonical/placeholder configs
* Improve error when placeholders are not updated
* Fix merge issue with creating more rows than left dataframe
* Use module for connectome placeholder generator
* Increase nodes for parquet conversion
* Increase tasks for synthesis
* Use Variant attributes enforced by schema

doc/source/conf.py

@@ -77,6 +77,6 @@
 
 for variant in iter_registered_variants():
     tool = variant.tool_definition
-    name = f"{variant.generator_name}__{variant.variant_name}__{variant.version}.svg"
+    name = f"{variant.generatorName}__{variant.variantName}__{variant.version}.svg"
     if isinstance(tool, cwl.Workflow):
         tool.write_image(filepath=graph_dir / name)

pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
   "pandas",
   "voxcell",
   "joblib",
-  "entity-management>=1.4.0",
+  "entity-management>=1.4.1",
   "pyarrow>=3.0.0",
   "fz-td-recipe>=0.2.0",  # support for json recipe
   "pydantic",

src/blue_cwl/generators/connectome/placeholder/v3/generate_connectome.cwl

@@ -1,12 +1,14 @@
 cwlVersion: v1.2
 class: CommandLineTool
 
-id: transform
-label: transform
+id: generate_connectome
+label: generate_connectome
 
 environment:
-  env_type: VENV
-  path: /gpfs/bbp.cscs.ch/project/proj134/scratch/zisis/sub-workflows/venv311
+  env_type: MODULE
+  modules:
+    - archive/2024-06
+    - py-connectome-manipulator/0.0.10
 
 executor:
   type: slurm
@@ -15,7 +17,7 @@ executor:
     account: proj134
     exclusive: true
     time: '16:00:00'
-    nodes: 5
+    nodes: 200
     cpus_per_task: 40
     ntasks_per_node: 1
     mem: 0

src/blue_cwl/generators/connectome/placeholder/v3/parquet_to_sonata.cwl

@@ -14,7 +14,7 @@ executor:
   type: slurm
   slurm_config:
     partition: prod
-    nodes: 3
+    nodes: 100
     ntasks_per_node: 10
     exclusive: true
     time: '8:00:00'

src/blue_cwl/generators/mmodel/neurons_mmodel/v3/assign_synthesized.cwl

@@ -18,7 +18,7 @@ executor:
     exclusive: true
     time: '12:00:00'
     cpus_per_task: 2
-    ntasks: 20
+    ntasks: 300
     mem: 0
   remote_config:
     host: bbpv1.epfl.ch

src/blue_cwl/mmodel/entity.py

@@ -2,7 +2,8 @@
 
 """Entities."""
 
-from entity_management.core import Entity
+from entity_management.base import Derivation
+from entity_management.core import AttrOf, DataDownload, Entity, attributes
 
 from blue_cwl.mmodel import schemas
 from blue_cwl.validation import validate_schema
@@ -19,6 +20,12 @@ def to_model(self) -> schemas.MModelConfigRaw:
         return schemas.MModelConfigRaw.from_dict(dataset)
 
 
+@attributes(
+    {
+        "distribution": AttrOf(DataDownload),
+        "derivation": AttrOf(Derivation, default=None),
+    }
+)
 class CanonicalMorphologyModelConfig(Entity):
     """Canonical morphology model config."""
 
@@ -32,6 +39,12 @@ def to_model(self) -> schemas.CanonicalDistributionConfig:
         return schemas.CanonicalDistributionConfig(data=dataset)
 
 
+@attributes(
+    {
+        "distribution": AttrOf(DataDownload),
+        "derivation": AttrOf(Derivation, default=None),
+    }
+)
 class PlaceholderMorphologyConfig(Entity):
     """Placeholder morphologies config."""
 

src/blue_cwl/utils.py

@@ -510,8 +510,7 @@ def split(df: pd.DataFrame, mask: np.ndarray) -> voxcell.CellCollection | None:
 
     if isinstance(properties, pd.DataFrame):
         mask = (
-            df[properties.columns].merge(properties, how="outer", indicator=True)["_merge"]
-            == "both"
+            df[properties.columns].merge(properties, how="left", indicator=True)["_merge"] == "both"
         )
     else:
         mask = np.logical_and.reduce(

src/blue_cwl/variant.py

@@ -40,13 +40,11 @@ class _CompatEntity(Entity):
 
 @attributes(
     {
-        "generator_name": AttrOf(str),
-        "variant_name": AttrOf(str),
+        "name": AttrOf(str),
+        "generatorName": AttrOf(str),
+        "variantName": AttrOf(str),
         "version": AttrOf(str),
-        "name": AttrOf(str, default=None),
-        "distribution": AttrOf(DataDownload, default=None),
-        "allocation_resources": AttrOf(_CompatEntity, default=None),
-        "definitions": AttrOf(_CompatEntity, default=None),
+        "distribution": AttrOf(DataDownload),
     },
     repr=False,
 )
@@ -55,14 +53,14 @@ class Variant(Entity):
 
     def __repr__(self):
         """Return repr string."""
-        return f"Variant({self.generator_name}, {self.variant_name}, {self.version})"
+        return f"Variant({self.generatorName}, {self.variantName}, {self.version})"
 
     @property
     def overview(self) -> str:
         """Return detailed representation."""
         return (
-            f"generator_name: {self.generator_name}\n"
-            f"variant_name  : {self.variant_name}\n"
+            f"generatorName : {self.generatorName}\n"
+            f"variantName   : {self.variantName}\n"
             f"version       : {self.version}\n\n"
             f"content\n-------\n\n{dump_yaml(self.get_content())}"
         )
@@ -130,8 +128,8 @@ def from_file(
     ) -> "Variant":
         """Create Variant object from variant file definition."""
         return cls(
-            generator_name=generator_name,
-            variant_name=variant_name,
+            generatorName=generator_name,
+            variantName=variant_name,
             version=version,
             distribution=_create_local_variant_distribution(path=filepath),
             name=f"{generator_name}|{variant_name}|{version}",
@@ -176,8 +174,8 @@ def evolve(self, **changes) -> "Variant":
             changes["distribution"] = _create_local_variant_distribution(path=changes.pop("path"))
 
         changes["name"] = (
-            f"{changes.get('generator_name', self.generator_name)}|"
-            f"{changes.get('variant_name', self.variant_name)}|"
+            f"{changes.get('generatorName', self.generatorName)}|"
+            f"{changes.get('variantName', self.variantName)}|"
             f"{changes.get('version', self.version)}"
         )
         return super().evolve(**changes)
@@ -199,8 +197,8 @@ def publish(
         # to get the remote variant id, if any, and update it.
         if not variant.get_id():
             resource_id = search_variant_in_nexus(
-                generator_name=self.generator_name,
-                variant_name=self.variant_name,
+                generator_name=self.generatorName,
+                variant_name=self.variantName,
                 version=self.version,
                 base=base,
                 org=org,
@@ -364,8 +362,8 @@ def search_variant_in_nexus(
         WHERE {{
           ?id a bmo:Variant ;
              nxv:deprecated false ;
-             bmo:generator_name '{generator_name}' ;
-             bmo:variant_name '{variant_name}' ;
+             bmo:generatorName '{generator_name}' ;
+             bmo:variantName '{variant_name}' ;
              sch:version '{version}' .
         }}
         LIMIT 1

src/blue_cwl/wrappers/mmodel.py

@@ -343,8 +343,14 @@ def assign_placeholders(
         on=["region", "mtype"],
     )
 
-    if cells.properties[SONATA_MORPHOLOGY].isnull().any():
-        raise CWLWorkflowError("Null entries encountered in morphology column.")
+    if (null_mask := cells.properties[SONATA_MORPHOLOGY].isnull()).any():
+        raise CWLWorkflowError(
+            "Null entries encountered in morphology column.\n"
+            "A new composition is possibly used but without updating the placeholder config to "
+            "account for new region/mtype entries.\n"
+            "Example for missing region/mtype pairs from placeholder configuration:\n"
+            + str(cells.properties.loc[null_mask, ["region", "mtype"]].drop_duplicates())
+        )
 
     cells.properties[SONATA_MORPHOLOGY_PRODUCER] = MorphologyProducer.PLACEHOLDER
 

tests/unit/data/variant_metadata.json

@@ -19,9 +19,9 @@
     "encodingFormat": "application/cwl",
     "name": "definition.cwl"
   },
-  "generator_name": "testing",
+  "generatorName": "testing",
   "name": "testing|position|v0.3.1",
-  "variant_name": "position",
+  "variantName": "position",
   "version": "v0.3.1",
   "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/unconstrained.json",
   "_createdAt": "2024-01-08T15:26:47.165033Z",

tests/unit/test_variant.py

@@ -133,8 +133,8 @@ def test_variant__repr(variant, request):
 def test_variant__attributes(variant, request):
     variant = request.getfixturevalue(variant)
 
-    assert variant.variant_name == "position"
-    assert variant.generator_name == "testing"
+    assert variant.variantName == "position"
+    assert variant.generatorName == "testing"
     assert variant.version == _VERSION
 
 
@@ -185,10 +185,10 @@ def test_variant__tool_definition(variant, request, monkeypatch, variant_file):
 )
 def test_variant__evolve(variant, request, variant_file):
     variant = request.getfixturevalue(variant)
-    new_variant = variant.evolve(variant_name="foo", path=variant_file)
+    new_variant = variant.evolve(variantName="foo", path=variant_file)
 
-    assert variant.generator_name == "testing"
-    assert variant.variant_name == "position"
+    assert variant.generatorName == "testing"
+    assert variant.variantName == "position"
     assert variant.version == _VERSION
     assert type(variant).__name__ == "Variant"
     assert variant.name == f"testing|position|{_VERSION}"
@@ -198,8 +198,8 @@ def test_variant__evolve(variant, request, variant_file):
     else:
         assert variant.distribution.contentUrl is not None
 
-    assert new_variant.generator_name == "testing"
-    assert new_variant.variant_name == "foo"
+    assert new_variant.generatorName == "testing"
+    assert new_variant.variantName == "foo"
     assert new_variant.version == _VERSION
     assert type(new_variant).__name__ == "Variant"
     assert new_variant.name == f"testing|foo|{_VERSION}"
@@ -219,8 +219,8 @@ def test_variant__publish__remote_variant(variant_from_id, monkeypatch):
         payload = patched.call_args.args[2]
 
         assert payload == {
-            "generator_name": "testing",
-            "variant_name": "position",
+            "generatorName": "testing",
+            "variantName": "position",
             "version": "v0.3.1",
             "name": "testing|position|v0.3.1",
             "distribution": {
@@ -271,8 +271,8 @@ def test_variant__publish__remote_variant__local_distr(variant_from_id, monkeypa
         payload = patched.call_args.args[2]
 
         assert payload == {
-            "generator_name": "testing",
-            "variant_name": "position",
+            "generatorName": "testing",
+            "variantName": "position",
             "version": "v0.3.1",
             "name": "testing|position|v0.3.1",
             "distribution": {
@@ -326,8 +326,8 @@ def test_variant__publish__local_variant(
         payload = patched.call_args.args[1]
 
         assert payload == {
-            "generator_name": "testing",
-            "variant_name": "position",
+            "generatorName": "testing",
+            "variantName": "position",
             "version": "v0.3.1",
             "name": "testing|position|v0.3.1",
             "distribution": {