Update tutorial cdm extension via selection of concepts

cognite/neat/_alpha.py

@@ -1,23 +1,24 @@
 import warnings
 
 
-class AlphaWarning(UserWarning):
+class ExperimentalFeatureWarning(UserWarning):
     def __init__(self, feature_name: str):
-        super().__init__(f"Alpha feature '{feature_name}' is subject to change without notice")
+        super().__init__(f"Experimental feature '{feature_name}' is subject to change without notice")
 
     def warn(self) -> None:
         warnings.warn(self, stacklevel=2)
 
 
-class AlphaFlags:
-    manual_rules_edit = AlphaWarning("enable_manual_edit")
-    same_space_properties_only_export = AlphaWarning("same-space-properties-only")
-    standardize_naming = AlphaWarning("standardize_naming")
-    standardize_space_and_version = AlphaWarning("standardize_space_and_version")
-    data_model_subsetting = AlphaWarning("data_model_subsetting")
-    ontology_read = AlphaWarning("ontology_read")
-    imf_read = AlphaWarning("imf_read")
-    dexpi_read = AlphaWarning("dexpi_read")
-    aml_read = AlphaWarning("aml_read")
-    csv_read = AlphaWarning("csv_read")
-    to_ontology = AlphaWarning("to_ontology")
+class ExperimentalFlags:
+    manual_rules_edit = ExperimentalFeatureWarning("enable_manual_edit")
+    same_space_properties_only_export = ExperimentalFeatureWarning("same-space-properties-only")
+    standardize_naming = ExperimentalFeatureWarning("standardize_naming")
+    standardize_space_and_version = ExperimentalFeatureWarning("standardize_space_and_version")
+    data_model_subsetting = ExperimentalFeatureWarning("data_model_subsetting")
+    core_data_model_subsetting = ExperimentalFeatureWarning("core_data_model_subsetting")
+    ontology_read = ExperimentalFeatureWarning("ontology_read")
+    imf_read = ExperimentalFeatureWarning("imf_read")
+    dexpi_read = ExperimentalFeatureWarning("dexpi_read")
+    aml_read = ExperimentalFeatureWarning("aml_read")
+    csv_read = ExperimentalFeatureWarning("csv_read")
+    to_ontology = ExperimentalFeatureWarning("to_ontology")

cognite/neat/_constants.py

@@ -64,6 +64,79 @@ def _is_in_browser() -> bool:
         "cdf_units",
     }
 )
+
+COGNITE_CORE_CONCEPTS = frozenset(
+    {
+        "CogniteFile",
+        "CogniteCubeMap",
+        "CogniteCADRevision",
+        "CognitePointCloudVolume",
+        "Cognite360ImageAnnotation",
+        "CogniteAnnotation",
+        "CogniteUnit",
+        "CogniteAsset",
+        "Cognite3DObject",
+        "Cognite3DRevision",
+        "Cognite360Image",
+        "CogniteDiagramAnnotation",
+        "Cognite360ImageCollection",
+        "Cognite360ImageStation",
+        "CognitePointCloudModel",
+        "CogniteTimeSeries",
+        "Cognite3DTransformation",
+        "CogniteEquipment",
+        "Cognite360ImageModel",
+        "CogniteAssetClass",
+        "CogniteAssetType",
+        "CogniteEquipmentType",
+        "Cognite3DModel",
+        "CogniteCADModel",
+        "CognitePointCloudRevision",
+        "CogniteCADNode",
+        "CogniteFileCategory",
+        "CogniteActivity",
+    }
+)
+
+
+COGNITE_CORE_FEATURES = frozenset(
+    {
+        "CogniteDescribable",
+        "CogniteSourceable",
+        "CogniteSourceSystem",
+        "CogniteSchedulable",
+        "CogniteVisualizable",
+    }
+)
+
+COGNITE_3D_CONCEPTS = frozenset(
+    {
+        "Cognite3DModel",
+        "Cognite3DObject",
+        "Cognite3DRevision",
+        "Cognite3DTransformation",
+        "Cognite360Image",
+        "Cognite360ImageAnnotation",
+        "Cognite360ImageCollection",
+        "Cognite360ImageModel",
+        "Cognite360ImageStation",
+        "CogniteCADModel",
+        "CogniteCADNode",
+        "CogniteCADRevision",
+        "CogniteCubeMap",
+        "CognitePointCloudModel",
+        "CognitePointCloudRevision",
+        "CognitePointCloudVolume",
+    }
+)
+
+COGNITE_ANNOTATION = frozenset(
+    {
+        "CogniteAnnotation",
+        "CogniteDiagramAnnotation",
+    }
+)
+
 DMS_LISTABLE_PROPERTY_LIMIT = 1000
 
 EXAMPLE_RULES = PACKAGE_DIRECTORY / "_rules" / "examples"

cognite/neat/_rules/transformers/_converters.py

@@ -17,6 +17,8 @@
 from cognite.neat._client import NeatClient
 from cognite.neat._client.data_classes.data_modeling import ContainerApplyDict, ViewApplyDict
 from cognite.neat._constants import (
+    COGNITE_CORE_CONCEPTS,
+    COGNITE_CORE_FEATURES,
     COGNITE_MODELS,
     COGNITE_SPACES,
     DMS_CONTAINER_PROPERTY_SIZE_LIMIT,
@@ -1795,6 +1797,76 @@ def _convert_metadata_to_info(cls, metadata: DMSMetadata) -> "InformationMetadat
         )
 
 
+class _SubsetEditableCDMRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
+    """Subsets editable CDM rules to only include desired set of CDM concepts.
+
+    !!! note "Platypus UI limitations"
+        This is temporal solution to enable cleaner extension of core data model,
+        assuring that Platypus UI will work correctly, including Data Model Editor,
+        Query Explorer and Search.
+    """
+
+    def __init__(self, views: set[ViewEntity]):
+        if not_in_cognite_core := {view.external_id for view in views} - COGNITE_CORE_CONCEPTS.union(
+            COGNITE_CORE_FEATURES
+        ):
+            raise NeatValueError(
+                f"Concept(s) {', '.join(not_in_cognite_core)} is/are not part of the Cognite Core Data Model. Aborting."
+            )
+
+        self._views = views
+
+    def transform(self, rules: DMSRules) -> DMSRules:
+        # should check to make sure data model is based on the editable CDM
+        # if not raise an error
+
+        subsetted_rules: dict[str, Any] = {
+            "metadata": rules.metadata.model_copy(),
+            "views": SheetList[DMSView](),
+            "properties": SheetList[DMSProperty](),
+            "containers": SheetList[DMSContainer](),
+            "enum": rules.enum,
+            "nodes": rules.nodes,
+        }
+
+        containers_to_keep = set()
+
+        if editable_views_to_keep := self._editable_views_to_keep(rules):
+            for view in rules.views:
+                if view.view in editable_views_to_keep or view.view.space in COGNITE_SPACES:
+                    subsetted_rules["views"].append(view)
+
+            for property_ in rules.properties:
+                if property_.view in editable_views_to_keep and (
+                    isinstance(property_.value_type, DataType)
+                    or isinstance(property_.value_type, DMSUnknownEntity)
+                    or (isinstance(property_.value_type, ViewEntity) and property_.value_type in editable_views_to_keep)
+                ):
+                    subsetted_rules["properties"].append(property_)
+                    if property_.container:
+                        containers_to_keep.add(property_.container)
+
+            if rules.containers:
+                for container in rules.containers:
+                    if container.container in containers_to_keep:
+                        subsetted_rules["containers"].append(container)
+            try:
+                return DMSRules.model_validate(subsetted_rules)
+            except ValidationError as e:
+                raise NeatValueError(f"Cannot subset rules: {e}") from e
+        else:
+            raise NeatValueError("Cannot subset rules: provided data model is not based on Core Data Model")
+
+    def _editable_views_to_keep(self, rules: DMSRules) -> set[ViewEntity]:
+        return {
+            view.view
+            for view in rules.views
+            if view.view.space not in COGNITE_SPACES
+            and view.implements
+            and any(implemented in self._views for implemented in view.implements)
+        }
+
+
 class SubsetDMSRules(VerifiedRulesTransformer[DMSRules, DMSRules]):
     """Subsets DMSRules to only include the specified views."""
 

cognite/neat/_session/_create.py

@@ -3,7 +3,6 @@
 from cognite.client.data_classes.data_modeling import DataModelIdentifier
 
 from cognite.neat._issues import IssueList
-from cognite.neat._rules.models import DMSRules, InformationRules
 from cognite.neat._rules.models.dms import DMSValidation
 from cognite.neat._rules.transformers import (
     IncludeReferenced,
@@ -29,7 +28,7 @@ def __init__(self, state: SessionState):
     def enterprise_model(
         self,
         data_model_id: DataModelIdentifier,
-        org_name: str = "My",
+        org_name: str = "CopyOf",
         dummy_property: str = "GUID",
     ) -> IssueList:
         """Uses the current data model as a basis to create enterprise data model
@@ -58,7 +57,7 @@ def enterprise_model(
                 - Charts
 
         """
-        last_rules = self._get_last_rules()
+        last_rules = self._state.rule_store.last_verified_rules
         issues = self._state.rule_transform(
             ToEnterpriseModel(
                 new_model_id=data_model_id,
@@ -71,12 +70,6 @@ def enterprise_model(
             self._state.last_reference = last_rules
         return issues
 
-    def _get_last_rules(self) -> InformationRules | DMSRules | None:
-        if not self._state.rule_store.provenance:
-            return None
-        last_entity = self._state.rule_store.provenance[-1].target_entity
-        return last_entity.dms or last_entity.information
-
     def solution_model(
         self,
         data_model_id: DataModelIdentifier,
@@ -110,7 +103,7 @@ def solution_model(
             the containers in the solution data model space.
 
         """
-        last_rules = self._get_last_rules()
+        last_rules = self._state.rule_store.last_verified_rules
         issues = self._state.rule_transform(
             ToSolutionModel(
                 new_model_id=data_model_id,
@@ -144,7 +137,7 @@ def data_product_model(
                 If you set same-space, only the properties of the views in the same space as the data model
                 will be included.
         """
-        last_rules = self._get_last_rules()
+        last_rules = self._state.rule_store.last_verified_rules
         view_ids, container_ids = DMSValidation(
             self._state.rule_store.last_verified_dms_rules
         ).imported_views_and_containers_ids()

cognite/neat/_session/_prepare.py

@@ -4,7 +4,7 @@
 
 from rdflib import URIRef
 
-from cognite.neat._alpha import AlphaFlags
+from cognite.neat._alpha import ExperimentalFlags
 from cognite.neat._graph.transformers import (
     ConnectionToLiteral,
     ConvertLiteral,
@@ -273,7 +273,7 @@ def standardize_naming(self) -> IssueList:
         For properties, the naming will be standardized to camelCase.
         """
         warnings.filterwarnings("default")
-        AlphaFlags.standardize_naming.warn()
+        ExperimentalFlags.standardize_naming.warn()
         return self._state.rule_transform(StandardizeNaming())
 
     def standardize_space_and_version(self) -> IssueList:
@@ -282,5 +282,5 @@ def standardize_space_and_version(self) -> IssueList:
         This method will standardize the space and version in the data model to the Cognite standard.
         """
         warnings.filterwarnings("default")
-        AlphaFlags.standardize_space_and_version.warn()
+        ExperimentalFlags.standardize_space_and_version.warn()
         return self._state.rule_transform(StandardizeSpaceAndVersion())

cognite/neat/_session/_read.py

@@ -4,7 +4,7 @@
 from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
 from cognite.client.utils.useful_types import SequenceNotStr
 
-from cognite.neat._alpha import AlphaFlags
+from cognite.neat._alpha import ExperimentalFlags
 from cognite.neat._client import NeatClient
 from cognite.neat._constants import (
     CLASSIC_CDF_NAMESPACE,
@@ -361,7 +361,7 @@ def __call__(self, io: Any, enable_manual_edit: bool = False) -> IssueList:
 
         if enable_manual_edit:
             warnings.filterwarnings("default")
-            AlphaFlags.manual_rules_edit.warn()
+            ExperimentalFlags.manual_rules_edit.warn()
 
         return self._state.rule_import(importers.ExcelImporter(path), enable_manual_edit)
 
@@ -421,7 +421,7 @@ class CSVReadAPI(BaseReadAPI):
 
     def __call__(self, io: Any, type: str, primary_key: str) -> None:
         warnings.filterwarnings("default")
-        AlphaFlags.csv_read.warn()
+        ExperimentalFlags.csv_read.warn()
 
         engine = import_engine()
         engine.set.format = "csv"
@@ -479,7 +479,7 @@ def dexpi(self, io: Any) -> None:
             - remove edges to nodes that do not exist in the extracted graph
         """
         warnings.filterwarnings("default")
-        AlphaFlags.dexpi_read.warn()
+        ExperimentalFlags.dexpi_read.warn()
 
         self._state._raise_exception_if_condition_not_met(
             "Read DEXPI file",
@@ -539,7 +539,7 @@ def aml(self, io: Any):
             - remove edges to nodes that do not exist in the extracted graph
         """
         warnings.filterwarnings("default")
-        AlphaFlags.aml_read.warn()
+        ExperimentalFlags.aml_read.warn()
 
         self._state._raise_exception_if_condition_not_met(
             "Read AML file",
@@ -599,7 +599,7 @@ def ontology(self, io: Any) -> IssueList:
             ```
         """
         warnings.filterwarnings("default")
-        AlphaFlags.ontology_read.warn()
+        ExperimentalFlags.ontology_read.warn()
 
         self._state._raise_exception_if_condition_not_met(
             "Read Ontology file",
@@ -622,7 +622,7 @@ def imf(self, io: Any) -> IssueList:
             ```
         """
         warnings.filterwarnings("default")
-        AlphaFlags.imf_read.warn()
+        ExperimentalFlags.imf_read.warn()
 
         self._state._raise_exception_if_condition_not_met(
             "Read IMF file",
@@ -674,12 +674,6 @@ class Examples:
     def __init__(self, state: SessionState) -> None:
         self._state = state
 
-    @property
-    def _get_client(self) -> NeatClient:
-        if self._state.client is None:
-            raise NeatValueError("No client provided. Please provide a client to read a data model.")
-        return self._state.client
-
     def nordic44(self) -> IssueList:
         """Reads the Nordic 44 knowledge graph into the NeatSession graph store."""
 
@@ -709,8 +703,11 @@ def core_data_model(self) -> IssueList:
         self._state._raise_exception_if_condition_not_met(
             "Read Core Data Model example",
             empty_rules_store_required=True,
+            client_required=True,
         )
 
         cdm_v1 = DataModelId.load(("cdf_cdm", "CogniteCore", "v1"))
-        importer: importers.DMSImporter = importers.DMSImporter.from_data_model_id(self._get_client, cdm_v1)
+        importer: importers.DMSImporter = importers.DMSImporter.from_data_model_id(
+            cast(NeatClient, self._state.client), cdm_v1
+        )
         return self._state.rule_import(importer)