Release 0.2.7 (#715)

# Description

Please describe the change you have made.

## Checklist

- [ ] Tests added/updated.
- [ ] Run Demo Job Locally.
- [ ] Documentation updated.
- [ ] Changelogs updated in
[CHANGELOG.cdf-tk.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.cdf-tk.md).
- [ ] Template changelogs updated in
[CHANGELOG.templates.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.templates.md).
- [ ] Version bumped.

[_version.py](https://github.com/cognitedata/toolkit/blob/main/cognite/cognite_toolkit/_version.py)
and

[pyproject.toml](https://github.com/cognitedata/toolkit/blob/main/pyproject.toml)
per [semantic versioning](https://semver.org/).

CHANGELOG.cdf-tk.md

@@ -15,6 +15,21 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.7] - 2024-06-28
+
+### Fixed
+
+- Function schedule for functions with a `:` in the external id would raise an `ValueError`.
+  This is now fixed.
+- Transformation notifications for transformations with a `:` in the external id would raise an `ValueError`.
+  This is now fixed.
+- When running `cdf-tk deploy`, you would get warnings about unrelated resources that were not part of the deployment.
+  This is now fixed.
+- The `transformations/notifications` endpoint was giving `500` errors for requests to non-exising transformations.
+  This is now handled by the toolkit and will not raise an error.
+- When doing variable replacement in a `sql` such as `dataset_id('{{dataset_external_id}}')`, the toolkit would
+  remove the quotes. This is now fixed.
+
 ## [0.2.6] - 2024-06-26
 
 ### Improved

CHANGELOG.templates.md

@@ -15,6 +15,10 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.7] - 2024-06-28
+
+No changes to templates.
+
 ## [0.2.6] - 2024-06-26
 
 No changes to templates.

cognite_toolkit/_cdf_tk/_migration.yaml

@@ -1,8 +1,13 @@
-- version: 0.2.6
+- version: 0.2.7
   cognite_modules: {}
   resources: {}
   tool: {}
   cognite_modules_hash: ""
+- version: 0.2.6
+  cognite_modules: {}
+  resources: {}
+  tool: {}
+  cognite_modules_hash: "ba21b355829fb39a04d741e154d86d70146d647c09f04c1ec0aa15167af0c1f2"
 - version: 0.2.5
   cognite_modules: {}
   resources: {}

cognite_toolkit/_cdf_tk/commands/build.py

@@ -225,7 +225,7 @@ def process_config_files(
                     if not is_function_non_yaml:
                         content = source_path.read_text()
                         state.hash_by_source_path[source_path] = calculate_str_or_file_hash(content)
-                        content = state.replace_variables(content)
+                        content = state.replace_variables(content, source_path.suffix)
                         destination.write_text(content)
                         state.source_by_build_path[destination] = source_path
                         file_warnings = self.validate(content, source_path, destination, state, verbose)
@@ -678,16 +678,20 @@ def update_local_variables(self, module_dir: Path) -> None:
     def create_file_name(self, filepath: Path) -> str:
         return _Helpers.create_file_name(filepath, self.number_by_resource_type)
 
-    def replace_variables(self, content: str) -> str:
+    def replace_variables(self, content: str, file_suffix: str = ".yaml") -> str:
         for name, variable in self.local_variables.items():
             replace = variable
-            if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
-                replace = f'"{replace}"'
-            elif replace is None:
-                replace = "null"
-
             _core_patter = rf"{{{{\s*{name}\s*}}}}"
-            content = re.sub(rf"'{_core_patter}'|{_core_patter}|" + rf'"{_core_patter}"', str(replace), content)
+            if file_suffix in {".yaml", ".yml", ".json"}:
+                # Preserve data types
+                if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
+                    replace = f'"{replace}"'
+                elif replace is None:
+                    replace = "null"
+                content = re.sub(rf"'{_core_patter}'|{_core_patter}|" + rf'"{_core_patter}"', str(replace), content)
+            else:
+                content = re.sub(_core_patter, str(replace), content)
+
         return content
 
     @classmethod

cognite_toolkit/_cdf_tk/commands/deploy.py

@@ -102,6 +102,7 @@ def execute(
             for folder_name, loader_classes in LOADER_BY_FOLDER_NAME.items()
             if folder_name in include and (build_dir / folder_name).is_dir()
             for loader_cls in loader_classes
+            if loader_cls.any_supported_files(build_dir / folder_name)
         }
         results = DeployResults([], "deploy", dry_run=dry_run)
 

cognite_toolkit/_cdf_tk/loaders/_base_loaders.py

@@ -97,6 +97,10 @@ def find_files(self, dir_or_file: Path | None = None) -> list[Path]:
         else:
             return []
 
+    @classmethod
+    def any_supported_files(cls, directory: Path) -> bool:
+        return any(cls.is_supported_file(file) for file in directory.glob("**/*"))
+
     @classmethod
     def is_supported_file(cls, file: Path) -> bool:
         if cls.filetypes and file.suffix[1:] not in cls.filetypes:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders.py

@@ -108,7 +108,6 @@
     ContainerApply,
     ContainerApplyList,
     ContainerList,
-    ContainerProperty,
     DataModel,
     DataModelApply,
     DataModelApplyList,
@@ -161,11 +160,13 @@
 
 from cognite_toolkit._cdf_tk._parameters import ANY_INT, ANY_STR, ANYTHING, ParameterSpec, ParameterSpecSet
 from cognite_toolkit._cdf_tk.exceptions import (
+    ToolkitFileNotFoundError,
     ToolkitInvalidParameterNameError,
     ToolkitRequiredValueError,
     ToolkitYAMLFormatError,
 )
 from cognite_toolkit._cdf_tk.tk_warnings import (
+    HighSeverityWarning,
     NamespacingConventionWarning,
     PrefixConventionWarning,
     WarningList,
@@ -982,6 +983,7 @@ class FunctionScheduleLoader(
     kind = "Schedule"
     dependencies = frozenset({FunctionLoader})
     _doc_url = "Function-schedules/operation/postFunctionSchedules"
+    _split_character = ":"
 
     @property
     def display_name(self) -> str:
@@ -1004,11 +1006,11 @@ def get_id(cls, item: FunctionScheduleWrite | FunctionSchedule | dict) -> str:
             if missing := tuple(k for k in {"functionExternalId", "cronExpression"} if k not in item):
                 # We need to raise a KeyError with all missing keys to get the correct error message.
                 raise KeyError(*missing)
-            return f"{item['functionExternalId']}:{item['cronExpression']}"
+            return f"{item['functionExternalId']}{cls._split_character}{item['cronExpression']}"
 
         if item.function_external_id is None or item.cron_expression is None:
             raise ToolkitRequiredValueError("FunctionSchedule must have functionExternalId and CronExpression set.")
-        return f"{item.function_external_id}:{item.cron_expression}"
+        return f"{item.function_external_id}{cls._split_character}{item.cron_expression}"
 
     @classmethod
     def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader], Hashable]]:
@@ -1023,7 +1025,7 @@ def load_resource(
             schedules = [schedules]
 
         for sched in schedules:
-            ext_id = f"{sched['functionExternalId']}:{sched['cronExpression']}"
+            ext_id = f"{sched['functionExternalId']}{self._split_character}{sched['cronExpression']}"
             if self.extra_configs.get(ext_id) is None:
                 self.extra_configs[ext_id] = {}
             self.extra_configs[ext_id]["authentication"] = sched.pop("authentication", {})
@@ -1046,13 +1048,23 @@ def _resolve_functions_ext_id(self, items: FunctionScheduleWriteList) -> Functio
         return items
 
     def retrieve(self, ids: SequenceNotStr[str]) -> FunctionSchedulesList:
-        functions = FunctionLoader(self.client, None).retrieve(list(set([id.split(":")[0] for id in ids])))
+        crons_by_function: dict[str, set[str]] = defaultdict(set)
+        for id_ in ids:
+            function_external_id, cron = id_.rsplit(self._split_character, 1)
+            crons_by_function[function_external_id].add(cron)
+        functions = FunctionLoader(self.client, None).retrieve(list(crons_by_function))
         schedules = FunctionSchedulesList([])
         for func in functions:
             ret = self.client.functions.schedules.list(function_id=func.id, limit=-1)
             for schedule in ret:
                 schedule.function_external_id = func.external_id
-            schedules.extend(ret)
+            schedules.extend(
+                [
+                    schedule
+                    for schedule in ret
+                    if schedule.cron_expression in crons_by_function[cast(str, func.external_id)]
+                ]
+            )
         return schedules
 
     def create(self, items: FunctionScheduleWriteList) -> FunctionSchedulesList:
@@ -1740,12 +1752,6 @@ def load_resource(
                     invalid_parameters,
                 )
 
-            source_oidc_credentials = (
-                resource.get("authentication", {}).get("read") or resource.get("authentication") or None
-            )
-            destination_oidc_credentials = (
-                resource.get("authentication", {}).get("write") or resource.get("authentication") or None
-            )
             if resource.get("dataSetExternalId") is not None:
                 ds_external_id = resource.pop("dataSetExternalId")
                 resource["dataSetId"] = ToolGlobals.verify_dataset(
@@ -1755,8 +1761,13 @@ def load_resource(
                 # Todo; Bug SDK missing default value
                 resource["conflictMode"] = "upsert"
 
+            source_oidc_credentials = (
+                resource.get("authentication", {}).get("read") or resource.get("authentication") or None
+            )
+            destination_oidc_credentials = (
+                resource.get("authentication", {}).get("write") or resource.get("authentication") or None
+            )
             transformation = TransformationWrite.load(resource)
-
             try:
                 transformation.source_oidc_credentials = source_oidc_credentials and OidcCredentials.load(
                     source_oidc_credentials
@@ -1779,7 +1790,8 @@ def load_resource(
                 transformation.query = query_file.read_text()
             elif transformation.query is not None and query_file is not None:
                 raise ToolkitYAMLFormatError(
-                    f"query property is abiguously defined in both the yaml file and a separate file named {query_file}"
+                    f"query property is ambiguously defined in both the yaml file and a separate file named {query_file}\n"
+                    f"Please remove one of the definitions, either the query property in {filepath} or the file {query_file}",
                 )
 
             transformations.append(transformation)
@@ -1957,6 +1969,7 @@ class TransformationNotificationLoader(
     kind = "Notification"
     dependencies = frozenset({TransformationLoader})
     _doc_url = "Transformation-Notifications/operation/createTransformationNotifications"
+    _split_character = "@@@"
 
     @property
     def display_name(self) -> str:
@@ -1968,9 +1981,9 @@ def get_id(cls, item: TransformationNotification | TransformationNotificationWri
             if missing := tuple(k for k in {"transformationExternalId", "destination"} if k not in item):
                 # We need to raise a KeyError with all missing keys to get the correct error message.
                 raise KeyError(*missing)
-            return f"{item['transformationExternalId']}:{item['destination']}"
+            return f"{item['transformationExternalId']}{cls._split_character}{item['destination']}"
 
-        return f"{item.transformation_external_id}:{item.destination}"
+        return f"{item.transformation_external_id}{cls._split_character}{item.destination}"
 
     @classmethod
     def get_required_capability(cls, items: TransformationNotificationWriteList) -> Capability | list[Capability]:
@@ -1979,23 +1992,27 @@ def get_required_capability(cls, items: TransformationNotificationWriteList) ->
         return []
 
     def create(self, items: TransformationNotificationWriteList) -> TransformationNotificationList:
-        # Todo bug in SDK not accepting TransformationNotificationWrite
         return self.client.transformations.notifications.create(items)  # type: ignore[return-value]
 
     def retrieve(self, ids: SequenceNotStr[str]) -> TransformationNotificationList:
         retrieved = TransformationNotificationList([])
         for id_ in ids:
             try:
-                transformation_external_id, destination = id_.split(":")
+                transformation_external_id, destination = id_.rsplit(self._split_character, maxsplit=1)
             except ValueError:
                 # This should never happen, and is a bug in the toolkit if it occurs. Creating a nice error message
                 # here so that if it does happen, it will be easier to debug.
                 raise ValueError(
-                    f"Invalid externalId: {id_}. Must be in the format 'transformationExternalId:destination'"
+                    f"Invalid externalId: {id_}. Must be in the format 'transformationExternalId{self._split_character}destination'"
                 )
-            result = self.client.transformations.notifications.list(
-                transformation_external_id=transformation_external_id, destination=destination, limit=-1
-            )
+            try:
+                result = self.client.transformations.notifications.list(
+                    transformation_external_id=transformation_external_id, destination=destination, limit=-1
+                )
+            except CogniteAPIError:
+                # The notification endpoint gives a 500 if the notification does not exist.
+                # The issue has been reported to the service team.
+                continue
             retrieved.extend(result)
         return retrieved
 
@@ -2021,8 +2038,8 @@ def update(self, items: TransformationNotificationWriteList) -> TransformationNo
         updated_by_id: dict[str, TransformationNotification] = {}
         if create:
             # Bug in SDK
-            created = self.client.transformations.notifications.create(create)  # type: ignore[arg-type]
-            updated_by_id.update({self.get_id(item): item for item in created})  # type: ignore[union-attr]
+            created = self.client.transformations.notifications.create(create)
+            updated_by_id.update({self.get_id(item): item for item in created})
         if unchanged:
             updated_by_id.update({id_: exiting_by_id[id_] for id_ in unchanged})
         return TransformationNotificationList([updated_by_id[id_] for id_ in item_by_id.keys()])
@@ -2139,14 +2156,14 @@ def load_resource(
             resources = [resources]
 
         for resource in resources:
-            if resource.get("dataSetExternalId") is not None:
+            if "dataSetExternalId" in resource:
                 ds_external_id = resource.pop("dataSetExternalId")
                 resource["dataSetId"] = ToolGlobals.verify_dataset(
                     ds_external_id,
                     skip_validation,
                     action="replace datasetExternalId with dataSetId in extraction pipeline",
                 )
-            if resource.get("createdBy") is None:
+            if "createdBy" not in resource:
                 # Todo; Bug SDK missing default value (this will be set on the server-side if missing)
                 resource["createdBy"] = "unknown"
 
@@ -2254,15 +2271,17 @@ def load_resource(
             resources = [resources]
 
         for resource in resources:
-            try:
-                if config := resource.get("config", None):
-                    resource["config"] = yaml.dump(config, indent=4)
-            except Exception:
-                print(
-                    f"[yellow]WARNING:[/] configuration for {resource.get('external_id')} could not be parsed as valid YAML, which is the recommended format.\n"
-                )
-                resource["config"] = resource.get("config", None)
-
+            config_raw = resource.get("config")
+            if isinstance(config_raw, (dict, list)):
+                try:
+                    resource["config"] = yaml.safe_dump(config_raw, indent=4)
+                except yaml.YAMLError as e:
+                    print(
+                        HighSeverityWarning(
+                            f"Configuration for {resource.get('external_id', 'missing')} could not be parsed "
+                            f"as valid YAML, which is the recommended format. Error: {e}"
+                        ).get_message()
+                    )
         if len(resources) == 1:
             return ExtractionPipelineConfigWrite.load(resources[0])
         else:
@@ -2439,8 +2458,7 @@ def load_resource(
         files_metadata: FileMetadataWriteList = FileMetadataWriteList.load(loaded_list)
         for meta in files_metadata:
             if meta.name and not Path(filepath.parent / meta.name).exists():
-                # Todo in `0.2.0` replace this with ToolkitFileNotFoundError
-                raise FileNotFoundError(f"Could not find file {meta.name} referenced in filepath {filepath.name}")
+                raise ToolkitFileNotFoundError(f"Could not find file {meta.name} referenced " f"in filepath {filepath}")
         return files_metadata
 
     def create(self, items: FileMetadataWriteList) -> FileMetadataList:
@@ -2674,27 +2692,22 @@ def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> ContainerApply | ContainerApplyList | None:
         raw_yaml = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables())
-        if not isinstance(raw_yaml, list):
-            raw_yaml = [raw_yaml]
-        # In the Python-SDK, list property of a container.properties.<property>.type.list is required.
-        # This is not the case in the API, so we need to set it here. (This is due to the PropertyType class
-        # is used as read and write in the SDK, and the read class has it required while the write class does not)
-        for raw_instance in raw_yaml:
+        dict_items = raw_yaml if isinstance(raw_yaml, list) else [raw_yaml]
+        for raw_instance in dict_items:
             for prop in raw_instance.get("properties", {}).values():
                 type_ = prop.get("type", {})
                 if "list" not in type_:
+                    # In the Python-SDK, list property of a container.properties.<property>.type.list is required.
+                    # This is not the case in the API, so we need to set it here. (This is due to the PropertyType class
+                    # is used as read and write in the SDK, and the read class has it required while the write class does not)
                     type_["list"] = False
-        items = ContainerApplyList.load(raw_yaml)
-        for item in items:
-            # Todo Bug in SDK, not setting defaults on load
-            for prop_name in item.properties.keys():
-                prop_dumped = item.properties[prop_name].dump()
-                if prop_dumped.get("nullable") is None:
-                    prop_dumped["nullable"] = False  # type: ignore[assignment]
-                if prop_dumped.get("autoIncrement") is None:
-                    prop_dumped["autoIncrement"] = False  # type: ignore[assignment]
-                item.properties[prop_name] = ContainerProperty.load(prop_dumped)
-        return items
+                # Todo Bug in SDK, not setting defaults on load
+                if "nullable" not in prop:
+                    prop["nullable"] = False
+                if "autoIncrement" not in prop:
+                    prop["autoIncrement"] = False
+
+        return ContainerApplyList.load(dict_items)
 
     def create(self, items: Sequence[ContainerApply]) -> ContainerList:
         return self.client.data_modeling.containers.apply(items)

cognite_toolkit/_system.yaml

@@ -25,4 +25,4 @@ packages:
     - example_pump_data_model
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.6
+cdf_toolkit_version: 0.2.7

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.2.6"
+__version__ = "0.2.7"