Release 0.2.10 (#736)

# Description

Please describe the change you have made.

## Checklist

- [ ] Tests added/updated.
- [ ] Run Demo Job Locally.
- [ ] Documentation updated.
- [ ] Changelogs updated in
[CHANGELOG.cdf-tk.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.cdf-tk.md).
- [ ] Template changelogs updated in
[CHANGELOG.templates.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.templates.md).
- [ ] Version bumped.

[_version.py](https://github.com/cognitedata/toolkit/blob/main/cognite/cognite_toolkit/_version.py)
and

[pyproject.toml](https://github.com/cognitedata/toolkit/blob/main/pyproject.toml)
per [semantic versioning](https://semver.org/).

CHANGELOG.cdf-tk.md

@@ -15,6 +15,16 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.10] - 2024-07-03
+
+### Fixed
+
+- When running `cdf-tk build`, if you use subdirectories in a resource directories, and have two resources with the
+  same file name, the Toolkit would overwrite the first resource with the second resource. This is now fixed. For
+  example, if you have `my_module/transformation/subdirectory1/my_transformation.Transformation.yaml` and
+  `my_module/transformation/subdirectory2/my_transformation.Transformation.yaml`, the Toolkit would only build the
+  second resource.
+
 ## [0.2.9] - 2024-07-02
 
 ### Changed

CHANGELOG.templates.md

@@ -15,6 +15,10 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.10] - 2024-07-03
+
+No changes to templates.
+
 ## [0.2.9] - 2024-07-02
 
 No changes to templates.

README.md

@@ -29,10 +29,10 @@ Then run `cdf-tk --help` to get started with the interactive command-line tool.
 ## For more information
 
 More details about the tool can be found at
-[developer.cognite.com](http://developer.cognite.com/sdks/toolkit).
+[docs.cognite.com](https://docs.cognite.com/cdf/deploy/cdf_toolkit/).
 
 You can find an overview of the modules and packages in the
-[module and package documentation](https://developer.cognite.com/sdks/toolkit/references/module_reference).
+[module and package documentation](https://docs.cognite.com/cdf/deploy/cdf_toolkit/references/module_reference).
 
 See [./CONTRIBUTING.md](./CONTRIBUTING.md) for information about how to contribute to the `cdf-tk` tool or
 templates.

cognite_toolkit/_cdf_tk/_migration.yaml

@@ -1,8 +1,13 @@
-- version: 0.2.9
+- version: 0.2.10
   cognite_modules: {}
   resources: {}
   tool: {}
   cognite_modules_hash: ""
+- version: 0.2.9
+  cognite_modules: {}
+  resources: {}
+  tool: {}
+  cognite_modules_hash: "fe2fed61b6d8c5b1968acc66562829d69df1702005979d93eba340fde6cf66be"
 - version: 0.2.8
   cognite_modules: {}
   resources: {}

cognite_toolkit/_cdf_tk/commands/build.py

@@ -294,7 +294,7 @@ def _replace_variables_validate_to_build_directory(
     ) -> Path:
         if verbose:
             print(f"    [bold green]INFO:[/] Processing {source_path.name}")
-        destination = build_dir / resource_directory / state.create_file_name(source_path)
+        destination = build_dir / resource_directory / state.create_file_name(source_path, resource_directory)
         destination.parent.mkdir(parents=True, exist_ok=True)
 
         content = source_path.read_text()
@@ -749,8 +749,15 @@ def local_variables(self) -> Mapping[str, str]:
     def update_local_variables(self, module_dir: Path) -> None:
         self._local_variables = _Helpers.create_local_config(self.variables_by_module_path, module_dir)
 
-    def create_file_name(self, filepath: Path) -> str:
-        return _Helpers.create_file_name(filepath, self.number_by_resource_type)
+    def create_file_name(self, filepath: Path, resource_directory: str) -> str:
+        filename = filepath.name
+        if filepath.suffix in EXCL_INDEX_SUFFIX:
+            return filename
+        # Get rid of the local index
+        filename = re.sub("^[0-9]+\\.", "", filename)
+        self.number_by_resource_type[resource_directory] += 1
+        filename = f"{self.number_by_resource_type[resource_directory]}.{filename}"
+        return filename
 
     def replace_variables(self, content: str, file_suffix: str = ".yaml") -> str:
         for name, variable in self.local_variables.items():
@@ -819,14 +826,3 @@ def _split_config(cls, config: dict[str, Any], configs: dict[str, dict[str, str]
                 cls._split_config(value, configs, prefix=f"{prefix}{key}")
             else:
                 configs.setdefault(prefix.removesuffix("."), {})[key] = value
-
-    @staticmethod
-    def create_file_name(filepath: Path, number_by_resource_type: dict[str, int]) -> str:
-        filename = filepath.name
-        if filepath.suffix in EXCL_INDEX_SUFFIX:
-            return filename
-        # Get rid of the local index
-        filename = re.sub("^[0-9]+\\.", "", filename)
-        number_by_resource_type[filepath.parent.name] += 1
-        filename = f"{number_by_resource_type[filepath.parent.name]}.{filename}"
-        return filename

cognite_toolkit/_system.yaml

@@ -25,4 +25,4 @@ packages:
     - example_pump_data_model
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.9
+cdf_toolkit_version: 0.2.10

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.2.9"
+__version__ = "0.2.10"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite_toolkit"
-version = "0.2.9"
+version = "0.2.10"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = ["Cognite AS <support@cognite.com>"]
 license = "Apache-2"

tests/data/project_for_test/_system.yaml

@@ -4,4 +4,4 @@ packages:
     - child_module
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.9
+cdf_toolkit_version: 0.2.10

tests/data/project_no_cognite_modules/_system.yaml

@@ -3,4 +3,4 @@
 packages: {}
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.9
+cdf_toolkit_version: 0.2.10

tests/data/project_no_cognite_modules/config.dev.yaml

@@ -13,3 +13,6 @@ variables:
       location: here
     another_module:
       space: sp_my_space
+      source: src2
+      ds_something: ds_something
+

tests/data/project_no_cognite_modules/modules/another_module/data_sets/datasets.yaml

@@ -0,0 +1,3 @@
+externalId: {{ds_something}}
+name: {{ds_something}}
+description: Data set for something

tests/data/project_no_cognite_modules/modules/another_module/transformations/subfolder/my.Transformation.sql

@@ -0,0 +1,8 @@
+select
+  cast(`externalId` as STRING) as externalId,
+  cast(`tag` as STRING) as name,
+  cast(`description` as STRING) as description,
+  cast(`sourceDb` as STRING) as source,
+  cast(`parentExternalId` as STRING) as parentExternalId
+from
+  `asset_{{location}`.`assets`;

tests/data/project_no_cognite_modules/modules/another_module/transformations/subfolder/my.Transformation.yaml

@@ -0,0 +1,8 @@
+externalId: tr_cdf:instances:upsert:tr:300b:exact_ais_v2_vessels
+name: cdf:instances:upsert:tr:300b:exact_ais_v2_vessels
+destination:
+  type: "asset_hierarchy"
+ignoreNullFields: true
+isPublic: true
+conflictMode: upsert
+dataSetExternalId: {{ds_something}}

tests/data/project_no_cognite_modules/modules/another_module/transformations/subfolder/some.Notification.yaml

@@ -0,0 +1,2 @@
+- transformationExternalId: tr_cdf:instances:upsert:tr:300b:exact_ais_v2_vessels
+  destination: example@cognite.com

tests/data/project_with_bad_modules/_system.yaml

@@ -3,4 +3,4 @@
 packages: {}
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.9
+cdf_toolkit_version: 0.2.10

tests/data/run_data/_system.yaml

@@ -25,4 +25,4 @@ packages:
     - example_pump_data_model
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.9
+cdf_toolkit_version: 0.2.10

tests/tests_unit/test_cdf_tk/test_commands/test_build.py

@@ -15,6 +15,7 @@
     ToolkitMissingModuleError,
 )
 from cognite_toolkit._cdf_tk.hints import ModuleDefinition
+from cognite_toolkit._cdf_tk.loaders import TransformationLoader
 from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
 from tests import data
 
@@ -79,6 +80,13 @@ def test_custom_project_no_warnings(self, tmp_path: Path, monkeypatch: MonkeyPat
         )
 
         assert not cmd.warning_list, f"No warnings should be raised. Got warnings: {cmd.warning_list}"
+        # There are two transformations in the project, expect two transformation files
+        transformation_files = [
+            f
+            for f in (tmp_path / "transformations").iterdir()
+            if f.is_file() and TransformationLoader.is_supported_file(f)
+        ]
+        assert len(transformation_files) == 2
 
 
 def valid_yaml_semantics_test_cases() -> Iterable[pytest.ParameterSet]: