Release 0.2.5 (#702)

# Description Please describe the change you have made. ## Checklist - [ ] Tests added/updated. - [ ] Run Demo Job Locally. - [ ] Documentation updated. - [ ] Changelogs updated in [CHANGELOG.cdf-tk.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.cdf-tk.md). - [ ] Template changelogs updated in [CHANGELOG.templates.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.templates.md). - [ ] Version bumped. [_version.py](https://github.com/cognitedata/toolkit/blob/main/cognite/cognite_toolkit/_version.py) and [pyproject.toml](https://github.com/cognitedata/toolkit/blob/main/pyproject.toml) per [semantic versioning](https://semver.org/).
cognitedata · Jun 25, 2024 · 50ebd45 · 50ebd45
2 parents a4f6f0d + 5c7bbbc
commit 50ebd45
Show file tree

Hide file tree

Showing 18 changed files with 112 additions and 14 deletions.
diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md
@@ -15,6 +15,19 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.5] - 2024-06-25
+
+### Fixed
+
+- When running `cdf-tk build`, with `RAW` tables in the selected modules, the Toolkit would always warn that the
+  tables were missing, even though they were present. This is now fixed.
+- When running `cdf-tk init --upgrade <YOUR PROJECT>` form version `0.1.4` the user would get a
+  `ERROR (ToolkitMigrationError): Failed to find migration from version 0.1.4.`. This is now fixed.
+- When running `cdf-tk build`, the Toolkit would give you warning when referencing a system `Space`, `View`, `Container`
+  or `DataModel`. This is now fixed.
+- [Feature Preview] When running `cdf-tk import transformation-cli` on a manifest with a query file that
+  is separated from the manifest, the toolkit would raise a `FileNotFoundError`. This is now fixed.
+
 ## [0.2.4] - 2024-06-24
 
 ### Added

diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md
@@ -15,6 +15,10 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.5] - 2024-06-25
+
+No changes to templates.
+
 ## [0.2.4] - 2024-06-24
 
 No changes to templates.

diff --git a/cognite_toolkit/_cdf_tk/_migration.yaml b/cognite_toolkit/_cdf_tk/_migration.yaml
@@ -1,8 +1,13 @@
-- version: 0.2.4
+- version: 0.2.5
   cognite_modules: {}
   resources: {}
   tool: {}
   cognite_modules_hash: ""
+- version: 0.2.4
+  cognite_modules: {}
+  resources: {}
+  tool: {}
+  cognite_modules_hash: "8e482f2aa0bdef71a6e1eb057077589fd3704323052f3ab3a000e1776710620d"
 - version: 0.2.3
   cognite_modules: {}
   resources: {}
@@ -99,6 +104,11 @@
   resources: {}
   tool: {}
   cognite_modules_hash: "4f782ad5c44e093d618df7844cece762954878bf1e0f576acb32975092131346"
+- version: 0.1.4
+  cognite_modules: {}
+  resources: {}
+  tool: {}
+  cognite_modules_hash: "60cde2d26a9ba5ab24ef78fa69c177cf5158e5778f5b2dd0d0d8efdfc5edc332"
 - version: 0.1.3
   cognite_modules: {}
   resources: {}

diff --git a/cognite_toolkit/_cdf_tk/commands/build.py b/cognite_toolkit/_cdf_tk/commands/build.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import datetime
 import difflib
 import io
@@ -43,14 +44,20 @@
 from cognite_toolkit._cdf_tk.hints import ModuleDefinition
 from cognite_toolkit._cdf_tk.loaders import (
     LOADER_BY_FOLDER_NAME,
+    ContainerLoader,
+    DataModelLoader,
     DatapointsLoader,
     FileLoader,
     FileMetadataLoader,
     FunctionLoader,
     GroupLoader,
     Loader,
+    NodeLoader,
     RawDatabaseLoader,
+    RawTableLoader,
     ResourceLoader,
+    SpaceLoader,
+    ViewLoader,
 )
 from cognite_toolkit._cdf_tk.tk_warnings import (
     FileReadWarning,
@@ -281,12 +288,27 @@ def _check_missing_dependencies(self, state: _BuildState, project_config_dir: Pa
         existing = {(resource_cls, id_) for resource_cls, ids in state.ids_by_resource_type.items() for id_ in ids}
         missing_dependencies = set(state.dependencies_by_required.keys()) - existing
         for resource_cls, id_ in missing_dependencies:
+            if self._is_system_resource(resource_cls, id_):
+                continue
             required_by = {
                 (required, path.relative_to(project_config_dir))
                 for required, path in state.dependencies_by_required[(resource_cls, id_)]
             }
             self.warn(MissingDependencyWarning(resource_cls.resource_cls.__name__, id_, required_by))
 
+    @staticmethod
+    def _is_system_resource(resource_cls: type[ResourceLoader], id_: Hashable) -> bool:
+        """System resources are deployed to all CDF project and should not be checked for dependencies."""
+        if resource_cls is SpaceLoader and isinstance(id_, str) and id_.startswith("cdf_"):
+            return True
+        elif (
+            resource_cls in {ContainerLoader, ViewLoader, DataModelLoader, NodeLoader}
+            and hasattr(id_, "space")
+            and id_.space.startswith("cdf_")
+        ):
+            return True
+        return False
+
     @staticmethod
     def _get_selected_variables(
         config_variables: dict[str, Any], module_directories: list[tuple[Path, list[Path]]]
@@ -519,9 +541,7 @@ def validate(
             )
 
         loader = self._get_loader(resource_folder, destination, source_path)
-        if loader is None:
-            return warning_list
-        if not issubclass(loader, ResourceLoader):
+        if loader is None or not issubclass(loader, ResourceLoader):
             return warning_list
 
         api_spec = self._get_api_spec(loader, destination)
@@ -545,6 +565,13 @@ def validate(
                 else:
                     state.ids_by_resource_type[loader][identifier] = source_path
 
+                if loader is RawDatabaseLoader:
+                    # We might also have Raw Tables that is in the same file.
+                    with contextlib.suppress(KeyError):
+                        table_id = RawTableLoader.get_id(item)
+                        if table_id not in state.ids_by_resource_type[RawTableLoader]:
+                            state.ids_by_resource_type[RawTableLoader][table_id] = source_path
+
                 warnings = loader.check_identifier_semantics(identifier, source_path, verbose)
                 warning_list.extend(warnings)
 

diff --git a/cognite_toolkit/_cdf_tk/prototypes/commands/import_.py b/cognite_toolkit/_cdf_tk/prototypes/commands/import_.py
@@ -63,7 +63,14 @@ def execute(
             # keys from the transformation
             schedule = self._convert_schedule(data, data["externalId"], yaml_file)
             notifications = self._convert_notifications(data, data["externalId"], yaml_file)
-            transformation, source_query_path = self._convert_transformation(data, yaml_file)
+            transformation, source_query_relative_path = self._convert_transformation(data, yaml_file)
+            source_query_path = yaml_file.parent / source_query_relative_path if source_query_relative_path else None
+
+            if source_query_path and not source_query_path.exists():
+                raise ToolkitValueError(
+                    f"Query file {source_query_path.as_posix()!r} does not exist. "
+                    f"This is referenced in {yaml_file.as_posix()!r}."
+                )
 
             if flatten:
                 destination_folder = destination

diff --git a/cognite_toolkit/_system.yaml b/cognite_toolkit/_system.yaml
@@ -25,4 +25,4 @@ packages:
     - example_pump_data_model
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.4
+cdf_toolkit_version: 0.2.5
diff --git a/cognite_toolkit/_version.py b/cognite_toolkit/_version.py
@@ -1 +1 @@
-__version__ = "0.2.4"
+__version__ = "0.2.5"
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite_toolkit"
-version = "0.2.4"
+version = "0.2.5"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = ["Cognite AS <support@cognite.com>"]
 license = "Apache-2"

diff --git a/tests/data/project_for_test/_system.yaml b/tests/data/project_for_test/_system.yaml
@@ -4,4 +4,4 @@ packages:
     - child_module
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.4
+cdf_toolkit_version: 0.2.5
diff --git a/tests/data/project_no_cognite_modules/_system.yaml b/tests/data/project_no_cognite_modules/_system.yaml
@@ -3,4 +3,4 @@
 packages: {}
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.4
+cdf_toolkit_version: 0.2.5
diff --git a/tests/data/project_no_cognite_modules/config.dev.yaml b/tests/data/project_no_cognite_modules/config.dev.yaml
@@ -10,5 +10,7 @@ variables:
   modules:
     a_module:
       ds_timeseries: ds_timeseries
+      source: src
+      location: here
     another_module:
       space: my_space
diff --git a/tests/data/project_no_cognite_modules/modules/a_module/data_sets/extraction.DataSet.yaml b/tests/data/project_no_cognite_modules/modules/a_module/data_sets/extraction.DataSet.yaml
@@ -0,0 +1,3 @@
+- externalId: ds_{{source}}_{{location}}_population
+  name: Extraction dataset
+  description: Data set for extraction data
diff --git a/...oject_no_cognite_modules/modules/a_module/extraction_pipelines/my.ExtractionPipeline.yaml b/...oject_no_cognite_modules/modules/a_module/extraction_pipelines/my.ExtractionPipeline.yaml
@@ -0,0 +1,8 @@
+externalId: ep_sap_extraction_pipeline
+name: SAP extraction
+description: SAP extraction
+dataSetExternalId: ds_{{source}}_{{location}}_population
+rawTables:
+- dbName: sap
+  tableName: equipment
+documentation: '## SAP extractor'
diff --git a/tests/data/project_no_cognite_modules/modules/a_module/raw/all.RawTable.yaml b/tests/data/project_no_cognite_modules/modules/a_module/raw/all.RawTable.yaml
@@ -0,0 +1,10 @@
+- dbName: sap
+  tableName: equipment
+- dbName: sap
+  tableName: maintenanceOrder
+- dbName: sap
+  tableName: notification
+- dbName: sap
+  tableName: asset
+- dbName: sap
+  tableName: state_store
diff --git a/tests/data/project_with_bad_modules/_system.yaml b/tests/data/project_with_bad_modules/_system.yaml
@@ -3,4 +3,4 @@
 packages: {}
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.4
+cdf_toolkit_version: 0.2.5
diff --git a/tests/data/run_data/_system.yaml b/tests/data/run_data/_system.yaml
@@ -25,4 +25,4 @@ packages:
     - example_pump_data_model
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.4
+cdf_toolkit_version: 0.2.5
diff --git a/tests/tests_unit/test_cdf_tk/test_commands/test_build.py b/tests/tests_unit/test_cdf_tk/test_commands/test_build.py
@@ -14,7 +14,7 @@
     ToolkitMissingModuleError,
 )
 from cognite_toolkit._cdf_tk.hints import ModuleDefinition
-from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
+from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, MissingDependencyWarning
 from tests import data
 
 
@@ -66,6 +66,20 @@ def test_module_with_non_resource_directories(self, tmp_path: Path) -> None:
             in cmd.warning_list
         )
 
+    def test_warning_on_existing_raw_table(self, tmp_path: Path) -> None:
+        # This test is here to avoid a regression of the issue
+        cmd = BuildCommand(print_warning=False)
+        cmd.execute(
+            verbose=False,
+            build_dir=tmp_path,
+            source_path=data.CUSTOM_PROJECT,
+            build_env_name="dev",
+            no_clean=False,
+        )
+
+        missing_resource_warning = [warn for warn in cmd.warning_list if isinstance(warn, MissingDependencyWarning)]
+        assert not missing_resource_warning, "Missing dependency warning should not be raised"
+
 
 def valid_yaml_semantics_test_cases() -> Iterable[pytest.ParameterSet]:
     yield pytest.param(

diff --git a/tests/tests_unit/test_cli/test_behavior.py b/tests/tests_unit/test_cli/test_behavior.py
@@ -253,7 +253,7 @@ def test_build_custom_project(
     build_tmp_path: Path,
     typer_context: typer.Context,
 ) -> None:
-    expected_resources = {"timeseries", "data_models", "data_sets"}
+    expected_resources = {"timeseries", "data_models", "data_sets", "raw", "extraction_pipelines"}
     build(
         typer_context,
         source_dir=str(CUSTOM_PROJECT),