Release 0.2.0b4 (#618)

# Description Please describe the change you have made. ## Checklist - [ ] Tests added/updated. - [ ] Run Demo Job Locally. - [ ] Documentation updated. - [ ] Changelogs updated in [CHANGELOG.cdf-tk.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.cdf-tk.md). - [ ] Template changelogs updated in [CHANGELOG.templates.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.templates.md). - [ ] Version bumped. [_version.py](https://github.com/cognitedata/toolkit/blob/main/cognite/cognite_toolkit/_version.py) and [pyproject.toml](https://github.com/cognitedata/toolkit/blob/main/pyproject.toml) per [semantic versioning](https://semver.org/).
cognitedata · Jun 6, 2024 · f0ad9cf · f0ad9cf
2 parents 2126890 + 4f9a404
commit f0ad9cf
Show file tree

Hide file tree

Showing 50 changed files with 701 additions and 404 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -116,6 +116,8 @@
                 //"--env=local",
                 //"--include=transformations"
                 //"--init-dir=foo"
+                //"--arrow"
+                //"--help"
             ],
             "console": "integratedTerminal",
             "justMyCode": false

diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md
@@ -15,6 +15,28 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.0b4] - 2024-06-06
+
+### Added
+
+- Support for resource type `TransformationNotification` in the `transformations` folder.
+
+### Changed
+
+- [BREAKING] In `functions`, the function config file must be in the root function directory. This means
+  that, for example, `my_module/function/some_folder/function.yaml` will no longer be included by
+  the `cdf-tk build` command. Instead, it must be in `my_module/function/function.yaml`. The motivation
+  is to allow arbitrary YAML files as part of the function code.
+- The toolkit now only gives a `TemplateVariableWarning` (`Variable my_variable has value <change_me> ...`) if
+  the variable is used by `selected` in the `config.[env].yaml`. This is to avoid unnecessary warnings.
+- The `FeaturePrevieWarnings` are no longer printed when running `cdf-tk deploy` or `cdf-tk clean`. These warnings
+  are from the `cognite-sdk` and can be confusing to the user.
+
+### Fixed
+
+- When running `cdf-tk init --upgrade` from version `0.1.4` the user would get a `ToolkitMigrationError`.
+  This is now fixed.
+
 ## [0.2.0b3] - 2024-06-04
 
 ### Added

diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md
@@ -15,6 +15,12 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.0b4] - 2024-06-06
+
+### Added
+
+- Extended `cognite_modules/examples/my_example_module` with a `TransformationNotification` case.
+
 ## [0.2.0b3] - 2024-06-04
 
 ### Added

diff --git a/cdf-tk-dev.py b/cdf-tk-dev.py
@@ -29,7 +29,7 @@
 import sys
 from pathlib import Path
 
-from cognite_toolkit._cdf_tk.templates.data_classes import Environment, InitConfigYAML
+from cognite_toolkit._cdf_tk.data_classes import Environment, InitConfigYAML
 
 REPO_ROOT = Path(__file__).resolve().parent
 root_folder = rf"{REPO_ROOT.absolute()}"

diff --git a/cognite_toolkit/_api/data_classes.py b/cognite_toolkit/_api/data_classes.py
@@ -11,9 +11,9 @@
 
 import pandas as pd
 
+from cognite_toolkit._cdf_tk.constants import COGNITE_MODULES_PATH
+from cognite_toolkit._cdf_tk.data_classes import ConfigEntry, Environment, InitConfigYAML
 from cognite_toolkit._cdf_tk.loaders import LOADER_BY_FOLDER_NAME
-from cognite_toolkit._cdf_tk.templates import COGNITE_MODULES_PATH
-from cognite_toolkit._cdf_tk.templates.data_classes import ConfigEntry, Environment, InitConfigYAML
 
 NOT_SET = object()
 

diff --git a/cognite_toolkit/_api/modules_api.py b/cognite_toolkit/_api/modules_api.py
@@ -14,9 +14,10 @@
 from cognite_toolkit._api.data_classes import _DUMMY_ENVIRONMENT, ModuleMeta, ModuleMetaList
 from cognite_toolkit._cdf import Common, clean, deploy
 from cognite_toolkit._cdf_tk.commands.build import BuildCommand
+from cognite_toolkit._cdf_tk.constants import COGNITE_MODULES, COGNITE_MODULES_PATH
+from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, Environment, InitConfigYAML, SystemYAML
 from cognite_toolkit._cdf_tk.loaders import ResourceTypes
-from cognite_toolkit._cdf_tk.templates import COGNITE_MODULES, COGNITE_MODULES_PATH, iterate_modules
-from cognite_toolkit._cdf_tk.templates.data_classes import BuildConfigYAML, Environment, InitConfigYAML, SystemYAML
+from cognite_toolkit._cdf_tk.utils import iterate_modules
 
 
 class ModulesAPI:

diff --git a/cognite_toolkit/_cdf.py b/cognite_toolkit/_cdf.py
@@ -25,6 +25,10 @@
     RunFunctionCommand,
     RunTransformationCommand,
 )
+from cognite_toolkit._cdf_tk.data_classes import (
+    ProjectDirectoryInit,
+    ProjectDirectoryUpgrade,
+)
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitError,
     ToolkitFileNotFoundError,
@@ -37,10 +41,6 @@
     TransformationLoader,
 )
 from cognite_toolkit._cdf_tk.prototypes import featureflag
-from cognite_toolkit._cdf_tk.templates.data_classes import (
-    ProjectDirectoryInit,
-    ProjectDirectoryUpgrade,
-)
 from cognite_toolkit._cdf_tk.utils import (
     CDFToolConfig,
     sentry_exception_filter,

diff --git a/...toolkit/_cdf_tk/templates/_migration.yaml → cognite_toolkit/_cdf_tk/_migration.yaml b/...toolkit/_cdf_tk/templates/_migration.yaml → cognite_toolkit/_cdf_tk/_migration.yaml
@@ -1,8 +1,20 @@
-- version: 0.2.0b3
+- version: 0.2.0b4
   cognite_modules: {}
   resources: {}
   tool: {}
   cognite_modules_hash: ""
+- version: 0.2.0b3
+  cognite_modules: {}
+  resources: {}
+  tool:
+    - title: "Function config YAML files must be in the function directori"
+      steps:
+        - Before functions config YAML files were allowed in subdirectories of the function directory. For example,
+          `functions/my_function/my_function.yaml` was allowed. This is no longer supported. The function config YAML
+          files must be in the function directory. For example, `functions/my_function.yaml`.
+        - The motivation is to allow having YAML files as part of the function code directory.
+        - If you have functions with config files in subdirectories, move them to the function directory.
+  cognite_modules_hash: "f0db57ca9f0a3ebad27eb9c348df271aeb7c7ce73e8080ecb45ecf6e2a32dcd1"
 - version: 0.2.0b2
   cognite_modules: {}
   resources: {}

diff --git a/cognite_toolkit/_cdf_tk/commands/auth.py b/cognite_toolkit/_cdf_tk/commands/auth.py
@@ -30,10 +30,8 @@
 from rich.prompt import Confirm, Prompt
 from rich.table import Table
 
+from cognite_toolkit._cdf_tk.constants import COGNITE_MODULES
 from cognite_toolkit._cdf_tk.exceptions import ToolkitInvalidSettingsError, ToolkitValidationError
-from cognite_toolkit._cdf_tk.templates import (
-    COGNITE_MODULES,
-)
 from cognite_toolkit._cdf_tk.tk_warnings import (
     HighSeverityWarning,
     LowSeverityWarning,

diff --git a/cognite_toolkit/_cdf_tk/commands/build.py b/cognite_toolkit/_cdf_tk/commands/build.py
@@ -7,7 +7,7 @@
 import sys
 import traceback
 from collections import ChainMap, defaultdict
-from collections.abc import Hashable, Mapping
+from collections.abc import Hashable, Iterator, Mapping, Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
@@ -21,7 +21,16 @@
 
 from cognite_toolkit._cdf_tk._parameters import ParameterSpecSet
 from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
-from cognite_toolkit._cdf_tk.constants import _RUNNING_IN_BROWSER
+from cognite_toolkit._cdf_tk.constants import (
+    _RUNNING_IN_BROWSER,
+    EXCL_INDEX_SUFFIX,
+    PROC_TMPL_VARS_SUFFIX,
+    ROOT_MODULES,
+)
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildConfigYAML,
+    SystemYAML,
+)
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitDuplicatedModuleError,
     ToolkitFileExistsError,
@@ -37,17 +46,6 @@
     Loader,
     ResourceLoader,
 )
-from cognite_toolkit._cdf_tk.templates._constants import EXCL_INDEX_SUFFIX, PROC_TMPL_VARS_SUFFIX, ROOT_MODULES
-from cognite_toolkit._cdf_tk.templates._utils import (
-    iterate_functions,
-    iterate_modules,
-    module_from_path,
-    resource_folder_from_path,
-)
-from cognite_toolkit._cdf_tk.templates.data_classes import (
-    BuildConfigYAML,
-    SystemYAML,
-)
 from cognite_toolkit._cdf_tk.tk_warnings import (
     FileReadWarning,
     HighSeverityWarning,
@@ -60,7 +58,12 @@
     WarningList,
 )
 from cognite_toolkit._cdf_tk.tk_warnings.fileread import DuplicatedItemWarning, MissingRequiredIdentifierWarning
-from cognite_toolkit._cdf_tk.utils import calculate_str_or_file_hash
+from cognite_toolkit._cdf_tk.utils import (
+    calculate_str_or_file_hash,
+    iterate_modules,
+    module_from_path,
+    resource_folder_from_path,
+)
 from cognite_toolkit._cdf_tk.validation import (
     validate_data_set_is_set,
     validate_modules_variables,
@@ -140,13 +143,20 @@ def build_config(
 
         selected_modules = config.get_selected_modules(system_config.packages, available_modules, verbose)
 
-        warnings = validate_modules_variables(config.variables, config.filepath)
+        module_directories = [
+            (module_dir, source_paths)
+            for module_dir, source_paths in iterate_modules(source_dir)
+            if self._is_selected_module(module_dir.relative_to(source_dir), selected_modules)
+        ]
+        selected_variables = self._get_selected_variables(config.variables, module_directories)
+
+        warnings = validate_modules_variables(selected_variables, config.filepath)
         if warnings:
             self.warn(LowSeverityWarning(f"Found the following warnings in config.{config.environment.name}.yaml:"))
             for warning in warnings:
                 print(f"    {warning.get_message()}")
 
-        state = self.process_config_files(source_dir, selected_modules, build_dir, config, verbose)
+        state = self.process_config_files(source_dir, module_directories, build_dir, config, verbose)
 
         build_environment = config.create_build_environment(state.hash_by_source_path)
         build_environment.dump_to_file(build_dir)
@@ -157,15 +167,13 @@ def build_config(
     def process_config_files(
         self,
         project_config_dir: Path,
-        selected_modules: list[str | tuple[str, ...]],
+        module_directories: Sequence[tuple[Path, list[Path]]],
         build_dir: Path,
         config: BuildConfigYAML,
         verbose: bool = False,
     ) -> _BuildState:
         state = _BuildState.create(config)
-        for module_dir, source_paths in iterate_modules(project_config_dir):
-            if not self._is_selected_module(module_dir.relative_to(project_config_dir), selected_modules):
-                continue
+        for module_dir, source_paths in module_directories:
             if verbose:
                 print(f"  [bold green]INFO:[/] Processing module {module_dir.name}")
 
@@ -180,10 +188,11 @@ def process_config_files(
                     destination = build_dir / resource_folder / state.create_file_name(source_path)
                     destination.parent.mkdir(parents=True, exist_ok=True)
 
-                    is_function_non_yaml = (
-                        resource_folder == FunctionLoader.folder_name and source_path.suffix.lower() != ".yaml"
+                    is_function_non_yaml = resource_folder == FunctionLoader.folder_name and (
+                        source_path.suffix.lower() != ".yaml" or source_path.parent.name != FunctionLoader.folder_name
                     )
                     # We only want to process the yaml files for functions as the function code is handled separately.
+                    # Note that yaml files that are NOT in the root function folder are considered function code.
                     if not is_function_non_yaml:
                         content = source_path.read_text()
                         state.hash_by_source_path[source_path] = calculate_str_or_file_hash(content)
@@ -256,6 +265,30 @@ def _check_missing_dependencies(self, state: _BuildState, project_config_dir: Pa
             }
             self.warn(MissingDependencyWarning(resource_cls.resource_cls.__name__, id_, required_by))
 
+    @staticmethod
+    def _get_selected_variables(
+        config_variables: dict[str, Any], module_directories: list[tuple[Path, list[Path]]]
+    ) -> dict[str, Any]:
+        selected_paths = {
+            dir_.parts[1:i]
+            for dir_, _ in module_directories
+            if len(dir_.parts) > 1
+            for i in range(2, len(dir_.parts) + 1)
+        }
+        selected_variables: dict[str, Any] = {}
+        to_check: list[tuple[tuple[str, ...], dict[str, Any]]] = [(tuple(), config_variables)]
+        while to_check:
+            path, current = to_check.pop()
+            for key, value in current.items():
+                if isinstance(value, dict):
+                    to_check.append(((*path, key), value))
+                elif path in selected_paths:
+                    selected = selected_variables
+                    for part in path:
+                        selected = selected.setdefault(part, {})
+                    selected[key] = value
+        return selected_variables
+
     @staticmethod
     def _is_selected_module(relative_module_dir: Path, selected_modules: list[str | tuple[str, ...]]) -> bool:
         module_parts = relative_module_dir.parts
@@ -321,14 +354,22 @@ def process_function_directory(
         build_dir: Path,
         verbose: bool = False,
     ) -> None:
+        if yaml_source_path.parent.name != FunctionLoader.folder_name:
+            self.warn(
+                LowSeverityWarning(
+                    f"The file {yaml_source_path} is considered part of the Function's code and will not be processed as a CDF resource. If this is a "
+                    f"function config please move it to {FunctionLoader.folder_name} folder."
+                )
+            )
+            return None
         try:
             functions: FunctionList = FunctionList.load(yaml.safe_load(yaml_dest_path.read_text()))
         except (KeyError, yaml.YAMLError) as e:
             raise ToolkitYAMLFormatError(f"Failed to load function file {yaml_source_path} due to: {e}")
 
         for func in functions:
             found = False
-            for function_subdirs in iterate_functions(module_dir):
+            for function_subdirs in self.iterate_functions(module_dir):
                 for function_dir in function_subdirs:
                     if (fn_xid := func.external_id) == function_dir.name:
                         found = True
@@ -515,6 +556,15 @@ def _get_loader(self, resource_folder: str, destination: Path) -> type[Loader] |
             )
         return loader
 
+    @staticmethod
+    def iterate_functions(module_dir: Path) -> Iterator[list[Path]]:
+        for function_dir in module_dir.glob(f"**/{FunctionLoader.folder_name}"):
+            if not function_dir.is_dir():
+                continue
+            function_directories = [path for path in function_dir.iterdir() if path.is_dir()]
+            if function_directories:
+                yield function_directories
+
 
 @dataclass
 class _BuildState:

diff --git a/cognite_toolkit/_cdf_tk/commands/clean.py b/cognite_toolkit/_cdf_tk/commands/clean.py
@@ -12,6 +12,10 @@
 from rich.panel import Panel
 
 from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
+from cognite_toolkit._cdf_tk.constants import BUILD_ENVIRONMENT_FILE
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildEnvironment,
+)
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitCleanResourceError,
     ToolkitNotADirectoryError,
@@ -25,12 +29,6 @@
 )
 from cognite_toolkit._cdf_tk.loaders._base_loaders import T_ID, Loader
 from cognite_toolkit._cdf_tk.loaders.data_classes import ResourceContainerDeployResult, ResourceDeployResult
-from cognite_toolkit._cdf_tk.templates import (
-    BUILD_ENVIRONMENT_FILE,
-)
-from cognite_toolkit._cdf_tk.templates.data_classes import (
-    BuildEnvironment,
-)
 from cognite_toolkit._cdf_tk.tk_warnings import (
     LowSeverityWarning,
     MediumSeverityWarning,

diff --git a/cognite_toolkit/_cdf_tk/commands/deploy.py b/cognite_toolkit/_cdf_tk/commands/deploy.py
@@ -13,7 +13,10 @@
 
 from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
 from cognite_toolkit._cdf_tk.commands.clean import CleanCommand
-from cognite_toolkit._cdf_tk.constants import _RUNNING_IN_BROWSER
+from cognite_toolkit._cdf_tk.constants import _RUNNING_IN_BROWSER, BUILD_ENVIRONMENT_FILE
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildEnvironment,
+)
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitCleanResourceError,
     ToolkitDeployResourceError,
@@ -34,12 +37,6 @@
     ResourceDeployResult,
     UploadDeployResult,
 )
-from cognite_toolkit._cdf_tk.templates import (
-    BUILD_ENVIRONMENT_FILE,
-)
-from cognite_toolkit._cdf_tk.templates.data_classes import (
-    BuildEnvironment,
-)
 from cognite_toolkit._cdf_tk.tk_warnings.other import (
     LowSeverityWarning,
     MediumSeverityWarning,

diff --git a/cognite_toolkit/_cdf_tk/commands/pull.py b/cognite_toolkit/_cdf_tk/commands/pull.py
@@ -16,14 +16,14 @@
 from rich.panel import Panel
 
 from cognite_toolkit._cdf_tk.commands.build import BuildCommand
+from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, SystemYAML
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitDuplicatedResourceError,
     ToolkitMissingResourceError,
     ToolkitNotADirectoryError,
 )
 from cognite_toolkit._cdf_tk.loaders import ResourceLoader
 from cognite_toolkit._cdf_tk.loaders._base_loaders import T_ID, T_WritableCogniteResourceList
-from cognite_toolkit._cdf_tk.templates.data_classes import BuildConfigYAML, SystemYAML
 from cognite_toolkit._cdf_tk.utils import CDFToolConfig, YAMLComment, YAMLWithComments, tmp_build_directory
 
 from ._base import ToolkitCommand