Release 0.2.0a2 (#441)

# Description

Please describe the change you have made.

## Checklist

- [ ] Tests added/updated.
- [ ] Run Demo Job Locally.
- [ ] Documentation updated.
- [ ] Changelogs updated in
[CHANGELOG.cdf-tk.md](https://github.com/cognitedata/cdf-project-templates/blob/main/CHANGELOG.cdf-tk.md).
- [ ] Template changelogs updated in
[CHANGELOG.templates.md](https://github.com/cognitedata/cdf-project-templates/blob/main/CHANGELOG.templates.md).
- [ ] Version bumped.

[_version.py](https://github.com/cognitedata/cdf-project-templates/blob/main/cognite/cognite_toolkit/_version.py)
and

[pyproject.toml](https://github.com/cognitedata/cdf-project-templates/blob/main/pyproject.toml)
per [semantic versioning](https://semver.org/).

.github/workflows/build.yml

@@ -53,6 +53,8 @@ jobs:
         run: |
           poetry install
       - name: Run pytest
+        env:
+          IS_GITHUB_ACTIONS: "true"
         run: pytest tests
   coverage:
     needs: test
@@ -78,11 +80,14 @@ jobs:
         run: |
           poetry install
       - name: Create test coverage report
+        env:
+          IS_GITHUB_ACTIONS: "true"
         run: pytest --cov=cognite_toolkit/ --cov-config=pyproject.toml --cov-report=xml:coverage.xml tests/
       - name: Push coverage report to PR
         uses: orgoro/coverage@v3.1
         with:
             coverageFile: coverage.xml
+            thresholdAll: 0.6
             token: ${{ secrets.GITHUB_TOKEN }}
   dry-run-demo:
     runs-on: ubuntu-latest

.github/workflows/release.yaml

@@ -45,6 +45,8 @@ jobs:
       - name: Install core dependencies
         run: poetry install
       - name: Run pytest
+        env:
+          IS_GITHUB_ACTIONS: "true"
         run: pytest tests
   build:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -12,10 +12,10 @@ repos:
           - --select=E,W,F,I,T,RUF,TID,UP
           - --fixable=E,W,F,I,T,RUF,TID,UP
           - --target-version=py39
-    rev: v0.2.2
+    rev: v0.3.4
 
   - repo: https://github.com/psf/black
-    rev: 24.2.0
+    rev: 24.3.0
     hooks:
       - id: black
         args:

CHANGELOG.cdf-tk.md

@@ -15,6 +15,27 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.0a2] - 2024-04-03
+
+### Added
+
+- Variables can now have extra spaces between curly braces and the variable name. For example, `{{  my_variable }}` is now
+  a valid variable. Before this change, you would have to write `{{my_variable}}`.
+- If an environment variable is not found in a resource file, for example, `${CDF_CLUSTER}`, when
+  running `cdf-tk deploy` the user will now get a warning message that the variable is missing. Before this change,
+  this would pass silently and potentially cause an error when trying to deploy to CDF that was hard to debug.
+
+### Fixed
+
+- When running `cdf-tk` with a Token for initialization, the `cdf-tk` would raise an `IndexError`. This is now fixed.
+- Container resources that did not have set the optional property `usedFor` would always be classified as changed,
+  when, for example, running `cdf-tk deploy --dry-run`. This is now fixed.
+
+### Changed
+
+- If two modules have the same name, the `cdf-tk build` command will now stop and raise an error. Before this change,
+  the `cdf-tk build` command would continue and overwrite the first module with the second module.
+
 ## [0.2.0a1] - 2024-03-20
 
 ### Added

CHANGELOG.templates.md

@@ -15,6 +15,10 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.0a2] - 2024-04-03
+
+No changes to templates.
+
 ## [0.2.0a1] - 2024-03-20
 
 - Added functionality for wildcard detection of tags in P&ID

cognite_toolkit/_cdf.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # The Typer parameters get mixed up if we use the __future__ import annotations
+import contextlib
 import os
 import sys
 from collections.abc import Sequence
@@ -55,7 +56,24 @@
         traces_sample_rate=1.0,
     )
 
-app = typer.Typer(pretty_exceptions_short=False, pretty_exceptions_show_locals=False, pretty_exceptions_enable=False)
+try:
+    app = typer.Typer(
+        pretty_exceptions_short=False, pretty_exceptions_show_locals=False, pretty_exceptions_enable=False
+    )
+except AttributeError as e:
+    # From Typer version 0.11 -> 0.12, breaks if you have an existing installation.
+    print(
+        "'cognite-toolkit' uses a dependency named 'typer'.",
+        "From 'typer' version 0.11 -> 0.12 there was a breaking change if you have an existing "
+        "installation of 'typer'.",
+        "Workaround is to uninstall 'typer-slim', and then, reinstall 'typer':",
+        "'pip uninstall typer-slim'",
+        "'pip install typer'",
+        sep="\n",
+    )
+    print("\nThis was triggered by the error:", e)
+    exit(1)
+
 auth_app = typer.Typer(
     pretty_exceptions_short=False, pretty_exceptions_show_locals=False, pretty_exceptions_enable=False
 )
@@ -175,8 +193,10 @@ def common(
                 path_str = dotenv_file.relative_to(Path.cwd())
             except ValueError:
                 path_str = dotenv_file.absolute()
-            print(f"Loading .env file: {path_str!s}")
-        load_dotenv(dotenv_file, override=override_env)
+            print(f"Loading .env file: {path_str!s}.")
+        has_loaded = load_dotenv(dotenv_file, override=override_env)
+        if not has_loaded:
+            print("  [bold yellow]WARNING:[/] No environment variables found in .env file.")
 
     ctx.obj = Common(
         verbose=verbose,
@@ -227,7 +247,9 @@ def build(
     if not source_path.is_dir():
         print(f"  [bold red]ERROR:[/] {source_path} does not exist")
         exit(1)
-    system_config = SystemYAML.load_from_directory(source_path / COGNITE_MODULES, build_env)
+    cognite_modules_path = source_path / COGNITE_MODULES
+
+    system_config = SystemYAML.load_from_directory(cognite_modules_path, build_env)
     config = BuildConfigYAML.load_from_directory(source_path, build_env)
     print(
         Panel(
@@ -630,7 +652,11 @@ def auth_verify(
     if create_group is not None and update_group != 0:
         print("[bold red]ERROR: [/] --create-group and --update-group are mutually exclusive.")
         exit(1)
-    ToolGlobals = CDFToolConfig.from_context(ctx)
+    with contextlib.redirect_stdout(None):
+        # Remove the Error message from failing to load the config
+        # This is verified in check_auth
+        ToolGlobals = CDFToolConfig.from_context(ctx)
+
     if group_file is None:
         template_dir = cast(Path, resources.files("cognite_toolkit"))
         group_path = template_dir.joinpath(

cognite_toolkit/_cdf_tk/load/_resource_loaders.py

@@ -1655,6 +1655,15 @@ def _iterate_over_edges(self, containers: ContainerList) -> Iterable[list[EdgeId
     def _chunker(seq: Sequence, size: int) -> Iterable[Sequence]:
         return (seq[pos : pos + size] for pos in range(0, len(seq), size))
 
+    def _is_equal_custom(self, local: ContainerApply, remote: Container) -> bool:
+        local_dumped = local.dump(camel_case=True)
+        if "usedFor" not in local_dumped:
+            # Setting used_for to "node" as it is the default value in the CDF and will be set by
+            # the server side if it is not set.
+            local_dumped["usedFor"] = "node"
+
+        return local_dumped == remote.as_write().dump(camel_case=True)
+
 
 class ViewLoader(ResourceLoader[ViewId, ViewApply, View, ViewApplyList, ViewList]):
     api_name = "data_modeling.views"

cognite_toolkit/_cdf_tk/templates/__init__.py

@@ -1,4 +1,10 @@
-from ._constants import BUILD_ENVIRONMENT_FILE, COGNITE_MODULES, COGNITE_MODULES_PATH, CUSTOM_MODULES
+from ._constants import (
+    BUILD_ENVIRONMENT_FILE,
+    COGNITE_MODULES,
+    COGNITE_MODULES_PATH,
+    CUSTOM_MODULES,
+    ROOT_MODULES,
+)
 from ._templates import build_config, check_yaml_semantics, create_local_config, split_config
 from ._utils import flatten_dict, iterate_modules, module_from_path, resource_folder_from_path
 
@@ -8,6 +14,7 @@
     "resource_folder_from_path",
     "COGNITE_MODULES",
     "CUSTOM_MODULES",
+    "ROOT_MODULES",
     "COGNITE_MODULES_PATH",
     "build_config",
     "BUILD_ENVIRONMENT_FILE",

cognite_toolkit/_cdf_tk/templates/_constants.py

@@ -12,6 +12,8 @@
 COGNITE_MODULES = "cognite_modules"
 CUSTOM_MODULES = "custom_modules"
 
+ROOT_MODULES = [COGNITE_MODULES, CUSTOM_MODULES]
+
 # Add any other files below that should be included in a build
 EXCL_FILES = ["README.md", DEFAULT_CONFIG_FILE]
 # Which suffixes to exclude when we create indexed files (i.e., they are bundled with their main config file)

cognite_toolkit/_cdf_tk/templates/_migration.yaml

@@ -1,8 +1,13 @@
-- version: 0.2.0a1
+- version: 0.2.0a2
   cognite_modules: {}
   resources: {}
   tool: {}
   cognite_modules_hash: ""
+- version: 0.2.0a1
+  cognite_modules: {}
+  resources: {}
+  tool: {}
+  cognite_modules_hash: "4f782ad5c44e093d618df7844cece762954878bf1e0f576acb32975092131346"
 - version: 0.1.2
   cognite_modules: {}
   resources: {}

cognite_toolkit/_cdf_tk/templates/_templates.py

@@ -22,7 +22,11 @@
 from cognite_toolkit._cdf_tk.load import LOADER_BY_FOLDER_NAME, FunctionLoader, Loader, ResourceLoader
 from cognite_toolkit._cdf_tk.utils import validate_case_raw, validate_data_set_is_set, validate_modules_variables
 
-from ._constants import COGNITE_MODULES, CUSTOM_MODULES, EXCL_INDEX_SUFFIX, PROC_TMPL_VARS_SUFFIX
+from ._constants import (
+    EXCL_INDEX_SUFFIX,
+    PROC_TMPL_VARS_SUFFIX,
+    ROOT_MODULES,
+)
 from ._utils import iterate_functions, iterate_modules, module_from_path, resource_folder_from_path
 from .data_classes import BuildConfigYAML, SystemYAML
 
@@ -50,7 +54,18 @@ def build_config(
 
     config.validate_environment()
 
-    available_modules = {module.name for module, _ in iterate_modules(source_dir)}
+    module_name_by_path = defaultdict(list)
+    for module, _ in iterate_modules(source_dir):
+        module_name_by_path[module.name].append(module.relative_to(source_dir))
+    if duplicate_modules := {
+        module_name: paths for module_name, paths in module_name_by_path.items() if len(paths) > 1
+    }:
+        print(f"  [bold red]ERROR:[/] Found the following duplicated module names in {source_dir.name}:")
+        for module_name, paths in duplicate_modules.items():
+            print(f"    {module_name}: {paths}")
+        exit(1)
+
+    available_modules = set(module_name_by_path.keys())
     system_config.validate_modules(available_modules, config.environment.selected_modules_and_packages)
 
     selected_modules = config.get_selected_modules(system_config.packages, available_modules, verbose)
@@ -530,10 +545,9 @@ class ResourceFiles:
 def create_local_config(config: dict[str, Any], module_dir: Path) -> Mapping[str, str]:
     maps = []
     parts = module_dir.parts
-    if parts[0] != COGNITE_MODULES and COGNITE_MODULES in parts:
-        parts = parts[parts.index(COGNITE_MODULES) :]
-    if parts[0] != CUSTOM_MODULES and CUSTOM_MODULES in parts:
-        parts = parts[parts.index(CUSTOM_MODULES) :]
+    for root_module in ROOT_MODULES:
+        if parts[0] != root_module and root_module in parts:
+            parts = parts[parts.index(root_module) :]
     for no in range(len(parts), -1, -1):
         if c := config.get(".".join(parts[:no])):
             maps.append(c)
@@ -569,7 +583,7 @@ def create_file_name(filepath: Path, number_by_resource_type: dict[str, int]) ->
 
 def replace_variables(content: str, local_config: Mapping[str, str]) -> str:
     for name, variable in local_config.items():
-        content = content.replace(f"{{{{{name}}}}}", str(variable))
+        content = re.sub(rf"{{{{\s*{name}\s*}}}}", str(variable), content)
     return content
 
 

cognite_toolkit/_cdf_tk/templates/_utils.py

@@ -36,6 +36,8 @@ def iterate_modules(root_dir: Path) -> Iterator[tuple[Path, list[Path]]]:
         Iterator[tuple[Path, list[Path]]]: A tuple containing the module directory and a list of all files in the module
 
     """
+    if not root_dir.exists():
+        return
     for module_dir in root_dir.iterdir():
         if not module_dir.is_dir():
             continue

cognite_toolkit/_cdf_tk/templates/data_classes/_project_directory.py

@@ -12,7 +12,10 @@
 from rich import print
 from rich.panel import Panel
 
-from cognite_toolkit._cdf_tk.templates._constants import COGNITE_MODULES, CUSTOM_MODULES
+from cognite_toolkit._cdf_tk.templates._constants import (
+    COGNITE_MODULES,
+    ROOT_MODULES,
+)
 from cognite_toolkit._cdf_tk.templates._utils import _get_cognite_module_version, iterate_modules
 from cognite_toolkit._cdf_tk.utils import calculate_directory_hash, read_yaml_file
 from cognite_toolkit._version import __version__ as current_version
@@ -39,17 +42,15 @@ class ProjectDirectory:
     _directories_to_copy: ClassVar[list[str]] = [
         "common_function_code",
     ]
-    _root_modules: ClassVar[list[str]] = [
-        COGNITE_MODULES,
-        CUSTOM_MODULES,
-    ]
 
     def __init__(self, project_dir: Path, dry_run: bool):
         self.project_dir = project_dir
         self._dry_run = dry_run
         self._source = Path(resources.files("cognite_toolkit"))  # type: ignore[arg-type]
         self.modules_by_root: dict[str, list[str]] = {}
-        for root_module in self._root_modules:
+        for root_module in ROOT_MODULES:
+            if not (self._source / root_module).exists():
+                continue
             self.modules_by_root[root_module] = [
                 f"{module.relative_to(self._source)!s}" for module, _ in iterate_modules(self._source / root_module)
             ]
@@ -91,10 +92,10 @@ def copy(self, verbose: bool) -> None:
             if not dry_run:
                 shutil.copytree(self._source / directory, self.project_dir / directory, dirs_exist_ok=True)
 
-        for root_module in self._root_modules:
+        for root_module, modules in self.modules_by_root.items():
             if verbose:
                 print(f"{copy_prefix} the following modules from  {root_module} to {self.target_dir_display}")
-                print(self.modules_by_root[root_module])
+                print(modules)
             if not dry_run:
                 (Path(self.project_dir) / root_module).mkdir(exist_ok=True)
                 # Default files are not copied, as they are only used to setup the config.yaml.