Release 0.2.6 (#709)

# Description

Please describe the change you have made.

## Checklist

- [ ] Tests added/updated.
- [ ] Run Demo Job Locally.
- [ ] Documentation updated.
- [ ] Changelogs updated in
[CHANGELOG.cdf-tk.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.cdf-tk.md).
- [ ] Template changelogs updated in
[CHANGELOG.templates.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.templates.md).
- [ ] Version bumped.

[_version.py](https://github.com/cognitedata/toolkit/blob/main/cognite/cognite_toolkit/_version.py)
and

[pyproject.toml](https://github.com/cognitedata/toolkit/blob/main/pyproject.toml)
per [semantic versioning](https://semver.org/).

CHANGELOG.cdf-tk.md

@@ -15,6 +15,25 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.6] - 2024-06-26
+
+### Improved
+
+- The `--verbose` flag is now moved to the end of the command. For example, instead of `cdf-tk --verbose build`,
+  you should now write `cdf-tk build --verbose`. The old syntax is still supported but will raise a deprecation warning.
+- When running `cdf-tk deploy --verbose` you will now get a detailed output for each resource that has changed
+  (or will change if you use --dry-run).
+- Allow values `test` and `qa` as `type` in the `config.[env].yaml` file.
+
+### Fixed
+
+- When running `cdf-tk build` with `Views` with custom filters, the Toolkit would likely give a `UnusedParameterWarning`.
+  This is now fixed by not validating the details of `View.filters`. The motivation is that `View.filters` is a complex
+  structure, and it is likely that you will get a false warning. The users that starts to use `View.filters` are
+  expected to know what they are doing.
+- If you run `cdf-tk deploy` and you had a child view that overrides a property from a parent view, the Toolkit would
+  log it as changed even though it was not. This is now fixed.
+
 ## [0.2.5] - 2024-06-25
 
 ### Fixed

CHANGELOG.templates.md

@@ -15,6 +15,10 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.6] - 2024-06-26
+
+No changes to templates.
+
 ## [0.2.5] - 2024-06-25
 
 No changes to templates.

cognite_toolkit/_cdf.py

@@ -15,6 +15,7 @@
 from rich.panel import Panel
 
 from cognite_toolkit._cdf_tk.commands.featureflag import FeatureFlag, Flags
+from cognite_toolkit._cdf_tk.tk_warnings import ToolkitDeprecationWarning
 
 if FeatureFlag.is_enabled(Flags.ASSETS):
     from cognite_toolkit._cdf_tk.prototypes import setup_asset_loader
@@ -269,10 +270,20 @@ def build(
             "--no-clean", "-c", help="Whether not to delete the build directory before building the configurations"
         ),
     ] = False,
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            "-v",
+            help="Turn on to get more verbose output when running the command",
+        ),
+    ] = False,
 ) -> None:
     """Build configuration files from the module templates to a local build directory."""
     cmd = BuildCommand(user_command=_get_user_command())
-    cmd.execute(ctx.obj.verbose, Path(source_dir), Path(build_dir), build_env_name, no_clean)
+    if ctx.obj.verbose:
+        print(ToolkitDeprecationWarning("cdf-tk --verbose", "cdf-tk build --verbose").get_message())
+    cmd.execute(verbose or ctx.obj.verbose, Path(source_dir), Path(build_dir), build_env_name, no_clean)
 
 
 @_app.command("deploy")
@@ -332,11 +343,21 @@ def deploy(
             help=f"Specify which resources to deploy, available options: {_AVAILABLE_DATA_TYPES}.",
         ),
     ] = None,
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            "-v",
+            help="Turn on to get more verbose output when running the command",
+        ),
+    ] = False,
 ) -> None:
     cmd = DeployCommand(print_warning=True, user_command=_get_user_command())
     include = _process_include(include, interactive)
     ToolGlobals = CDFToolConfig.from_context(ctx)
-    cmd.execute(ToolGlobals, build_dir, build_env_name, dry_run, drop, drop_data, include, ctx.obj.verbose)
+    if ctx.obj.verbose:
+        print(ToolkitDeprecationWarning("cdf-tk --verbose", "cdf-tk deploy --verbose").get_message())
+    cmd.execute(ToolGlobals, build_dir, build_env_name, dry_run, drop, drop_data, include, verbose or ctx.obj.verbose)
 
 
 @_app.command("clean")
@@ -381,13 +402,23 @@ def clean(
             help=f"Specify which resource types to deploy, supported types: {_AVAILABLE_DATA_TYPES}",
         ),
     ] = None,
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            "-v",
+            help="Turn on to get more verbose output when running the command",
+        ),
+    ] = False,
 ) -> None:
     """Clean up a CDF environment as set in environments.yaml restricted to the entities in the configuration files in the build directory."""
     # Override cluster and project from the options/env variables
     cmd = CleanCommand(print_warning=True, user_command=_get_user_command())
     include = _process_include(include, interactive)
     ToolGlobals = CDFToolConfig.from_context(ctx)
-    cmd.execute(ToolGlobals, build_dir, build_env_name, dry_run, include, ctx.obj.verbose)
+    if ctx.obj.verbose:
+        print(ToolkitDeprecationWarning("cdf-tk --verbose", "cdf-tk clean --verbose").get_message())
+    cmd.execute(ToolGlobals, build_dir, build_env_name, dry_run, include, verbose or ctx.obj.verbose)
 
 
 @auth_app.callback(invoke_without_command=True)
@@ -445,6 +476,14 @@ def auth_verify(
             "Set to the source id of the new group.",
         ),
     ] = None,
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            "-v",
+            help="Turn on to get more verbose output when running the command",
+        ),
+    ] = False,
 ) -> None:
     """When you have the necessary information about your identity provider configuration,
     you can use this command to configure the tool and verify that the token has the correct access rights to the project.
@@ -465,7 +504,9 @@ def auth_verify(
         # Remove the Error message from failing to load the config
         # This is verified in check_auth
         ToolGlobals = CDFToolConfig(cluster=ctx.obj.cluster, project=ctx.obj.project, skip_initialization=True)
-    cmd.execute(ToolGlobals, dry_run, interactive, group_file, update_group, create_group, ctx.obj.verbose)
+    if ctx.obj.verbose:
+        print(ToolkitDeprecationWarning("cdf-tk --verbose", "cdf-tk auth verify --verbose").get_message())
+    cmd.execute(ToolGlobals, dry_run, interactive, group_file, update_group, create_group, verbose or ctx.obj.verbose)
 
 
 def main_init(
@@ -682,9 +723,19 @@ def run_function_cmd(
             help="Build environment to build for",
         ),
     ] = "dev",
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            "-v",
+            help="Turn on to get more verbose output when running the command",
+        ),
+    ] = False,
 ) -> None:
     """This command will run the specified function using a one-time session."""
     cmd = RunFunctionCommand(user_command=_get_user_command())
+    if ctx.obj.verbose:
+        print(ToolkitDeprecationWarning("cdf-tk --verbose", "cdf-tk run function --verbose").get_message())
     cmd.execute(
         CDFToolConfig.from_context(ctx),
         external_id,
@@ -696,7 +747,7 @@ def run_function_cmd(
         source_dir,
         schedule,
         build_env_name,
-        ctx.obj.verbose,
+        verbose or ctx.obj.verbose,
     )
 
 
@@ -742,10 +793,26 @@ def pull_transformation_cmd(
             help="Whether to do a dry-run, do dry-run if present.",
         ),
     ] = False,
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            "-v",
+            help="Turn on to get more verbose output when running the command",
+        ),
+    ] = False,
 ) -> None:
     """This command will pull the specified transformation and update its YAML file in the module folder"""
+    if ctx.obj.verbose:
+        print(ToolkitDeprecationWarning("cdf-tk --verbose", "cdf-tk pull transformation --verbose").get_message())
     PullCommand(user_command=_get_user_command()).execute(
-        source_dir, external_id, env, dry_run, ctx.obj.verbose, CDFToolConfig.from_context(ctx), TransformationLoader
+        source_dir,
+        external_id,
+        env,
+        dry_run,
+        verbose or ctx.obj.verbose,
+        CDFToolConfig.from_context(ctx),
+        TransformationLoader,
     )
 
 
@@ -793,14 +860,25 @@ def pull_node_cmd(
             help="Whether to do a dry-run, do dry-run if present.",
         ),
     ] = False,
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            "-v",
+            help="Turn on to get more verbose output when running the command",
+        ),
+    ] = False,
 ) -> None:
     """This command will pull the specified node and update its YAML file in the module folder."""
+    if ctx.obj.verbose:
+        print(ToolkitDeprecationWarning("cdf-tk --verbose", "cdf-tk pull node --verbose").get_message())
+
     PullCommand(user_command=_get_user_command()).execute(
         source_dir,
         NodeId(space, external_id),
         env,
         dry_run,
-        ctx.obj.verbose,
+        verbose or ctx.obj.verbose,
         CDFToolConfig.from_context(ctx),
         NodeLoader,
     )
@@ -858,15 +936,25 @@ def dump_datamodel_cmd(
             allow_dash=True,
         ),
     ] = "tmp",
+    verbose: Annotated[
+        bool,
+        typer.Option(
+            "--verbose",
+            "-v",
+            help="Turn on to get more verbose output when running the command",
+        ),
+    ] = False,
 ) -> None:
     """This command will dump the selected data model as yaml to the folder specified, defaults to /tmp."""
     cmd = DumpCommand(user_command=_get_user_command())
+    if ctx.obj.verbose:
+        print(ToolkitDeprecationWarning("cdf-tk --verbose", "cdf-tk dump datamodel --verbose").get_message())
     cmd.execute(
         CDFToolConfig.from_context(ctx),
         DataModelId(space, external_id, version),
         Path(output_dir),
         clean,
-        ctx.obj.verbose,
+        verbose or ctx.obj.verbose,
     )
 
 
@@ -919,9 +1007,19 @@ def dump_asset_cmd(
                 help="Delete the output directory before pulling the assets.",
             ),
         ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
     ) -> None:
         """This command will dump the selected assets as yaml to the folder specified, defaults to /tmp."""
         cmd = DumpAssetsCommand(user_command=_get_user_command())
+        if ctx.obj.verbose:
+            print(ToolkitDeprecationWarning("cdf-tk --verbose", "cdf-tk dump asset --verbose").get_message())
         cmd.execute(
             CDFToolConfig.from_context(ctx),
             hierarchy,
@@ -930,7 +1028,7 @@ def dump_asset_cmd(
             output_dir,
             clean_,
             format_,  # type: ignore [arg-type]
-            ctx.obj.verbose,
+            verbose or ctx.obj.verbose,
         )
 
 

cognite_toolkit/_cdf_tk/_migration.yaml

@@ -1,8 +1,13 @@
-- version: 0.2.5
+- version: 0.2.6
   cognite_modules: {}
   resources: {}
   tool: {}
   cognite_modules_hash: ""
+- version: 0.2.5
+  cognite_modules: {}
+  resources: {}
+  tool: {}
+  cognite_modules_hash: "8705ad966cb954422f5a63c3a89b02078cf1f9b3331155cd061babd518145f9d"
 - version: 0.2.4
   cognite_modules: {}
   resources: {}

cognite_toolkit/_cdf_tk/commands/deploy.py

@@ -4,6 +4,7 @@
 import traceback
 from graphlib import TopologicalSorter
 from pathlib import Path
+from typing import Any
 
 from cognite.client.data_classes._base import T_CogniteResourceList
 from cognite.client.exceptions import CogniteAPIError, CogniteDuplicatedError
@@ -46,6 +47,7 @@
 from cognite_toolkit._cdf_tk.utils import (
     CDFToolConfig,
     read_yaml_file,
+    to_diff,
 )
 
 from ._utils import _print_ids_or_length, _remove_duplicates
@@ -230,7 +232,7 @@ def sort_key(p: Path) -> int:
             self.warn(LowSeverityWarning(f"Skipping duplicate {loader.display_name} {duplicate}."))
 
         nr_of_created = nr_of_changed = nr_of_unchanged = 0
-        to_create, to_update, unchanged = self.to_create_changed_unchanged_triple(loaded_resources, loader)
+        to_create, to_update, unchanged = self.to_create_changed_unchanged_triple(loaded_resources, loader, verbose)
 
         if dry_run:
             if (
@@ -286,6 +288,7 @@ def to_create_changed_unchanged_triple(
         self,
         resources: T_CogniteResourceList,
         loader: ResourceLoader,
+        verbose: bool = False,
     ) -> tuple[T_CogniteResourceList, T_CogniteResourceList, T_CogniteResourceList]:
         """Returns a triple of lists of resources that should be created, updated, and are unchanged."""
         resource_ids = loader.get_ids(resources)
@@ -308,20 +311,33 @@ def to_create_changed_unchanged_triple(
             cdf_resource_by_id = {loader.get_id(resource): resource for resource in cdf_resources}
 
         for item in resources:
-            cdf_resource = cdf_resource_by_id.get(loader.get_id(item))
-            try:
-                are_equal = cdf_resource and loader.are_equal(item, cdf_resource)
-            except CogniteAPIError as e:
-                self.warn(
-                    MediumSeverityWarning(
-                        f"Failed to compare {loader.display_name} {loader.get_id(item)} for equality. Proceeding assuming not data in CDF. Error {e}."
+            identifier = loader.get_id(item)
+            cdf_resource = cdf_resource_by_id.get(identifier)
+            local_dumped: dict[str, Any] = {}
+            cdf_dumped: dict[str, Any] = {}
+            are_equal = False
+            if cdf_resource:
+                try:
+                    are_equal, local_dumped, cdf_dumped = loader.are_equal(item, cdf_resource, return_dumped=True)
+                except CogniteAPIError as e:
+                    self.warn(
+                        MediumSeverityWarning(
+                            f"Failed to compare {loader.display_name} {loader.get_id(item)} for equality. Proceeding assuming not data in CDF. Error {e}."
+                        )
                     )
-                )
-                print(Panel(traceback.format_exc()))
-                are_equal = False
+                    print(Panel(traceback.format_exc()))
+
             if are_equal:
                 unchanged.append(item)
             elif cdf_resource:
+                if verbose:
+                    print(
+                        Panel(
+                            "\n".join(to_diff(cdf_dumped, local_dumped)),
+                            title=f"{loader.display_name}: {identifier}",
+                            expand=False,
+                        )
+                    )
                 to_update.append(item)
             else:
                 to_create.append(item)