Release 0.2.9 (#732)

# Description

Please describe the change you have made.

## Checklist

- [ ] Tests added/updated.
- [ ] Run Demo Job Locally.
- [ ] Documentation updated.
- [ ] Changelogs updated in
[CHANGELOG.cdf-tk.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.cdf-tk.md).
- [ ] Template changelogs updated in
[CHANGELOG.templates.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.templates.md).
- [ ] Version bumped.

[_version.py](https://github.com/cognitedata/toolkit/blob/main/cognite/cognite_toolkit/_version.py)
and

[pyproject.toml](https://github.com/cognitedata/toolkit/blob/main/pyproject.toml)
per [semantic versioning](https://semver.org/).

CHANGELOG.cdf-tk.md

@@ -15,6 +15,12 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.9] - 2024-07-02
+
+### Changed
+
+- Tracking usage of Toolkit commands.
+
 ## [0.2.8] - 2024-07-01
 
 ### Added

CHANGELOG.templates.md

@@ -15,6 +15,10 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.9] - 2024-07-02
+
+No changes to templates.
+
 ## [0.2.8] - 2024-07-01
 
 ### Fixed

CONTRIBUTING.md

@@ -100,11 +100,12 @@ To release a new version of the `cdf-tk` tool and the templates, you need to do
       comments and that the changes can be easily understood. Also verify that any breaking changes
       are clearly marked as such (`**BREAKING**`).
    2. Do the same update to `CHANGELOG.templates.md` file.
-   3. Update the following files with the new version number:
+   3. Update the files with the new version number:
       - `cognite_toolkit/_version.py`
       - `pyproject.toml`
       - `_system.yaml` (multiple)
 
+      You can use the `python bump --minor --alpha` command to bump the version in all files.
    4. Run `poetry lock` to update the `poetry.lock` file.
    5. Run `pytest tests` locally to ensure that tests pass.
    6. Run `python module_upgrade/run_check.py` to ensure that the `cdf-tk modules upgrade` command works as expected.

cognite_toolkit/_cdf.py

@@ -32,6 +32,7 @@
     AuthCommand,
     BuildCommand,
     CleanCommand,
+    CollectCommand,
     DeployCommand,
     DescribeCommand,
     DumpCommand,
@@ -99,13 +100,13 @@
 feature_flag_app = typer.Typer(**default_typer_kws, hidden=True)  # type: ignore [arg-type]
 user_app = typer.Typer(**default_typer_kws, hidden=True)  # type: ignore [arg-type]
 
+
 _app.add_typer(auth_app, name="auth")
 _app.add_typer(describe_app, name="describe")
 _app.add_typer(run_app, name="run")
 _app.add_typer(pull_app, name="pull")
 _app.add_typer(dump_app, name="dump")
 _app.add_typer(feature_flag_app, name="features")
-_app.add_typer(user_app, name="user")
 
 
 def app() -> NoReturn:
@@ -128,6 +129,44 @@ def app() -> NoReturn:
 
             _app.add_typer(import_app, name="import")
 
+        # Secret plugin, this will be removed without warning
+        # This should not be documented, or raise any error or warnings,
+        # just fail silently if the plugin is not found or not correctly setup.
+        dev_py = Path.cwd() / "dev.py"
+        if dev_py.exists():
+            from importlib.util import module_from_spec, spec_from_file_location
+
+            spec = spec_from_file_location("dev", dev_py)
+            if spec and spec.loader:
+                dev_module = module_from_spec(spec)
+                spec.loader.exec_module(dev_module)
+                if "CDF_TK_PLUGIN" in dev_module.__dict__:
+                    command_by_name = {cmd.name: cmd for cmd in _app.registered_commands}
+                    group_by_name = {group.name: group for group in _app.registered_groups}
+                    for name, type_app in dev_module.__dict__["CDF_TK_PLUGIN"].items():
+                        if not isinstance(type_app, typer.Typer):
+                            continue
+                        if name in command_by_name:
+                            # We are not allowed to replace an existing command.
+                            continue
+                        elif name in group_by_name:
+                            group = group_by_name[name]
+                            if group.typer_instance is None:
+                                continue
+                            existing_command_names = {cmd.name for cmd in group.typer_instance.registered_commands}
+                            for new_command in type_app.registered_commands:
+                                if new_command.name in existing_command_names:
+                                    # We are not allowed to replace an existing command.
+                                    continue
+                                group.typer_instance.command(new_command.name)(new_command.callback)  # type: ignore [type-var]
+                        else:
+                            if type_app.registered_groups:
+                                _app.add_typer(type_app, name=name)
+                            else:
+                                for app_cmd in type_app.registered_commands:
+                                    if app_cmd.name not in command_by_name:
+                                        _app.command(app_cmd.name)(app_cmd.callback)  # type: ignore [type-var]
+
         _app()
     except ToolkitError as err:
         print(f"  [bold red]ERROR ([/][red]{type(err).__name__}[/][bold red]):[/] {err}")
@@ -420,6 +459,17 @@ def clean(
     cmd.run(lambda: cmd.execute(ToolGlobals, build_dir, build_env_name, dry_run, include, verbose or ctx.obj.verbose))
 
 
+@_app.command("collect", hidden=True)
+def collect(
+    action: str = typer.Argument(
+        help="Whether to explicitly opt-in or opt-out of usage data collection. [opt-in, opt-out]"
+    ),
+) -> None:
+    """Collect usage information for the toolkit."""
+    cmd = CollectCommand()
+    cmd.run(lambda: cmd.execute(action))  # type: ignore [arg-type]
+
+
 @auth_app.callback(invoke_without_command=True)
 def auth_main(ctx: typer.Context) -> None:
     """Test, validate, and configure authentication and authorization for CDF projects."""

cognite_toolkit/_cdf_tk/_migration.yaml

@@ -1,8 +1,13 @@
-- version: 0.2.8
+- version: 0.2.9
   cognite_modules: {}
   resources: {}
   tool: {}
   cognite_modules_hash: ""
+- version: 0.2.8
+  cognite_modules: {}
+  resources: {}
+  tool: {}
+  cognite_modules_hash: "e0375c4c018b891133857b2a9884d0d11483f644b3ced103dc7987024c97405a"
 - version: 0.2.7
   cognite_modules: {}
   resources: {}

cognite_toolkit/_cdf_tk/commands/__init__.py

@@ -1,6 +1,7 @@
 from .auth import AuthCommand
 from .build import BuildCommand
 from .clean import CleanCommand
+from .collect import CollectCommand
 from .deploy import DeployCommand
 from .describe import DescribeCommand
 from .dump import DumpCommand
@@ -11,6 +12,7 @@
 __all__ = [
     "AuthCommand",
     "BuildCommand",
+    "CollectCommand",
     "CleanCommand",
     "DeployCommand",
     "DescribeCommand",

cognite_toolkit/_cdf_tk/commands/_base.py

@@ -34,16 +34,27 @@ def __init__(self, print_warning: bool = True, skip_tracking: bool = False):
             self.user_command = "cdf-tk"
         self.warning_list = WarningList[ToolkitWarning]()
         self.tracker = Tracker(self.user_command)
-        self.skip_tracking = skip_tracking
+        self.skip_tracking = self.tracker.opted_out or skip_tracking
 
     def _track_command(self, result: str | Exception) -> None:
-        # Local import to avoid circular imports
-
         if self.skip_tracking or "PYTEST_CURRENT_TEST" in os.environ:
             return
         self.tracker.track_command(self.warning_list, result, type(self).__name__.removesuffix("Command"))
 
     def run(self, execute: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
+        if (
+            not self.tracker.opted_in
+            and not self.tracker.opted_out
+            and not self.user_command.startswith("cdf-tk collect")
+        ):
+            print(
+                "You acknowledge and agree that the CLI tool may collect usage information, user environment, "
+                "and crash reports for the purposes of providing services of functions that are relevant "
+                "to use of the CLI tool and product improvements. "
+                "To remove this message run 'cdf-tk collect opt-in', "
+                "or to stop collecting usage information run 'cdf-tk collect opt-out'."
+            )
+
         try:
             result = execute(*args, **kwargs)
         except Exception as e:

cognite_toolkit/_cdf_tk/commands/collect.py

@@ -0,0 +1,17 @@
+from typing import Literal
+
+from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
+
+from ._base import ToolkitCommand
+
+
+class CollectCommand(ToolkitCommand):
+    def execute(self, action: Literal["opt-in", "opt-out"]) -> None:
+        if action == "opt-in":
+            self.tracker.enable()
+            print("You have successfully opted in to data collection.")
+        elif action == "opt-out":
+            self.tracker.disable()
+            print("You have successfully opted out of data collection.")
+        else:
+            raise ToolkitValueError(f"Invalid action: {action}. Must be 'opt-in' or 'opt-out'.")

cognite_toolkit/_cdf_tk/tracker.py

@@ -7,6 +7,7 @@
 import threading
 import uuid
 from collections import Counter
+from functools import cached_property
 from pathlib import Path
 
 from mixpanel import Consumer, Mixpanel
@@ -21,6 +22,7 @@ class Tracker:
     def __init__(self, user_command: str) -> None:
         self.user_command = user_command
         self.mp = Mixpanel(_COGNITE_TOOLKIT_MIXPANEL_TOKEN, consumer=Consumer(api_host="api-eu.mixpanel.com"))
+        self._opt_status_file = Path(tempfile.gettempdir()) / "tk-opt-status.bin"
 
     def track_command(self, warning_list: WarningList[ToolkitWarning], result: str | Exception, cmd: str) -> None:
         distinct_id = self.get_distinct_id()
@@ -114,3 +116,23 @@ def _cicd(self) -> str:
             return "azure"
 
         return "local"
+
+    def enable(self) -> None:
+        self._opt_status_file.write_text("opted-in")
+
+    def disable(self) -> None:
+        self._opt_status_file.write_text("opted-out")
+
+    @cached_property
+    def _opt_status(self) -> str:
+        if self._opt_status_file.exists():
+            return self._opt_status_file.read_text()
+        return ""
+
+    @property
+    def opted_out(self) -> bool:
+        return self._opt_status == "opted-out"
+
+    @property
+    def opted_in(self) -> bool:
+        return self._opt_status == "opted-in"

cognite_toolkit/_system.yaml

@@ -25,4 +25,4 @@ packages:
     - example_pump_data_model
 
 # This part is used by cdf-toolkit to keep track of the version and help you upgrade.
-cdf_toolkit_version: 0.2.8
+cdf_toolkit_version: 0.2.9

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.2.8"
+__version__ = "0.2.9"

dev.py

@@ -0,0 +1,92 @@
+"""This is a small CLI used to develop Toolkit."""
+
+from pathlib import Path
+
+import typer
+from packaging.version import Version, parse
+
+from cognite_toolkit._version import __version__
+
+REPO_ROOT = Path(__file__).parent
+
+bump_app = typer.Typer(
+    add_completion=False,
+    help=__doc__,
+    pretty_exceptions_short=False,
+    pretty_exceptions_show_locals=False,
+    pretty_exceptions_enable=False,
+)
+
+
+@bump_app.command()
+def bump(
+    major: bool = False,
+    minor: bool = False,
+    patch: bool = False,
+    alpha: bool = False,
+    beta: bool = False,
+    verbose: bool = False,
+) -> None:
+    version_files = [
+        REPO_ROOT / "pyproject.toml",
+        REPO_ROOT / "cognite_toolkit" / "_version.py",
+        REPO_ROOT / "cognite_toolkit" / "_system.yaml",
+        *(REPO_ROOT / "tests" / "data").rglob("_system.yaml"),
+    ]
+    version = parse(__version__)
+
+    if alpha and version.is_prerelease and version.pre[0] == "a":
+        suffix = f"a{version.pre[1] + 1}"
+    elif alpha and version.is_prerelease and version.pre[0] == "b":
+        raise typer.BadParameter("Cannot bump to alpha version when current version is a beta prerelease.")
+    elif alpha and not version.is_prerelease:
+        suffix = "a1"
+    elif beta and version.is_prerelease and version.pre[0] == "a":
+        suffix = "b1"
+    elif beta and version.is_prerelease and version.pre[0] == "b":
+        suffix = f"b{version.pre[1] + 1}"
+    elif beta and not version.is_prerelease:
+        raise typer.BadParameter("Cannot bump to beta version when current version is not an alpha prerelease.")
+    else:
+        suffix = ""
+
+    if major:
+        new_version = Version(f"{version.major + 1}.0.0{suffix}")
+    elif minor:
+        new_version = Version(f"{version.major}.{version.minor + 1}.0{suffix}")
+    elif patch:
+        new_version = Version(f"{version.major}.{version.minor}.{version.micro + 1}{suffix}")
+    elif alpha or beta:
+        new_version = Version(f"{version.major}.{version.minor}.{version.micro}{suffix}")
+    else:
+        raise typer.BadParameter("You must specify one of major, minor, patch, alpha, or beta.")
+
+    for file in version_files:
+        file.write_text(file.read_text().replace(str(version), str(new_version), 1))
+        if verbose:
+            typer.echo(f"Bumped version from {version} to {new_version} in {file}.")
+
+    typer.echo(f"Bumped version from {version} to {new_version} in {len(version_files)} files.")
+
+
+# This is just for demo purposes, to test the secret plugin in the Toolkit CLI
+import_app = typer.Typer(
+    pretty_exceptions_short=False, pretty_exceptions_show_locals=False, pretty_exceptions_enable=False
+)
+
+
+@import_app.command("cdf")
+def cdf(
+    ctx: typer.Context,
+) -> None:
+    """Import resources into Cognite Data Fusion."""
+    print("Ran CDF Import Command")
+
+
+CDF_TK_PLUGIN = {
+    "bump": bump_app,
+    "import": import_app,
+}
+
+if __name__ == "__main__":
+    bump_app()