Release 0.3.14

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
     rev: v0.8.0
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.42.0
+    rev: v0.43.0
     hooks:
     - id: markdownlint
 

CHANGELOG.cdf-tk.md

@@ -15,6 +15,12 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.3.14] - 2024-11-27
+
+### Added
+
+- Support for authentication with `CogIDP`
+
 ## [0.3.13] - 2024-11-26
 
 ### Fixed

CHANGELOG.templates.md

@@ -15,6 +15,10 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.3.14] - 2024-11-27
+
+No changes to templates.
+
 ## [0.3.13] - 2024-11-26
 
 ## Changed

cdf.toml

@@ -20,4 +20,4 @@ dump = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.3.13"
+version = "0.3.14"

cognite_toolkit/_builtin_modules/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.3.13"
+version = "0.3.14"
 
 
 [plugins]

cognite_toolkit/_builtin_modules/cdf_ingestion/auth/user.Group.yaml

@@ -100,6 +100,14 @@ capabilities:
         datasetScope:
           ids:
             - {{ dataset }}
+  - workflowOrchestrationAcl:
+      actions:
+          - READ
+          - WRITE
+      scope:
+        datasetScope:
+          ids:
+            - {{ dataset }}
 # Bug in /context/diagram/detect/ endpoint that requires this
   - assetsAcl:
       actions:

cognite_toolkit/_cdf_tk/commands/auth.py

@@ -25,6 +25,7 @@
     FunctionsAcl,
     GroupsAcl,
     ProjectsAcl,
+    SessionsAcl,
 )
 from cognite.client.data_classes.iam import Group, GroupList, GroupWrite, TokenInspection
 from cognite.client.exceptions import CogniteAPIError
@@ -103,23 +104,23 @@ def verify(
         ToolGlobals: CDFToolConfig,
         dry_run: bool,
         no_prompt: bool = False,
-        demo_user: str | None = None,
+        demo_principal: str | None = None,
     ) -> VerifyAuthResult:
         """Authorization verification for the Toolkit.
 
         Args:
             ToolGlobals: The Toolkit configuration.
             dry_run: If the verification should be run in dry-run mode.
             no_prompt: If the verification should be run without any prompts.
-            demo_user: This is used for demo purposes. If passed a temporary Toolkit group is created
-                and the user is added to the group.
+            demo_principal: This is used for demo purposes. If passed, a different group name will be used
+                to create the Toolkit group. This is group is intended to be deleted after the demo.
 
         Returns:
             VerifyAuthResult: The result of the verification.
         """
 
         is_interactive = not no_prompt
-        is_demo = demo_user is not None
+        is_demo = demo_principal is not None
         if ToolGlobals.project is None:
             raise AuthorizationError("CDF_PROJECT is not set.")
         cdf_project = ToolGlobals.project
@@ -142,7 +143,7 @@ def verify(
             raise AuthorizationError("The current user is not member of any groups in the CDF project.")
 
         loader_capabilities, loaders_by_capability_tuple = self._get_capabilities_by_loader(ToolGlobals)
-        toolkit_group = self._create_toolkit_group(loader_capabilities, demo_user)
+        toolkit_group = self._create_toolkit_group(loader_capabilities, demo_principal)
 
         if not is_demo:
             print(
@@ -392,11 +393,16 @@ def _create_toolkit_group(loader_capabilities: list[Capability], demo_user: str
             name=TOOLKIT_SERVICE_PRINCIPAL_GROUP_NAME if demo_user is None else TOOLKIT_DEMO_GROUP_NAME,
             capabilities=[
                 *loader_capabilities,
-                # Add project ACL to be able to list and read projects, as the
+                # Add project ACL to be able to list and read projects, as the Toolkit needs to know the project id.
                 ProjectsAcl(
                     [ProjectsAcl.Action.Read, ProjectsAcl.Action.List, ProjectsAcl.Action.Update],
                     ProjectsAcl.Scope.All(),
                 ),
+                # Added Session ACL as this is required by CogIDP service principal
+                SessionsAcl(
+                    [SessionsAcl.Action.Create, SessionsAcl.Action.List, SessionsAcl.Action.Delete],
+                    SessionsAcl.Scope.All(),
+                ),
             ],
         )
         if demo_user:

cognite_toolkit/_cdf_tk/tk_warnings/__init__.py

@@ -25,6 +25,7 @@
 from .other import (
     HighSeverityWarning,
     HTTPWarning,
+    IgnoredValueWarning,
     IncorrectResourceWarning,
     LowSeverityWarning,
     MediumSeverityWarning,
@@ -45,6 +46,7 @@
     "GeneralWarning",
     "HTTPWarning",
     "HighSeverityWarning",
+    "IgnoredValueWarning",
     "IncorrectResourceWarning",
     "LowSeverityWarning",
     "MediumSeverityWarning",

cognite_toolkit/_cdf_tk/tk_warnings/other.py

@@ -156,3 +156,14 @@ class HTTPWarning(ToolkitWarning):
 
     def get_message(self) -> str:
         return f"Failed to {self.action}. HTTP status code {self.status_code}: {self.message}"
+
+
+@dataclass(frozen=True)
+class IgnoredValueWarning(ToolkitWarning):
+    severity: ClassVar[SeverityLevel] = SeverityLevel.LOW
+    name: str
+    value: str
+    reason: str
+
+    def get_message(self) -> str:
+        return f"Ignoring value {self.value!r} for {self.name!r}: {self.reason!r}"

cognite_toolkit/_cdf_tk/utils/auth.py

@@ -20,7 +20,7 @@
 from collections.abc import Sequence
 from dataclasses import dataclass, field, fields
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, TypeAlias, overload
+from typing import Any, Literal, TypeAlias, overload
 
 import questionary
 import typer
@@ -61,15 +61,11 @@
     ResourceRetrievalError,
     ToolkitResourceMissingError,
 )
-from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
+from cognite_toolkit._cdf_tk.tk_warnings import IgnoredValueWarning, MediumSeverityWarning
 from cognite_toolkit._version import __version__
 
-if TYPE_CHECKING:
-    pass
-
-
 LoginFlow: TypeAlias = Literal["client_credentials", "token", "device_code", "interactive"]
-Provider: TypeAlias = Literal["entra_id", "other"]
+Provider: TypeAlias = Literal["entra_id", "cog_idp", "other"]
 
 LOGIN_FLOW_DESCRIPTION = {
     "client_credentials": "Setup a service principal with client credentials",
@@ -80,6 +76,7 @@
 
 PROVDER_DESCRIPTION = {
     "entra_id": "Use Microsoft Entra ID to authenticate",
+    "cog_idp": "Use Cognite IDP to authenticate",
     "other": "Use other IDP to authenticate",
 }
 
@@ -174,6 +171,8 @@ class AuthVariables:
 
     def __post_init__(self) -> None:
         # Set defaults based on cluster and tenant_id
+        if self.client_secret:
+            self.set_client_secret_defaults()
         if self.cluster:
             self.set_cluster_defaults()
         if self.tenant_id:
@@ -185,6 +184,16 @@ def __post_init__(self) -> None:
             )
             self.login_flow = "token"
 
+    def set_client_secret_defaults(self) -> None:
+        if self.client_secret and self.client_secret.startswith("cdf_sa_sct"):
+            self.provider = "cog_idp"
+            self.token_url = self.token_url or "https://auth.cognite.com/oauth2/token"
+            if self.scopes is not None:
+                IgnoredValueWarning(
+                    "IDP_SCOPES", self.scopes, "Provider si Cog-IDP does not need scopes"
+                ).print_warning()
+            self.scopes = None
+
     def set_token_id_defaults(self) -> None:
         if self.tenant_id:
             self.token_url = self.token_url or f"https://login.microsoftonline.com/{self.tenant_id}/oauth2/v2.0/token"
@@ -194,7 +203,8 @@ def set_cluster_defaults(self) -> None:
         if self.cluster:
             self.cdf_url = self.cdf_url or f"https://{self.cluster}.cognitedata.com"
             self.audience = self.audience or f"https://{self.cluster}.cognitedata.com"
-            self.scopes = self.scopes or f"https://{self.cluster}.cognitedata.com/.default"
+            if self.provider != "cog_idp":
+                self.scopes = self.scopes or f"https://{self.cluster}.cognitedata.com/.default"
 
     @property
     def is_complete(self) -> bool:
@@ -511,7 +521,7 @@ def __init__(
 
         global_config.disable_pypi_version_check = True
         global_config.silence_feature_preview_warnings = True
-        if _RUNNING_IN_BROWSER:
+        if _RUNNING_IN_BROWSER and auth_vars is None:
             self._initialize_in_browser()
             return
 
@@ -590,6 +600,19 @@ def initialize_from_auth_variables(self, auth: AuthVariables, clear_cache: bool
                 scopes=self._scopes,
             )
             self._credentials_provider = OAuthInteractive(**self._credentials_args)
+        elif auth.login_flow == "client_credentials" and auth.provider == "cog_idp":
+            if not (auth.client_id and auth.client_secret):
+                raise AuthenticationError(
+                    "Login flow=client_credentials is set but missing required authentication "
+                    "variables: IDP_CLIENT_ID and IDP_CLIENT_SECRET. Cannot authenticate the client."
+                )
+            self._credentials_args = dict(
+                token_url=auth.token_url,
+                client_id=auth.client_id,
+                client_secret=auth.client_secret,
+                scopes=None,
+            )
+            self._credentials_provider = OAuthClientCredentials(**self._credentials_args)
         elif auth.login_flow == "client_credentials" or auth.login_flow is None:
             if auth.login_flow is None:
                 print(

cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml

@@ -14,7 +14,7 @@ jobs:
     pool:
       vmImage: 'ubuntu-latest'
     container: 
-      image: 'cognite/toolkit:0.3.13'
+      image: 'cognite/toolkit:0.3.14'
       env:
         CDF_CLUSTER: $(CDF_CLUSTER)
         CDF_PROJECT: $(CDF_PROJECT)

cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml

@@ -16,7 +16,7 @@ jobs:
     pool:
       vmImage: 'ubuntu-latest'
     container: 
-      image: 'cognite/toolkit:0.3.13'
+      image: 'cognite/toolkit:0.3.14'
       env:
         CDF_CLUSTER: $(CDF_CLUSTER)
         CDF_PROJECT: $(CDF_PROJECT)

cognite_toolkit/_repo_files/GitHub/.github/workflows/build.yaml

@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.3.13
+      image: cognite/toolkit:0.3.14
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml

@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.3.13
+      image: cognite/toolkit:0.3.14
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

cognite_toolkit/_version.py

@@ -1 +1 @@
-__version__ = "0.3.13"
+__version__ = "0.3.14"

cognite_toolkit/demo/_base.py

@@ -7,6 +7,7 @@
 from rich.panel import Panel
 
 from cognite_toolkit._cdf_tk.commands import AuthCommand, BuildCommand, DeployCommand, ModulesCommand
+from cognite_toolkit._cdf_tk.constants import MODULES
 from cognite_toolkit._cdf_tk.loaders import LOADER_BY_FOLDER_NAME
 from cognite_toolkit._cdf_tk.utils.auth import AuthVariables, CDFToolConfig
 
@@ -43,14 +44,16 @@ def _organization_dir(self) -> Path:
         organization_path.mkdir(exist_ok=True)
         return organization_path
 
-    def quickstart(self, client_id: str | None = None, client_secret: str | None = None) -> None:
+    def quickstart(
+        self, organization_name: str | None, client_id: str | None = None, client_secret: str | None = None
+    ) -> None:
         print(Panel("Running Toolkit QuickStart..."))
         user = self._cdf_tool_config.toolkit_client.iam.user_profiles.me()
         if sum([client_id is None, client_secret is None]) == 1:
             raise ValueError("Both client_id and client_secret must be provided or neither.")
         if client_id is None and client_secret is None:
             print("Client ID and secret not provided. Assuming user has all the necessary permissions.")
-            self._init_build_deploy(user)
+            self._init_build_deploy(user, organization_name)
             return
 
         group_id: int | None = None
@@ -61,7 +64,7 @@ def quickstart(self, client_id: str | None = None, client_secret: str | None = N
                 self._cdf_tool_config,
                 dry_run=False,
                 no_prompt=True,
-                demo_user=client_id,
+                demo_principal=client_id,
             )
             group_id = auth_result.toolkit_group_id
             if auth_result.function_status is None:
@@ -83,18 +86,16 @@ def quickstart(self, client_id: str | None = None, client_secret: str | None = N
                     cluster=self._cdf_tool_config.cdf_cluster,
                     project=self._cdf_tool_config.project,
                     login_flow="client_credentials",
-                    provider="other",
                     client_id=client_id,
                     client_secret=client_secret,
-                    token_url=f"{self._cdf_tool_config.toolkit_client.config.base_url}/oauth2/token",
                 )
             )
-            self._init_build_deploy(user)
+            self._init_build_deploy(user, organization_name)
         finally:
             if group_id is not None:
                 self._cdf_tool_config.toolkit_client.iam.groups.delete(id=group_id)
 
-    def _init_build_deploy(self, user: UserProfile) -> None:
+    def _init_build_deploy(self, user: UserProfile, organization_name: str | None = None) -> None:
         modules_cmd = ModulesCommand()
         modules_cmd.run(
             lambda: modules_cmd.init(
@@ -112,8 +113,15 @@ def _init_build_deploy(self, user: UserProfile) -> None:
         # To avoid warnings about not set values
         config_raw = config_raw.replace("<not set>", "123456-to-be-replaced")
         config_raw = config_raw.replace("<my-project-dev>", self._cdf_tool_config.project)
+        if organization_name is not None:
+            config_raw = config_raw.replace("YourOrg", organization_name)
         config_yaml.write_text(config_raw)
 
+        # The Workflow trigger expects credentials to be set in the environment, so we delete it as
+        # the user is expected to trigger the workflow manually.
+        for workflow_trigger_file in (self._organization_dir / MODULES).rglob("*WorkflowTrigger.yaml"):
+            workflow_trigger_file.unlink()
+
         build = BuildCommand()
         build.run(
             lambda: build.execute(

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite_toolkit"
-version = "0.3.13"
+version = "0.3.14"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = ["Cognite AS <support@cognite.com>"]
 license = "Apache-2"

tests/data/build_group_with_unknown_acl/_build_environment.yaml

@@ -4,4 +4,4 @@ project: pytest-project
 type: dev
 selected:
 - modules
-cdf_toolkit_version: 0.3.13
+cdf_toolkit_version: 0.3.14

tests/data/cdf_toml_data/cdf.toml

@@ -1,7 +1,7 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.3.13"
+version = "0.3.14"
 
 [plugins]
 graphql = true

tests/data/complete_org/cdf.toml

@@ -1,4 +1,4 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.3.13"
+version = "0.3.14"