feat: databricks token + m2m auth support

Added support for loading and putting databricks datasources using token or m2m auth as well as testing the data source connection.

JIRA: LX-691
risk: low

.python-version

@@ -1 +1 @@
-3.13.1
+3.12.0

gooddata-sdk/gooddata_sdk/catalog/data_source/declarative_model/data_source.py

@@ -12,12 +12,13 @@
 from gooddata_api_client.model.test_definition_request import TestDefinitionRequest
 
 from gooddata_sdk.catalog.base import Base, value_in_allowed
-from gooddata_sdk.catalog.entity import TokenCredentialsFromFile
+from gooddata_sdk.catalog.entity import ClientSecretCredentialsFromFile, TokenCredentialsFromFile
 from gooddata_sdk.catalog.parameter import CatalogParameter
 from gooddata_sdk.catalog.permission.declarative_model.permission import CatalogDeclarativeDataSourcePermission
 from gooddata_sdk.utils import create_directory, get_ds_credentials, read_layout_from_file, write_layout_to_file
 
 BIGQUERY_TYPE = "BIGQUERY"
+DATABRICKS_TYPE = "DATABRICKS"
 LAYOUT_DATA_SOURCES_DIR = "data_sources"
 
 
@@ -33,6 +34,15 @@ def _inject_base(self, credentials: dict[str, Any]) -> DeclarativeDataSources:
                 if data_source.type == BIGQUERY_TYPE:
                     token = TokenCredentialsFromFile.token_from_file(credentials[data_source.id])
                     data_sources.append(data_source.to_api(token=token))
+                elif data_source.type == DATABRICKS_TYPE:
+                    if data_source.client_id is not None:
+                        client_secret = ClientSecretCredentialsFromFile.client_secret_from_file(
+                            credentials[data_source.id]
+                        )
+                        data_sources.append(data_source.to_api(client_secret=client_secret))
+                    else:
+                        token = TokenCredentialsFromFile.token_from_file(credentials[data_source.id])
+                        data_sources.append(data_source.to_api(token=token))
                 else:
                     data_sources.append(data_source.to_api(password=credentials[data_source.id]))
             else:
@@ -125,8 +135,13 @@ def to_test_request(
             kwargs["private_key"] = private_key
         if private_key_passphrase is not None:
             kwargs["private_key_passphrase"] = private_key
+        if self.client_id is not None:
+            kwargs["client_id"] = self.client_id
         if client_secret is not None:
             kwargs["client_secret"] = client_secret
+        if self.parameters is not None:
+            kwargs["parameters"] = [param.to_data_source_parameter() for param in self.parameters]
+
         return TestDefinitionRequest(type=self.type, url=self.url, **kwargs)
 
     @staticmethod

gooddata-sdk/gooddata_sdk/catalog/data_source/service.py

@@ -17,6 +17,7 @@
 from gooddata_sdk.catalog.data_source.action_model.responses.scan_sql_response import ScanSqlResponse
 from gooddata_sdk.catalog.data_source.declarative_model.data_source import (
     BIGQUERY_TYPE,
+    DATABRICKS_TYPE,
     CatalogDeclarativeDataSource,
     CatalogDeclarativeDataSources,
 )
@@ -25,7 +26,7 @@
     CatalogScanResultPdm,
 )
 from gooddata_sdk.catalog.data_source.entity_model.data_source import CatalogDataSource
-from gooddata_sdk.catalog.entity import TokenCredentialsFromFile
+from gooddata_sdk.catalog.entity import ClientSecretCredentialsFromFile, TokenCredentialsFromFile
 from gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.ldm import CatalogDeclarativeModel
 from gooddata_sdk.client import GoodDataApiClient
 from gooddata_sdk.utils import get_ds_credentials, load_all_entities_dict, read_layout_from_file
@@ -474,6 +475,19 @@ def test_data_sources_connection(
                     response = self._actions_api.test_data_source_definition(
                         declarative_data_source.to_test_request(token=token)
                     )
+                elif declarative_data_source.type == DATABRICKS_TYPE:
+                    if declarative_data_source.client_id is not None:
+                        client_secret = ClientSecretCredentialsFromFile.client_secret_from_file(
+                            credentials[declarative_data_source.id]
+                        )
+                        response = self._actions_api.test_data_source_definition(
+                            declarative_data_source.to_test_request(client_secret=client_secret)
+                        )
+                    else:
+                        token = TokenCredentialsFromFile.token_from_file(credentials[declarative_data_source.id])
+                        response = self._actions_api.test_data_source_definition(
+                            declarative_data_source.to_test_request(token=token)
+                        )
                 else:
                     response = self._actions_api.test_data_source_definition(
                         declarative_data_source.to_test_request(password=credentials[declarative_data_source.id])
@@ -528,6 +542,6 @@ def _credentials_from_file(credentials_path: Path) -> dict[str, Any]:
         if data.get("data_sources") is None:
             raise ValueError("The file has a wrong structure. There should be a root key 'data_sources'.")
         if len(data["data_sources"]) == 0:
-            raise ValueError("There are no pairs of data source id and token.")
+            raise ValueError("There are no pairs of data source id and credentials.")
         credentials = data["data_sources"]
         return credentials

gooddata-sdk/gooddata_sdk/catalog/entity.py

@@ -274,3 +274,31 @@ def from_api(cls, attributes: dict[str, Any]) -> ClientSecretCredentials:
             # You have to fill it to keep it or update it
             client_secret="",
         )
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class ClientSecretCredentialsFromFile(Credentials):
+    file_path: Path
+    client_secret: str = attr.field(init=False, repr=lambda value: "***")
+
+    def __attrs_post_init__(self) -> None:
+        self.client_secret = self.client_secret_from_file(self.file_path)
+
+    def to_api_args(self) -> dict[str, Any]:
+        return {
+            self.CLIENT_SECRET: self.client_secret,
+        }
+
+    @classmethod
+    def is_part_of_api(cls, entity: dict[str, Any]) -> bool:
+        return cls.CLIENT_SECRET in entity
+
+    @classmethod
+    def from_api(cls, entity: dict[str, Any]) -> ClientSecretCredentialsFromFile:
+        # Credentials are not returned for security reasons
+        raise NotImplementedError
+
+    @staticmethod
+    def client_secret_from_file(file_path: Union[str, Path]) -> str:
+        with open(file_path, "rb") as fp:
+            return base64.b64encode(fp.read()).decode("utf-8")

gooddata-sdk/gooddata_sdk/catalog/parameter.py

@@ -1,5 +1,6 @@
 # (C) 2022 GoodData Corporation
 import attr
+from gooddata_api_client.model.data_source_parameter import DataSourceParameter
 from gooddata_api_client.model.parameter import Parameter
 
 from gooddata_sdk.catalog.base import Base
@@ -13,3 +14,6 @@ class CatalogParameter(Base):
     @staticmethod
     def client_class() -> type[Parameter]:
         return Parameter
+
+    def to_data_source_parameter(self) -> DataSourceParameter:
+        return DataSourceParameter(name=self.name, value=self.value)

gooddata-sdk/tests/catalog/expected/declarative_data_sources.json

@@ -23,6 +23,65 @@
       "type": "POSTGRESQL",
       "url": "jdbc:postgresql://localhost:5432/demo",
       "username": "demouser"
+    },
+    {
+      "id": "demo-test-ds-databricks-client-secret",
+      "name": "demo-test-ds-databricks-client-secret",
+      "permissions": [
+        {
+          "assignee": {
+            "id": "demo2",
+            "type": "user"
+          },
+          "name": "MANAGE"
+        },
+        {
+          "assignee": {
+            "id": "demoGroup",
+            "type": "userGroup"
+          },
+          "name": "USE"
+        }
+      ],
+      "schema": "demo",
+      "type": "DATABRICKS",
+      "url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;",
+      "clientId": "client-id",
+      "parameters": [
+        {
+          "name": "catalog",
+          "value": "demo"
+        }
+      ]
+    },
+    {
+      "id": "demo-test-ds-databricks-token",
+      "name": "demo-test-ds-databricks-token",
+      "permissions": [
+        {
+          "assignee": {
+            "id": "demo2",
+            "type": "user"
+          },
+          "name": "MANAGE"
+        },
+        {
+          "assignee": {
+            "id": "demoGroup",
+            "type": "userGroup"
+          },
+          "name": "USE"
+        }
+      ],
+      "schema": "demo",
+      "type": "DATABRICKS",
+      "url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;",
+      "parameters": [
+        {
+          "name": "catalog",
+          "value": "demo"
+        }
+      ]
     }
   ]
 }

gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_client_secret.json

@@ -0,0 +1,34 @@
+{
+  "dataSources": [
+    {
+      "id": "demo-test-ds-databricks-client-secret",
+      "name": "demo-test-ds-databricks-client-secret",
+      "permissions": [
+        {
+          "assignee": {
+            "id": "demo2",
+            "type": "user"
+          },
+          "name": "MANAGE"
+        },
+        {
+          "assignee": {
+            "id": "demoGroup",
+            "type": "userGroup"
+          },
+          "name": "USE"
+        }
+      ],
+      "schema": "demo",
+      "type": "DATABRICKS",
+      "url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;",
+      "clientId": "client-id",
+      "parameters": [
+        {
+          "name": "catalog",
+          "value": "demo"
+        }
+      ]
+    }
+  ]
+}

gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_token.json

@@ -0,0 +1,33 @@
+{
+  "dataSources": [
+    {
+      "id": "demo-test-ds-databricks-token",
+      "name": "demo-test-ds-databricks-token",
+      "permissions": [
+        {
+          "assignee": {
+            "id": "demo2",
+            "type": "user"
+          },
+          "name": "MANAGE"
+        },
+        {
+          "assignee": {
+            "id": "demoGroup",
+            "type": "userGroup"
+          },
+          "name": "USE"
+        }
+      ],
+      "schema": "demo",
+      "type": "DATABRICKS",
+      "url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;",
+      "parameters": [
+        {
+          "name": "catalog",
+          "value": "demo"
+        }
+      ]
+    }
+  ]
+}

gooddata-sdk/tests/catalog/fixtures/data_sources/demo_cache_strategy.yaml

@@ -1,4 +1,4 @@
-# (C) 2024 GoodData Corporation
+# (C) 2025 GoodData Corporation
 version: 1
 interactions:
   - request:
@@ -69,7 +69,7 @@ interactions:
         X-XSS-Protection:
           - '0'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -161,7 +161,7 @@ interactions:
         X-XSS-Protection:
           - '0'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -245,7 +245,7 @@ interactions:
         X-XSS-Protection:
           - '0'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -283,6 +283,43 @@ interactions:
                   type: userGroup
                 name: USE
             password: demopass
+          - id: demo-test-ds-databricks-client-secret
+            name: demo-test-ds-databricks-client-secret
+            schema: demo
+            type: DATABRICKS
+            url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;
+            parameters:
+              - name: catalog
+                value: demo
+            permissions:
+              - assignee:
+                  id: demo2
+                  type: user
+                name: MANAGE
+              - assignee:
+                  id: demoGroup
+                  type: userGroup
+                name: USE
+            clientId: client-id
+            clientSecret: databricks-client-secret
+          - id: demo-test-ds-databricks-token
+            name: demo-test-ds-databricks-token
+            schema: demo
+            type: DATABRICKS
+            url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;
+            parameters:
+              - name: catalog
+                value: demo
+            permissions:
+              - assignee:
+                  id: demo2
+                  type: user
+                name: MANAGE
+              - assignee:
+                  id: demoGroup
+                  type: userGroup
+                name: USE
+            token: databricks-token
       headers:
         Accept-Encoding:
           - br, gzip, deflate
@@ -342,7 +379,7 @@ interactions:
         X-XSS-Protection:
           - '0'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:51 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string: ''