From 95a5ad0ee0cab5854b9b184917d8bf1b40221efc Mon Sep 17 00:00:00 2001 From: Christopher Bonilla Date: Mon, 6 Jan 2025 10:50:27 +0100 Subject: [PATCH] feat: databricks token + m2m auth support Added support for loading and putting databricks datasources using token or m2m auth as well as testing the data source connection. JIRA: LX-691 risk: low --- .python-version | 2 +- .../declarative_model/data_source.py | 17 +- .../catalog/data_source/service.py | 18 +- gooddata-sdk/gooddata_sdk/catalog/entity.py | 28 ++++ .../gooddata_sdk/catalog/parameter.py | 4 + .../expected/declarative_data_sources.json | 59 +++++++ ...data_sources_databricks_client_secret.json | 34 ++++ ...arative_data_sources_databricks_token.json | 33 ++++ .../data_sources/demo_cache_strategy.yaml | 47 +++++- .../demo_delete_declarative_data_sources.yaml | 45 ++++- ...load_and_put_declarative_data_sources.yaml | 130 ++++++++++++-- .../demo_put_declarative_data_sources.yaml | 158 +++++++++++++++++- ...t_declarative_data_sources_connection.yaml | 88 +++++++++- .../demo_store_declarative_data_sources.yaml | 98 +++++++++-- .../demo_test_declarative_data_sources.yaml | 45 ++++- .../data_sources_credentials.yaml | 2 + ...demo-test-ds-databricks-client-secret.yaml | 19 +++ .../demo-test-ds-databricks-token.yaml | 18 ++ .../tests/catalog/test_catalog_data_source.py | 101 ++++++++++- gooddata-sdk/tests/gd_test_config.yaml | 2 + 20 files changed, 887 insertions(+), 61 deletions(-) create mode 100644 gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_client_secret.json create mode 100644 gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_token.json create mode 100644 gooddata-sdk/tests/catalog/load/gooddata_layouts/default/data_sources/demo-test-ds-databricks-client-secret/demo-test-ds-databricks-client-secret.yaml create mode 100644 gooddata-sdk/tests/catalog/load/gooddata_layouts/default/data_sources/demo-test-ds-databricks-token/demo-test-ds-databricks-token.yaml diff --git a/.python-version b/.python-version index c10780c62..92536a9e4 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.13.1 +3.12.0 diff --git a/gooddata-sdk/gooddata_sdk/catalog/data_source/declarative_model/data_source.py b/gooddata-sdk/gooddata_sdk/catalog/data_source/declarative_model/data_source.py index 6726deb8c..d9e04a759 100644 --- a/gooddata-sdk/gooddata_sdk/catalog/data_source/declarative_model/data_source.py +++ b/gooddata-sdk/gooddata_sdk/catalog/data_source/declarative_model/data_source.py @@ -12,12 +12,13 @@ from gooddata_api_client.model.test_definition_request import TestDefinitionRequest from gooddata_sdk.catalog.base import Base, value_in_allowed -from gooddata_sdk.catalog.entity import TokenCredentialsFromFile +from gooddata_sdk.catalog.entity import ClientSecretCredentialsFromFile, TokenCredentialsFromFile from gooddata_sdk.catalog.parameter import CatalogParameter from gooddata_sdk.catalog.permission.declarative_model.permission import CatalogDeclarativeDataSourcePermission from gooddata_sdk.utils import create_directory, get_ds_credentials, read_layout_from_file, write_layout_to_file BIGQUERY_TYPE = "BIGQUERY" +DATABRICKS_TYPE = "DATABRICKS" LAYOUT_DATA_SOURCES_DIR = "data_sources" @@ -33,6 +34,15 @@ def _inject_base(self, credentials: dict[str, Any]) -> DeclarativeDataSources: if data_source.type == BIGQUERY_TYPE: token = TokenCredentialsFromFile.token_from_file(credentials[data_source.id]) data_sources.append(data_source.to_api(token=token)) + elif data_source.type == DATABRICKS_TYPE: + if data_source.client_id is not None: + client_secret = ClientSecretCredentialsFromFile.client_secret_from_file( + credentials[data_source.id] + ) + data_sources.append(data_source.to_api(client_secret=client_secret)) + else: + token = TokenCredentialsFromFile.token_from_file(credentials[data_source.id]) + data_sources.append(data_source.to_api(token=token)) else: data_sources.append(data_source.to_api(password=credentials[data_source.id])) else: @@ -125,8 +135,13 @@ def to_test_request( kwargs["private_key"] = private_key if private_key_passphrase is not None: kwargs["private_key_passphrase"] = private_key + if self.client_id is not None: + kwargs["client_id"] = self.client_id if client_secret is not None: kwargs["client_secret"] = client_secret + if self.parameters is not None: + kwargs["parameters"] = [param.to_data_source_parameter() for param in self.parameters] + return TestDefinitionRequest(type=self.type, url=self.url, **kwargs) @staticmethod diff --git a/gooddata-sdk/gooddata_sdk/catalog/data_source/service.py b/gooddata-sdk/gooddata_sdk/catalog/data_source/service.py index 86ab5c57f..920b9725b 100644 --- a/gooddata-sdk/gooddata_sdk/catalog/data_source/service.py +++ b/gooddata-sdk/gooddata_sdk/catalog/data_source/service.py @@ -17,6 +17,7 @@ from gooddata_sdk.catalog.data_source.action_model.responses.scan_sql_response import ScanSqlResponse from gooddata_sdk.catalog.data_source.declarative_model.data_source import ( BIGQUERY_TYPE, + DATABRICKS_TYPE, CatalogDeclarativeDataSource, CatalogDeclarativeDataSources, ) @@ -25,7 +26,7 @@ CatalogScanResultPdm, ) from gooddata_sdk.catalog.data_source.entity_model.data_source import CatalogDataSource -from gooddata_sdk.catalog.entity import TokenCredentialsFromFile +from gooddata_sdk.catalog.entity import ClientSecretCredentialsFromFile, TokenCredentialsFromFile from gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.ldm import CatalogDeclarativeModel from gooddata_sdk.client import GoodDataApiClient from gooddata_sdk.utils import get_ds_credentials, load_all_entities_dict, read_layout_from_file @@ -474,6 +475,19 @@ def test_data_sources_connection( response = self._actions_api.test_data_source_definition( declarative_data_source.to_test_request(token=token) ) + elif declarative_data_source.type == DATABRICKS_TYPE: + if declarative_data_source.client_id is not None: + client_secret = ClientSecretCredentialsFromFile.client_secret_from_file( + credentials[declarative_data_source.id] + ) + response = self._actions_api.test_data_source_definition( + declarative_data_source.to_test_request(client_secret=client_secret) + ) + else: + token = TokenCredentialsFromFile.token_from_file(credentials[declarative_data_source.id]) + response = self._actions_api.test_data_source_definition( + declarative_data_source.to_test_request(token=token) + ) else: response = self._actions_api.test_data_source_definition( declarative_data_source.to_test_request(password=credentials[declarative_data_source.id]) @@ -528,6 +542,6 @@ def _credentials_from_file(credentials_path: Path) -> dict[str, Any]: if data.get("data_sources") is None: raise ValueError("The file has a wrong structure. There should be a root key 'data_sources'.") if len(data["data_sources"]) == 0: - raise ValueError("There are no pairs of data source id and token.") + raise ValueError("There are no pairs of data source id and credentials.") credentials = data["data_sources"] return credentials diff --git a/gooddata-sdk/gooddata_sdk/catalog/entity.py b/gooddata-sdk/gooddata_sdk/catalog/entity.py index 6d1ef069c..289d0156c 100644 --- a/gooddata-sdk/gooddata_sdk/catalog/entity.py +++ b/gooddata-sdk/gooddata_sdk/catalog/entity.py @@ -274,3 +274,31 @@ def from_api(cls, attributes: dict[str, Any]) -> ClientSecretCredentials: # You have to fill it to keep it or update it client_secret="", ) + + +@attr.s(auto_attribs=True, kw_only=True) +class ClientSecretCredentialsFromFile(Credentials): + file_path: Path + client_secret: str = attr.field(init=False, repr=lambda value: "***") + + def __attrs_post_init__(self) -> None: + self.client_secret = self.client_secret_from_file(self.file_path) + + def to_api_args(self) -> dict[str, Any]: + return { + self.CLIENT_SECRET: self.client_secret, + } + + @classmethod + def is_part_of_api(cls, entity: dict[str, Any]) -> bool: + return cls.CLIENT_SECRET in entity + + @classmethod + def from_api(cls, entity: dict[str, Any]) -> ClientSecretCredentialsFromFile: + # Credentials are not returned for security reasons + raise NotImplementedError + + @staticmethod + def client_secret_from_file(file_path: Union[str, Path]) -> str: + with open(file_path, "rb") as fp: + return base64.b64encode(fp.read()).decode("utf-8") diff --git a/gooddata-sdk/gooddata_sdk/catalog/parameter.py b/gooddata-sdk/gooddata_sdk/catalog/parameter.py index f26dd7524..ff02ce4a7 100644 --- a/gooddata-sdk/gooddata_sdk/catalog/parameter.py +++ b/gooddata-sdk/gooddata_sdk/catalog/parameter.py @@ -1,5 +1,6 @@ # (C) 2022 GoodData Corporation import attr +from gooddata_api_client.model.data_source_parameter import DataSourceParameter from gooddata_api_client.model.parameter import Parameter from gooddata_sdk.catalog.base import Base @@ -13,3 +14,6 @@ class CatalogParameter(Base): @staticmethod def client_class() -> type[Parameter]: return Parameter + + def to_data_source_parameter(self) -> DataSourceParameter: + return DataSourceParameter(name=self.name, value=self.value) diff --git a/gooddata-sdk/tests/catalog/expected/declarative_data_sources.json b/gooddata-sdk/tests/catalog/expected/declarative_data_sources.json index 20b93012b..38dafdb21 100644 --- a/gooddata-sdk/tests/catalog/expected/declarative_data_sources.json +++ b/gooddata-sdk/tests/catalog/expected/declarative_data_sources.json @@ -23,6 +23,65 @@ "type": "POSTGRESQL", "url": "jdbc:postgresql://localhost:5432/demo", "username": "demouser" + }, + { + "id": "demo-test-ds-databricks-client-secret", + "name": "demo-test-ds-databricks-client-secret", + "permissions": [ + { + "assignee": { + "id": "demo2", + "type": "user" + }, + "name": "MANAGE" + }, + { + "assignee": { + "id": "demoGroup", + "type": "userGroup" + }, + "name": "USE" + } + ], + "schema": "demo", + "type": "DATABRICKS", + "url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;", + "clientId": "client-id", + "parameters": [ + { + "name": "catalog", + "value": "demo" + } + ] + }, + { + "id": "demo-test-ds-databricks-token", + "name": "demo-test-ds-databricks-token", + "permissions": [ + { + "assignee": { + "id": "demo2", + "type": "user" + }, + "name": "MANAGE" + }, + { + "assignee": { + "id": "demoGroup", + "type": "userGroup" + }, + "name": "USE" + } + ], + "schema": "demo", + "type": "DATABRICKS", + "url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;", + "parameters": [ + { + "name": "catalog", + "value": "demo" + } + ] } ] } diff --git a/gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_client_secret.json b/gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_client_secret.json new file mode 100644 index 000000000..5a4211ad0 --- /dev/null +++ b/gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_client_secret.json @@ -0,0 +1,34 @@ +{ + "dataSources": [ + { + "id": "demo-test-ds-databricks-client-secret", + "name": "demo-test-ds-databricks-client-secret", + "permissions": [ + { + "assignee": { + "id": "demo2", + "type": "user" + }, + "name": "MANAGE" + }, + { + "assignee": { + "id": "demoGroup", + "type": "userGroup" + }, + "name": "USE" + } + ], + "schema": "demo", + "type": "DATABRICKS", + "url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;", + "clientId": "client-id", + "parameters": [ + { + "name": "catalog", + "value": "demo" + } + ] + } + ] +} diff --git a/gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_token.json b/gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_token.json new file mode 100644 index 000000000..90d8a5833 --- /dev/null +++ b/gooddata-sdk/tests/catalog/expected/declarative_data_sources_databricks_token.json @@ -0,0 +1,33 @@ +{ + "dataSources": [ + { + "id": "demo-test-ds-databricks-token", + "name": "demo-test-ds-databricks-token", + "permissions": [ + { + "assignee": { + "id": "demo2", + "type": "user" + }, + "name": "MANAGE" + }, + { + "assignee": { + "id": "demoGroup", + "type": "userGroup" + }, + "name": "USE" + } + ], + "schema": "demo", + "type": "DATABRICKS", + "url": "jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa;", + "parameters": [ + { + "name": "catalog", + "value": "demo" + } + ] + } + ] +} diff --git a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_cache_strategy.yaml b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_cache_strategy.yaml index c7c2f4dfe..77c29d743 100644 --- a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_cache_strategy.yaml +++ b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_cache_strategy.yaml @@ -1,4 +1,4 @@ -# (C) 2024 GoodData Corporation +# (C) 2025 GoodData Corporation version: 1 interactions: - request: @@ -69,7 +69,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -161,7 +161,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -245,7 +245,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -283,6 +283,43 @@ interactions: type: userGroup name: USE password: demopass + - id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + clientId: client-id + clientSecret: databricks-client-secret + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + token: databricks-token headers: Accept-Encoding: - br, gzip, deflate @@ -342,7 +379,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:51 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' diff --git a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_delete_declarative_data_sources.yaml b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_delete_declarative_data_sources.yaml index 9f4e94fe6..eb8163177 100644 --- a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_delete_declarative_data_sources.yaml +++ b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_delete_declarative_data_sources.yaml @@ -1,4 +1,4 @@ -# (C) 2024 GoodData Corporation +# (C) 2025 GoodData Corporation version: 1 interactions: - request: @@ -66,7 +66,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' @@ -137,7 +137,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -163,6 +163,43 @@ interactions: type: userGroup name: USE password: demopass + - id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + clientId: client-id + clientSecret: databricks-client-secret + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + token: databricks-token headers: Accept-Encoding: - br, gzip, deflate @@ -222,7 +259,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' diff --git a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_load_and_put_declarative_data_sources.yaml b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_load_and_put_declarative_data_sources.yaml index aae0ac490..593a0ba1c 100644 --- a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_load_and_put_declarative_data_sources.yaml +++ b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_load_and_put_declarative_data_sources.yaml @@ -1,4 +1,4 @@ -# (C) 2024 GoodData Corporation +# (C) 2025 GoodData Corporation version: 1 interactions: - request: @@ -66,7 +66,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' @@ -135,7 +135,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' @@ -164,7 +164,7 @@ interactions: Connection: - keep-alive Content-Length: - - '275' + - '255' Content-Security-Policy: - 'default-src ''self'' *.wistia.com *.wistia.net; script-src ''self'' ''unsafe-inline'' ''unsafe-eval'' *.wistia.com *.wistia.net *.hsforms.net *.hsforms.com @@ -204,7 +204,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -214,8 +214,7 @@ interactions: attributes: name: Default Organization hostname: localhost - allowedOrigins: [] - oauthClientId: b538d572-c26e-4758-a98b-7ac92840dfa5 + oauthClientId: e76fea3d-624e-4a32-9b15-912e7b107085 links: self: http://localhost:3000/api/v1/entities/admin/organizations/default - request: @@ -251,6 +250,43 @@ interactions: type: userGroup name: USE password: demopass + - id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + clientId: client-id + clientSecret: databricks-client-secret + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + token: databricks-token - id: demo-vertica-ds name: demo-vertica-ds schema: demo @@ -316,7 +352,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:47 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' @@ -387,9 +423,9 @@ interactions: X-XSS-Protection: - '0' content-length: - - '773' + - '1698' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:47 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -424,6 +460,41 @@ interactions: type: POSTGRESQL url: jdbc:postgresql://localhost:5432/demo username: demouser + - clientId: client-id + id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa - id: demo-vertica-ds name: demo-vertica-ds permissions: [] @@ -451,6 +522,43 @@ interactions: type: userGroup name: USE password: demopass + - id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + clientId: client-id + clientSecret: databricks-client-secret + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + token: databricks-token headers: Accept-Encoding: - br, gzip, deflate @@ -510,7 +618,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:47 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' diff --git a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_put_declarative_data_sources.yaml b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_put_declarative_data_sources.yaml index 0cab990f0..e1fc5aeab 100644 --- a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_put_declarative_data_sources.yaml +++ b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_put_declarative_data_sources.yaml @@ -1,4 +1,4 @@ -# (C) 2024 GoodData Corporation +# (C) 2025 GoodData Corporation version: 1 interactions: - request: @@ -69,9 +69,9 @@ interactions: X-XSS-Protection: - '0' content-length: - - '303' + - '1228' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:48 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -91,6 +91,41 @@ interactions: type: POSTGRESQL url: jdbc:postgresql://localhost:5432/demo username: demouser + - clientId: client-id + id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa - request: method: PUT uri: http://localhost:3000/api/v1/layout/dataSources @@ -112,6 +147,43 @@ interactions: type: userGroup name: USE password: demopass + - id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + clientId: client-id + clientSecret: databricks-client-secret + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + token: databricks-token headers: Accept-Encoding: - br, gzip, deflate @@ -171,7 +243,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:49 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' @@ -242,9 +314,9 @@ interactions: X-XSS-Protection: - '0' content-length: - - '303' + - '1228' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:49 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -264,6 +336,41 @@ interactions: type: POSTGRESQL url: jdbc:postgresql://localhost:5432/demo username: demouser + - clientId: client-id + id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa - request: method: PUT uri: http://localhost:3000/api/v1/layout/dataSources @@ -285,6 +392,43 @@ interactions: type: userGroup name: USE password: demopass + - id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + clientId: client-id + clientSecret: databricks-client-secret + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + token: databricks-token headers: Accept-Encoding: - br, gzip, deflate @@ -344,7 +488,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:49 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' diff --git a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_put_declarative_data_sources_connection.yaml b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_put_declarative_data_sources_connection.yaml index 3960ffac0..c047a722b 100644 --- a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_put_declarative_data_sources_connection.yaml +++ b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_put_declarative_data_sources_connection.yaml @@ -1,4 +1,4 @@ -# (C) 2024 GoodData Corporation +# (C) 2025 GoodData Corporation version: 1 interactions: - request: @@ -69,9 +69,9 @@ interactions: X-XSS-Protection: - '0' content-length: - - '303' + - '1228' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:47 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -91,6 +91,41 @@ interactions: type: POSTGRESQL url: jdbc:postgresql://localhost:5432/demo username: demouser + - clientId: client-id + id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa - request: method: POST uri: http://localhost:3000/api/v1/actions/dataSource/test @@ -167,13 +202,13 @@ interactions: content-length: - '81' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:48 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: queryDurationMillis: createCacheTable: 0 - simpleSelect: 0 + simpleSelect: 2 successful: true - request: method: PUT @@ -255,7 +290,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:48 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' @@ -328,7 +363,7 @@ interactions: content-length: - '303' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:48 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -369,6 +404,43 @@ interactions: type: userGroup name: USE password: demopass + - id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + clientId: client-id + clientSecret: databricks-client-secret + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + token: databricks-token headers: Accept-Encoding: - br, gzip, deflate @@ -428,7 +500,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:48 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' diff --git a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_store_declarative_data_sources.yaml b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_store_declarative_data_sources.yaml index 98a9075bf..8c45698d4 100644 --- a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_store_declarative_data_sources.yaml +++ b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_store_declarative_data_sources.yaml @@ -1,4 +1,4 @@ -# (C) 2024 GoodData Corporation +# (C) 2025 GoodData Corporation version: 1 interactions: - request: @@ -69,9 +69,9 @@ interactions: X-XSS-Protection: - '0' content-length: - - '303' + - '1228' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -91,6 +91,41 @@ interactions: type: POSTGRESQL url: jdbc:postgresql://localhost:5432/demo username: demouser + - clientId: client-id + id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa - request: method: GET uri: http://localhost:3000/api/v1/layout/dataSources @@ -158,9 +193,9 @@ interactions: X-XSS-Protection: - '0' content-length: - - '303' + - '1228' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -180,6 +215,41 @@ interactions: type: POSTGRESQL url: jdbc:postgresql://localhost:5432/demo username: demouser + - clientId: client-id + id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa - request: method: GET uri: http://localhost:3000/api/v1/entities/organization @@ -245,7 +315,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' @@ -274,7 +344,7 @@ interactions: Connection: - keep-alive Content-Length: - - '275' + - '255' Content-Security-Policy: - 'default-src ''self'' *.wistia.com *.wistia.net; script-src ''self'' ''unsafe-inline'' ''unsafe-eval'' *.wistia.com *.wistia.net *.hsforms.net *.hsforms.com @@ -314,7 +384,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -324,8 +394,7 @@ interactions: attributes: name: Default Organization hostname: localhost - allowedOrigins: [] - oauthClientId: b538d572-c26e-4758-a98b-7ac92840dfa5 + oauthClientId: e76fea3d-624e-4a32-9b15-912e7b107085 links: self: http://localhost:3000/api/v1/entities/admin/organizations/default - request: @@ -393,7 +462,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: '' @@ -422,7 +491,7 @@ interactions: Connection: - keep-alive Content-Length: - - '275' + - '255' Content-Security-Policy: - 'default-src ''self'' *.wistia.com *.wistia.net; script-src ''self'' ''unsafe-inline'' ''unsafe-eval'' *.wistia.com *.wistia.net *.hsforms.net *.hsforms.com @@ -462,7 +531,7 @@ interactions: X-XSS-Protection: - '0' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:25 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:46 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -472,7 +541,6 @@ interactions: attributes: name: Default Organization hostname: localhost - allowedOrigins: [] - oauthClientId: b538d572-c26e-4758-a98b-7ac92840dfa5 + oauthClientId: e76fea3d-624e-4a32-9b15-912e7b107085 links: self: http://localhost:3000/api/v1/entities/admin/organizations/default diff --git a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_test_declarative_data_sources.yaml b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_test_declarative_data_sources.yaml index e3262a431..3cffd6cf2 100644 --- a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_test_declarative_data_sources.yaml +++ b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_test_declarative_data_sources.yaml @@ -1,4 +1,4 @@ -# (C) 2024 GoodData Corporation +# (C) 2025 GoodData Corporation version: 1 interactions: - request: @@ -69,9 +69,9 @@ interactions: X-XSS-Protection: - '0' content-length: - - '303' + - '1228' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:26 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:49 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -91,6 +91,41 @@ interactions: type: POSTGRESQL url: jdbc:postgresql://localhost:5432/demo username: demouser + - clientId: client-id + id: demo-test-ds-databricks-client-secret + name: demo-test-ds-databricks-client-secret + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa + - id: demo-test-ds-databricks-token + name: demo-test-ds-databricks-token + parameters: + - name: catalog + value: demo + permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE + schema: demo + type: DATABRICKS + url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;UserAgentEntry=gooddata+cloud-native/v3;httpPath=/sql/1.0/warehouses/9876fdsa - request: method: POST uri: http://localhost:3000/api/v1/actions/dataSource/test @@ -166,7 +201,7 @@ interactions: content-length: - '274' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: @@ -251,7 +286,7 @@ interactions: content-length: - '81' set-cookie: - - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:17:27 GMT; + - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 06 Jan 2025 11:44:50 GMT; Path=/; HTTPOnly; SameSite=Lax body: string: diff --git a/gooddata-sdk/tests/catalog/load/data_source_credentials/data_sources_credentials.yaml b/gooddata-sdk/tests/catalog/load/data_source_credentials/data_sources_credentials.yaml index 22fd5dc78..5a64e3594 100644 --- a/gooddata-sdk/tests/catalog/load/data_source_credentials/data_sources_credentials.yaml +++ b/gooddata-sdk/tests/catalog/load/data_source_credentials/data_sources_credentials.yaml @@ -2,3 +2,5 @@ data_sources: demo-test-ds: "demopass" demo-bigquery-ds: "~/home/secrets.json" + demo-test-ds-databricks-client-secret: "databricks-client-secret" + demo-test-ds-databricks-token: "databricks-token" diff --git a/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/data_sources/demo-test-ds-databricks-client-secret/demo-test-ds-databricks-client-secret.yaml b/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/data_sources/demo-test-ds-databricks-client-secret/demo-test-ds-databricks-client-secret.yaml new file mode 100644 index 000000000..3d86bb6fe --- /dev/null +++ b/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/data_sources/demo-test-ds-databricks-client-secret/demo-test-ds-databricks-client-secret.yaml @@ -0,0 +1,19 @@ +# (C) 2024 GoodData Corporation +id: demo-test-ds-databricks-client-secret +name: demo-test-ds-databricks-client-secret +permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE +schema: demo +type: DATABRICKS +url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; +clientId: client-id +parameters: + - name: catalog + value: "demo" diff --git a/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/data_sources/demo-test-ds-databricks-token/demo-test-ds-databricks-token.yaml b/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/data_sources/demo-test-ds-databricks-token/demo-test-ds-databricks-token.yaml new file mode 100644 index 000000000..43482f094 --- /dev/null +++ b/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/data_sources/demo-test-ds-databricks-token/demo-test-ds-databricks-token.yaml @@ -0,0 +1,18 @@ +# (C) 2024 GoodData Corporation +id: demo-test-ds-databricks-token +name: demo-test-ds-databricks-token +permissions: + - assignee: + id: demo2 + type: user + name: MANAGE + - assignee: + id: demoGroup + type: userGroup + name: USE +schema: demo +type: DATABRICKS +url: jdbc:databricks://dbc-1234-abc.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/9876fdsa; +parameters: + - name: catalog + value: "demo" diff --git a/gooddata-sdk/tests/catalog/test_catalog_data_source.py b/gooddata-sdk/tests/catalog/test_catalog_data_source.py index 377ccffbd..41d3ba616 100644 --- a/gooddata-sdk/tests/catalog/test_catalog_data_source.py +++ b/gooddata-sdk/tests/catalog/test_catalog_data_source.py @@ -43,6 +43,7 @@ VerticaAttributes, ) from gooddata_sdk.catalog.data_source.entity_model.data_source import DatabaseAttributes +from gooddata_sdk.catalog.entity import ClientSecretCredentialsFromFile from tests_support.file_utils import load_json from tests_support.vcrpy_utils import get_vcr @@ -415,6 +416,19 @@ def test_delete_declarative_data_sources(test_config): credentials_path = _current_dir / "load" / "data_source_credentials" / "data_sources_credentials.yaml" expected_json_path = _current_dir / "expected" / "declarative_data_sources.json" + def token_from_file_side_effect(arg): + if arg == "~/home/secrets.json": + return test_config["bigquery_token"] + elif arg == "databricks-token": + return test_config["databricks_token"] + else: + raise ValueError(f"Unexpected argument: {arg}") + + TokenCredentialsFromFile.token_from_file = MagicMock(side_effect=token_from_file_side_effect) + ClientSecretCredentialsFromFile.client_secret_from_file = MagicMock( + return_value=test_config["databricks_client_secret"] + ) + try: sdk.catalog_data_source.put_declarative_data_sources(CatalogDeclarativeDataSources(data_sources=[])) data_sources_o = sdk.catalog_data_source.get_declarative_data_sources() @@ -445,18 +459,34 @@ def test_load_and_put_declarative_data_sources(test_config): expected_json_path = _current_dir / "expected" / "declarative_data_sources.json" try: sdk.catalog_data_source.put_declarative_data_sources(CatalogDeclarativeDataSources(data_sources=[])) - TokenCredentialsFromFile.token_from_file = MagicMock(return_value=test_config["bigquery_token"]) + + def token_from_file_side_effect(arg): + if arg == "~/home/secrets.json": + return test_config["bigquery_token"] + elif arg == "databricks-token": + return test_config["databricks_token"] + else: + raise ValueError(f"Unexpected argument: {arg}") + + TokenCredentialsFromFile.token_from_file = MagicMock(side_effect=token_from_file_side_effect) + ClientSecretCredentialsFromFile.client_secret_from_file = MagicMock( + return_value=test_config["databricks_client_secret"] + ) sdk.catalog_data_source.load_and_put_declarative_data_sources(load_folder, credentials_path) data_sources_o = sdk.catalog_data_source.get_declarative_data_sources() - assert len(data_sources_o.data_sources) == 3 + assert len(data_sources_o.data_sources) == 5 assert [data_source.id for data_source in data_sources_o.data_sources] == [ "demo-bigquery-ds", "demo-test-ds", + "demo-test-ds-databricks-client-secret", + "demo-test-ds-databricks-token", "demo-vertica-ds", ] assert [data_source.type for data_source in data_sources_o.data_sources] == [ "BIGQUERY", "POSTGRESQL", + "DATABRICKS", + "DATABRICKS", "VERTICA", ] assert len(data_sources_o.data_sources[0].parameters) == 1 @@ -473,6 +503,21 @@ def test_put_declarative_data_sources_connection(test_config): path = _current_dir / "expected" / "declarative_data_sources.json" credentials_path = _current_dir / "load" / "data_source_credentials" / "data_sources_credentials.yaml" data_sources_e = sdk.catalog_data_source.get_declarative_data_sources() + # Must filter out databricks data sources for this test as they do not have valid URLs + data_sources_e.data_sources = [item for item in data_sources_e.data_sources if "databricks" not in item.id] + + def token_from_file_side_effect(arg): + if arg == "~/home/secrets.json": + return test_config["bigquery_token"] + elif arg == "databricks-token": + return test_config["databricks_token"] + else: + raise ValueError(f"Unexpected argument: {arg}") + + TokenCredentialsFromFile.token_from_file = MagicMock(side_effect=token_from_file_side_effect) + ClientSecretCredentialsFromFile.client_secret_from_file = MagicMock( + return_value=test_config["databricks_client_secret"] + ) try: sdk.catalog_data_source.put_declarative_data_sources(data_sources_e, credentials_path, test_data_sources=True) @@ -506,6 +551,8 @@ def test_declarative_data_sources(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) credentials_path = _current_dir / "load" / "data_source_credentials" / "data_sources_credentials.yaml" data_sources_e = sdk.catalog_data_source.get_declarative_data_sources() + # Must filter out databricks data sources for this test as they do not have valid URLs + data_sources_e.data_sources = [item for item in data_sources_e.data_sources if "databricks" not in item.id] try: sdk.catalog_data_source.test_data_sources_connection(data_sources_e) @@ -514,6 +561,43 @@ def test_declarative_data_sources(test_config): sdk.catalog_data_source.test_data_sources_connection(data_sources_e, credentials_path) +@gd_vcr.use_cassette(str(_fixtures_dir / "demo_test_declarative_data_sources_databricks_client_secret.yaml")) +def test_declarative_data_sources_databricks_client_secret(test_config): + sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) + path = _current_dir / "expected" / "declarative_data_sources_databricks_client_secret.json" + ClientSecretCredentialsFromFile.client_secret_from_file = MagicMock( + return_value=test_config["databricks_client_secret"] + ) + sdk._client._actions_api.test_data_source_definition = MagicMock() + + credentials_path = _current_dir / "load" / "data_source_credentials" / "data_sources_credentials.yaml" + data_sources_e = CatalogDeclarativeDataSources.from_dict(load_json(path)) + + sdk.catalog_data_source.test_data_sources_connection(data_sources_e, credentials_path) + args, kwargs = sdk._client._actions_api.test_data_source_definition.call_args + assert len(args) == 1 + assert args[0]._data_store["url"] == data_sources_e.data_sources[0].url + assert args[0]._data_store["client_id"] == data_sources_e.data_sources[0].client_id + assert args[0]._data_store["client_secret"] == test_config["databricks_client_secret"] + + +@gd_vcr.use_cassette(str(_fixtures_dir / "demo_test_declarative_data_sources_databricks_token.yaml")) +def test_declarative_data_sources_databricks_token(test_config): + sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) + path = _current_dir / "expected" / "declarative_data_sources_databricks_token.json" + TokenCredentialsFromFile.token_from_file = MagicMock(return_value=test_config["databricks_token"]) + sdk._client._actions_api.test_data_source_definition = MagicMock() + + credentials_path = _current_dir / "load" / "data_source_credentials" / "data_sources_credentials.yaml" + data_sources_e = CatalogDeclarativeDataSources.from_dict(load_json(path)) + + sdk.catalog_data_source.test_data_sources_connection(data_sources_e, credentials_path) + args, kwargs = sdk._client._actions_api.test_data_source_definition.call_args + assert len(args) == 1 + assert args[0]._data_store["url"] == data_sources_e.data_sources[0].url + assert args[0]._data_store["token"] == test_config["databricks_token"] + + @gd_vcr.use_cassette(str(_fixtures_dir / "demo_cache_strategy.yaml")) def test_cache_strategy(test_config: dict): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) @@ -521,6 +605,19 @@ def test_cache_strategy(test_config: dict): path = _current_dir / "expected" / "declarative_data_sources.json" credentials_path = _current_dir / "load" / "data_source_credentials" / "data_sources_credentials.yaml" + def token_from_file_side_effect(arg): + if arg == "~/home/secrets.json": + return test_config["bigquery_token"] + elif arg == "databricks-token": + return test_config["databricks_token"] + else: + raise ValueError(f"Unexpected argument: {arg}") + + TokenCredentialsFromFile.token_from_file = MagicMock(side_effect=token_from_file_side_effect) + ClientSecretCredentialsFromFile.client_secret_from_file = MagicMock( + return_value=test_config["databricks_client_secret"] + ) + try: sdk.catalog_data_source.patch_data_source_attributes(data_source_id, {"cache_strategy": "NEVER"}) updated = sdk.catalog_data_source.get_data_source(data_source_id=data_source_id) diff --git a/gooddata-sdk/tests/gd_test_config.yaml b/gooddata-sdk/tests/gd_test_config.yaml index 7951b884d..a2e74e1b8 100644 --- a/gooddata-sdk/tests/gd_test_config.yaml +++ b/gooddata-sdk/tests/gd_test_config.yaml @@ -20,3 +20,5 @@ bigquery_token: "eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsICJwcm9qZWN0X2lkIjogIlBST0p bigquery_token_file: '{"type": "service_account", "project_id": "PROJECT_ID", "private_key_id": "KEY_ID", "private_key": "-----BEGIN PRIVATE KEY-----\\nPRIVATE_KEY\\n-----END PRIVATE KEY-----\\n", "client_email": "SERVICE_ACCOUNT_EMAIL", "client_id": "CLIENT_ID", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://accounts.google.com/o/oauth2/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/SERVICE_ACCOUNT_EMAIL"}' visualization_id: "customers_trend" visualization_name: "Customers Trend" +databricks_client_secret: "databricks-client-secret" +databricks_token: "databricks-token"