From c4dbce17175e963ab8b123eab99b84aa661b3d53 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Thu, 10 Oct 2024 07:28:49 -0700 Subject: [PATCH 1/4] Make grant configs a no-op for DuckDB --- dbt/include/duckdb/macros/adapters.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dbt/include/duckdb/macros/adapters.sql b/dbt/include/duckdb/macros/adapters.sql index 84f2ac0b..442dc0bc 100644 --- a/dbt/include/duckdb/macros/adapters.sql +++ b/dbt/include/duckdb/macros/adapters.sql @@ -237,3 +237,10 @@ def materialize(df, con): {% do return(options) %} {%- endmacro %} + +{% macro duckdb__apply_grants(relation, grant_config, should_revoke=True) %} + {#-- If grant_config is {} or None, this is a no-op --#} + {% if grant_config %} + {{ log("Grants are not (currently!) supported for DuckDB", info=True) }} + {% endif %} +{% endmacro %} From bbe8b18295b575663c10f9c602cd4e59aad3636a Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Thu, 10 Oct 2024 08:33:55 -0700 Subject: [PATCH 2/4] Include the relation in question in the log message for the grant no-op --- dbt/include/duckdb/macros/adapters.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/duckdb/macros/adapters.sql b/dbt/include/duckdb/macros/adapters.sql index 442dc0bc..a8e6d0a5 100644 --- a/dbt/include/duckdb/macros/adapters.sql +++ b/dbt/include/duckdb/macros/adapters.sql @@ -241,6 +241,6 @@ def materialize(df, con): {% macro duckdb__apply_grants(relation, grant_config, should_revoke=True) %} {#-- If grant_config is {} or None, this is a no-op --#} {% if grant_config %} - {{ log("Grants are not (currently!) supported for DuckDB", info=True) }} + {{ log('Grants for relation ' ~ relation.render() ~ ' are not supported by DuckDB', info=True) }} {% endif %} {% endmacro %} From e39958d0ce18d6eccea1b0e8b04532c4ae070a9c Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Thu, 10 Oct 2024 11:25:45 -0700 Subject: [PATCH 3/4] Just send this to the dbt.log instead of to stdout to make it less noisy --- dbt/include/duckdb/macros/adapters.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/duckdb/macros/adapters.sql b/dbt/include/duckdb/macros/adapters.sql index a8e6d0a5..47bca241 100644 --- a/dbt/include/duckdb/macros/adapters.sql +++ b/dbt/include/duckdb/macros/adapters.sql @@ -241,6 +241,6 @@ def materialize(df, con): {% macro duckdb__apply_grants(relation, grant_config, should_revoke=True) %} {#-- If grant_config is {} or None, this is a no-op --#} {% if grant_config %} - {{ log('Grants for relation ' ~ relation.render() ~ ' are not supported by DuckDB', info=True) }} + {{ log('Grants for relation ' ~ relation.render() ~ ' are not supported by DuckDB') }} {% endif %} {% endmacro %} From 9011f115295ac8656844513f47e81b2fdfae49ae Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Thu, 10 Oct 2024 13:34:04 -0700 Subject: [PATCH 4/4] Move this message to a one-time warning function --- dbt/adapters/duckdb/connections.py | 11 +++++++++++ dbt/adapters/duckdb/impl.py | 15 +++++++++++++-- dbt/include/duckdb/macros/adapters.sql | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/duckdb/connections.py b/dbt/adapters/duckdb/connections.py index cbd68dc9..359e153e 100644 --- a/dbt/adapters/duckdb/connections.py +++ b/dbt/adapters/duckdb/connections.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from multiprocessing.context import SpawnContext from typing import Optional +from typing import Set from typing import Tuple from typing import TYPE_CHECKING @@ -25,6 +26,7 @@ class DuckDBConnectionManager(SQLConnectionManager): TYPE = "duckdb" _LOCK = threading.RLock() _ENV = None + _LOGGED_MESSAGES: Set[str] = set() def __init__(self, config: AdapterRequiredConfig, mp_context: SpawnContext) -> None: super().__init__(config, mp_context) @@ -68,6 +70,15 @@ def close(cls, connection: Connection) -> Connection: connection = super(SQLConnectionManager, cls).close(connection) return connection + @classmethod + def warn_once(cls, msg: str): + """Post a warning message once per dbt execution.""" + with cls._LOCK: + if msg in cls._LOGGED_MESSAGES: + return + cls._LOGGED_MESSAGES.add(msg) + logger.warning(msg) + def cancel(self, connection: Connection): if self._ENV is not None: logger.debug( diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index 89de7873..b24c01db 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -22,6 +22,7 @@ from dbt.adapters.duckdb.relation import DuckDBRelation from dbt.adapters.duckdb.utils import TargetConfig from dbt.adapters.duckdb.utils import TargetLocation +from dbt.adapters.events.logging import AdapterLogger from dbt.adapters.sql import SQLAdapter @@ -31,6 +32,8 @@ if TYPE_CHECKING: import agate +logger = AdapterLogger("DuckDB") + class DuckDBAdapter(SQLAdapter): ConnectionManager = DuckDBConnectionManager @@ -171,6 +174,11 @@ def external_read_location(self, write_location: str, rendered_options: dict) -> return ".".join(["/".join(globs), str(rendered_options.get("format", "parquet"))]) return write_location + @available + def warn_once(self, msg: str): + """Post a warning message once per dbt execution.""" + DuckDBConnectionManager.warn_once(msg) + def valid_incremental_strategies(self) -> Sequence[str]: """DuckDB does not currently support MERGE statement.""" return ["append", "delete+insert"] @@ -255,7 +263,8 @@ def _clean_up_temp_relation_for_incremental(self, config): if self.is_motherduck() and hasattr(config, "model"): if "incremental" == config.model.get_materialization(): temp_relation = self.Relation( - path=self.get_temp_relation_path(config.model), type=RelationType.Table + path=self.get_temp_relation_path(config.model), + type=RelationType.Table, ) self.drop_relation(temp_relation) @@ -278,7 +287,9 @@ def get_temp_relation_path(self, model: Any): table that is dropped at the end of the incremental macro or post-model hook. """ return Path( - schema=self._temp_schema_name, database=model.database, identifier=model.identifier + schema=self._temp_schema_name, + database=model.database, + identifier=model.identifier, ) def post_model_hook(self, config: Any, context: Any) -> None: diff --git a/dbt/include/duckdb/macros/adapters.sql b/dbt/include/duckdb/macros/adapters.sql index 47bca241..29d2d570 100644 --- a/dbt/include/duckdb/macros/adapters.sql +++ b/dbt/include/duckdb/macros/adapters.sql @@ -241,6 +241,6 @@ def materialize(df, con): {% macro duckdb__apply_grants(relation, grant_config, should_revoke=True) %} {#-- If grant_config is {} or None, this is a no-op --#} {% if grant_config %} - {{ log('Grants for relation ' ~ relation.render() ~ ' are not supported by DuckDB') }} + {{ adapter.warn_once('Grants for relations are not supported by DuckDB') }} {% endif %} {% endmacro %}