From f26fadd46e1bb66b490207f1e65a16b51ec45635 Mon Sep 17 00:00:00 2001 From: Victoria Perez Mola <4315804+Victoriapm@users.noreply.github.com> Date: Fri, 17 Feb 2023 12:28:51 +0100 Subject: [PATCH] dbt Constraints / model contracts (#426) * Create bigquery adapter macro for constraints * fix order * update checks validation and constraints list * add test * add changie * update issue number * fix trailing space * fix test name * update BQ data types * Update test_bigquery_constraints.py * Update test_bigquery_constraints.py * fix quotes Co-authored-by: Sung Won Chung * Update test_bigquery_constraints.py * fix constratints config * fix config * Update test_bigquery_constraints.py * update constraints config * Switch to new functional tests * Fix failing tests * Small cleanup * Reset to dbt-core main * Resolve PR comment * PR feedback --------- Co-authored-by: Sung Won Chung Co-authored-by: Jeremy Cohen Co-authored-by: Michelle Ark --- .../unreleased/Features-20221220-193731.yaml | 7 +++ dbt/include/bigquery/macros/adapters.sql | 5 ++ .../macros/utils/get_columns_spec_ddl.sql | 16 +++++ dev-requirements.txt | 1 - tests/functional/adapter/test_constraints.py | 59 +++++++++++++++++++ 5 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20221220-193731.yaml create mode 100644 dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql create mode 100644 tests/functional/adapter/test_constraints.py diff --git a/.changes/unreleased/Features-20221220-193731.yaml b/.changes/unreleased/Features-20221220-193731.yaml new file mode 100644 index 000000000..dbd16ae0f --- /dev/null +++ b/.changes/unreleased/Features-20221220-193731.yaml @@ -0,0 +1,7 @@ +kind: Features +body: 'dbt-constraints support for BigQuery as per dbt-core issue #1358' +time: 2022-12-20T19:37:31.982821+01:00 +custom: + Author: victoriapm + Issue: "444" + PR: "426" diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index 07cf3c3e5..d80adc70c 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -52,8 +52,13 @@ {{ sql_header if sql_header is not none }} create or replace table {{ relation }} + {% if config.get('constraints_enabled', False) %} + {{ get_assert_columns_equivalent(sql) }} + {{ get_columns_spec_ddl() }} + {% endif %} {{ partition_by(partition_config) }} {{ cluster_by(raw_cluster_by) }} + {{ bigquery_table_options(config, model, temporary) }} as ( {{ compiled_code }} diff --git a/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql new file mode 100644 index 000000000..4f2720b7e --- /dev/null +++ b/dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql @@ -0,0 +1,16 @@ +{% macro bigquery__get_columns_spec_ddl() %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set ns = namespace(at_least_one_check=False) -%} + {%- set user_provided_columns = model['columns'] -%} + ( + {% for i in user_provided_columns -%} + {%- set col = user_provided_columns[i] -%} + {% set constraints = col['constraints'] -%} + {%- set ns.at_least_one_check = ns.at_least_one_check or col['constraints_check'] %} + {{ col['name'] }} {{ col['data_type'] }} {% for x in constraints %} {{ x or "" }} {% endfor %} {{ "," if not loop.last }} + {%- endfor %} + ) + {%- if ns.at_least_one_check -%} + {{exceptions.warn("We noticed you have `constraints_check` configs, these are NOT compatible with BigQuery and will be ignored")}} + {%- endif %} +{% endmacro %} diff --git a/dev-requirements.txt b/dev-requirements.txt index de146ae27..f489a04b3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,6 @@ # TODO: how to automate switching from develop to version branches? git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter - black~=22.8.0 bumpversion~=0.6.0 flake8 diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py new file mode 100644 index 000000000..b9a271b8c --- /dev/null +++ b/tests/functional/adapter/test_constraints.py @@ -0,0 +1,59 @@ +import pytest +from dbt.tests.util import relation_from_name +from dbt.tests.adapter.constraints.test_constraints import ( + BaseConstraintsColumnsEqual, + BaseConstraintsRuntimeEnforcement +) +from dbt.tests.adapter.constraints.fixtures import ( + my_model_sql, + my_model_wrong_order_sql, + my_model_wrong_name_sql, + model_schema_yml, +) + +_expected_sql_bigquery = """ +create or replace table {0} ( + id integer not null , + color string , + date_day date +) +OPTIONS() +as ( + select + 1 as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day +); +""" + +# Different on BigQuery: +# - does not support a data type named 'text' (TODO handle this via type translation/aliasing!) +# - raises an explicit error, if you try to set a primary key constraint, because it's not enforced +constraints_yml = model_schema_yml.replace("text", "string").replace("primary key", "") + +class TestBigQueryConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_wrong_order_sql, + "my_model_wrong_name.sql": my_model_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } + + +class TestBigQueryConstraintsRuntimeEnforcement(BaseConstraintsRuntimeEnforcement): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": constraints_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self, project): + relation = relation_from_name(project.adapter, "my_model") + return _expected_sql_bigquery.format(relation) + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["Required field id cannot be null"]