Model-Level Constraints

.changes/unreleased/Features-20230406-104433.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add support for model-level constraints
+time: 2023-04-06T10:44:33.045896-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "569"

dbt/adapters/bigquery/impl.py

@@ -1,6 +1,8 @@
 from dataclasses import dataclass
 import threading
 from typing import Dict, List, Optional, Any, Set, Union, Type
+
+from dbt.contracts.graph.nodes import ColumnLevelConstraint, ModelLevelConstraint, ConstraintType
 from dbt.dataclass_schema import dbtClassMixin, ValidationError
 
 import dbt.deprecations
@@ -907,3 +909,27 @@ def python_submission_helpers(self) -> Dict[str, Type[PythonJobHelper]]:
             "cluster": ClusterDataprocHelper,
             "serverless": ServerlessDataProcHelper,
         }
+
+    @classmethod
+    def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> str:
+        if constraint.type == ConstraintType.not_null:
+            return super().render_column_constraint(constraint)
+        elif (
+            constraint.type == ConstraintType.primary_key
+            or constraint.type == ConstraintType.foreign_key
+        ):
+            c = super().render_column_constraint(constraint)
+            return f"{c} not enforced"
+        else:
+            return ""
+
+    @classmethod
+    def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[str]:
+        if (
+            constraint.type == ConstraintType.primary_key
+            or constraint.type == ConstraintType.foreign_key
+        ):
+            c = super().render_model_constraint(constraint)
+            return f"{c} not enforced" if c else None
+        else:
+            return None

dbt/include/bigquery/macros/adapters.sql

@@ -55,7 +55,7 @@
       {%- set contract_config = config.get('contract') -%}
       {%- if contract_config.enforced -%}
         {{ get_assert_columns_equivalent(compiled_code) }}
-        {{ get_columns_spec_ddl() }}
+        {{ get_table_columns_and_constraints() }}
         {%- set compiled_code = get_select_subquery(compiled_code) %}
       {% endif %}
     {{ partition_by(partition_config) }}

dbt/include/bigquery/macros/utils/get_columns_spec_ddl.sql

@@ -1,31 +1,3 @@
-{% macro bigquery__get_columns_spec_ddl() %}
-  {# loop through user_provided_columns to create DDL with data types and constraints #}
-    {%- set ns = namespace(at_least_one_check=False, at_least_one_pk=False) -%}
-    {%- set user_provided_columns = model['columns'] -%}
-    (
-    {% for i in user_provided_columns %}
-      {%- set col = user_provided_columns[i] -%}
-      {%- set constraints = col['constraints'] -%}
-      {{ col['name'] }} {{ col['data_type'] }}
-      {%- for c in constraints -%}
-        {%- if c.type == "check" -%}
-          {%- set ns.at_least_one_check = True -%}
-        {%- elif c.type == "primary_key" -%}
-          {%- set ns.at_least_one_pk = True -%}
-        {%- else %} {{ adapter.render_raw_column_constraint(c) }}
-        {%- endif -%}
-      {%- endfor -%}
-      {{ "," if not loop.last }}
-    {% endfor -%}
-    )
-  {%- if ns.at_least_one_check -%}
-      {{exceptions.warn("We noticed you have check constraints in your configs. These are not compatible with BigQuery and will be ignored.")}}
-  {%- endif -%}
-  {%- if ns.at_least_one_pk -%}
-    {{exceptions.warn("We noticed you have primary key constraints in your configs. These are not compatible with BigQuery and will be ignored.")}}
-  {%- endif -%}
-{% endmacro %}
-
 {% macro bigquery__format_column(column) -%}
   {% set data_type = column.data_type %}
   {% set formatted = column.column.lower() ~ " " ~ data_type %}

dev-requirements.txt

@@ -1,7 +1,7 @@
 # install latest changes in dbt-core
 # TODO: how to automate switching from develop to version branches?
-git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
-git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter
+git+https://github.com/dbt-labs/dbt-core.git@paw/ct-1922-model-level-constraints#egg=dbt-core&subdirectory=core
+git+https://github.com/dbt-labs/dbt-core.git@paw/ct-1922-model-level-constraints#egg=dbt-tests-adapter&subdirectory=tests/adapter
 
 # if version 1.x or greater -> pin to major version
 # if version 0.x -> pin to minor

tests/functional/adapter/test_constraints.py

@@ -7,7 +7,7 @@
     BaseConstraintsRuntimeDdlEnforcement,
     BaseConstraintsRollback,
     BaseIncrementalConstraintsRuntimeDdlEnforcement,
-    BaseIncrementalConstraintsRollback,
+    BaseIncrementalConstraintsRollback, BaseModelConstraintsRuntimeEnforcement,
 )
 from dbt.tests.adapter.constraints.fixtures import (
     my_model_sql,
@@ -18,12 +18,12 @@
     my_model_wrong_name_sql,
     my_model_view_wrong_name_sql,
     my_model_incremental_wrong_name_sql,
-    model_schema_yml,
+    model_schema_yml, constrained_model_schema_yml,
 )
 
 _expected_sql_bigquery = """
 create or replace table <model_identifier> (
-    id integer not null,
+    id integer not null primary key not enforced,
     color string,
     date_day string
 )
@@ -43,8 +43,8 @@
 # Different on BigQuery:
 # - does not support a data type named 'text' (TODO handle this via type translation/aliasing!)
 # - raises an explicit error, if you try to set a primary key constraint, because it's not enforced
-constraints_yml = model_schema_yml.replace("text", "string").replace("primary key", "")
-
+constraints_yml = model_schema_yml.replace("text", "string")
+model_constraints_yml = constrained_model_schema_yml.replace("text", "string")
 
 class BigQueryColumnEqualSetup:
     @pytest.fixture
@@ -167,4 +167,36 @@ def models(self):
 
     @pytest.fixture(scope="class")
     def expected_error_messages(self):
-        return ["Required field id cannot be null"]
+        return ["Required field id cannot be null"]
+
+
+class TestBigQueryModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement):
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "my_model.sql": my_incremental_model_sql,
+            "constraints_schema.yml": model_constraints_yml,
+        }
+
+    @pytest.fixture(scope="class")
+    def expected_sql(self):
+        return """
+create or replace table <model_identifier> (
+    id integer not null,
+    color string,
+    date_day string,
+    primary key (id) not enforced
+)
+OPTIONS()
+as (
+    select id,
+    color, 
+    date_day from 
+  ( 
+    select 1 as id, 
+    'blue' as color, 
+    '2019-01-01' as date_day
+  ) as model_subq
+);
+"""