Feature/databricks insert by period

insert_by_period/dbt_project.yml

@@ -11,7 +11,11 @@ config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 # This setting configures which "profile" dbt uses for this project.
-profile: 'insert_by_period'
+profile: 'dbt_labs_experimental_features__databricks'
+
+dispatch:
+ - macro_namespace: dbt_utils
+   search_order: ['spark_utils', 'dbt_utils']
 
 # These configurations specify where dbt should look for different types of files.
 # The `model-paths` config, for example, states that models in this project can be

insert_by_period/integration_tests/ci/sample.profiles.yml

@@ -47,3 +47,12 @@ integration_tests:
       warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
       schema: dbt_utils_integration_tests_snowflake
       threads: 10
+
+    databricks:
+      type: databricks
+      catalog: "{{ env_var('DATABRICKS_TEST_CATALOG') }}"
+      host: "{{ env_var('DATABRICKS_TEST_HOST') }}"
+      http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}"
+      token: "{{ env_var('DATABRICKS_TEST_TOKEN') }}"
+      schema: dbt_utils_integration_tests_databricks
+      threads: 4

insert_by_period/integration_tests/dbt_project.yml

@@ -2,12 +2,18 @@
 name: 'insert_by_period_integration_tests'
 version: '1.0'
 
-profile: 'integration_tests'
-
 # require-dbt-version: inherit this from dbt-utils
 
 config-version: 2
 
+# This setting configures which "profile" dbt uses for this project.
+profile: 'integration_tests'
+
+dispatch:
+ - macro_namespace: dbt_utils
+   search_order: ['spark_utils', 'dbt_utils', 'insert_by_period_integration_tests']
+
+
 model-paths: ["models"]
 analysis-paths: ["analysis"]
 test-paths: ["tests"]

insert_by_period/integration_tests/packages.yml

@@ -2,4 +2,6 @@
 packages:
     - local: ../
     - package: dbt-labs/dbt_utils
-      version: [">0.9.0", "<2.0.0"]
+      version: [">0.9.0", "<2.0.0"]
+    - package: dbt-labs/spark_utils
+      version: 0.3.0

insert_by_period/macros/create_relation_for_insert_by_period.sql

@@ -11,3 +11,8 @@
     {% do return (api.Relation.create(identifier=tmp_identifier,
                                                schema=None, type=type)) %}
 {%- endmacro %}
+
+{% macro databricks__create_relation_for_insert_by_period(tmp_identifier, schema, type) -%}
+    {% do return (api.Relation.create(identifier=tmp_identifier,
+                                               schema=None, type=type)) %}
+{%- endmacro %}

insert_by_period/macros/insert_by_period_materialization.sql

@@ -97,7 +97,12 @@
       );
     {%- endcall %}
     {% set result = load_result('main-' ~ i) %}
-    {% if 'response' in result.keys() %} {# added in v0.19.0 #}
+
+    {% if target.type == 'databricks' %} {# databricks provides rows affected in 'data' and not 'response' #}
+      {% if 'data' in result.keys() %}
+        {% set rows_inserted = result['data'][0][0] | int %}
+      {% endif %}
+    {% elif 'response' in result.keys() %} {# added in v0.19.0 #}
         {% set rows_inserted = result['response']['rows_affected'] %}
     {% else %} {# older versions #}
         {% set rows_inserted = result['status'].split(" ")[2] | int %}