Merge branch 'main' into pa/format-consistency

brooklyn-data · Aug 21, 2024 · 497bb01 · 497bb01
2 parents bf46483 + 3afcfbd
commit 497bb01
Show file tree

Hide file tree

Showing 15 changed files with 215 additions and 221 deletions.
diff --git a/.github/workflows/ci_lint_package.yml b/.github/workflows/ci_lint_package.yml
@@ -50,7 +50,7 @@ jobs:
           architecture: "x64"
 
       - name: Install Python packages
-        run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.3.2
+        run: python -m pip install dbt-snowflake~=1.8.0 sqlfluff-templater-dbt~=3.0.0
 
       - name: Test database connection
         run: dbt debug

diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml
@@ -21,7 +21,7 @@ env:
   DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
   DBT_ENV_SECRET_GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
   # Env var to test version
-  LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_6_0 # A dbt version supported by both the last release and this one
+  LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_8_0 # A dbt version supported by both the last release and this one
   # Env vars to test invocations model
   DBT_CLOUD_PROJECT_ID: 123
   DBT_CLOUD_JOB_ID: ABC
@@ -115,7 +115,7 @@ jobs:
       matrix:
         warehouse: ["snowflake", "bigquery", "postgres"]
         # When supporting a new version, update the list here
-        version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"]
+        version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"]
     runs-on: ubuntu-latest
     environment:
       name: Approve Integration Tests

diff --git a/.github/workflows/main_lint_package.yml b/.github/workflows/main_lint_package.yml
@@ -46,7 +46,7 @@ jobs:
           architecture: "x64"
 
       - name: Install Python packages
-        run: python -m pip install dbt-snowflake~=1.6.0 sqlfluff-templater-dbt~=2.3.2
+        run: python -m pip install dbt-snowflake~=1.8.0 sqlfluff-templater-dbt~=3.0.0
 
       - name: Test database connection
         run: dbt debug

diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml
@@ -35,7 +35,7 @@ jobs:
     strategy:
       matrix:
         warehouse: ["snowflake", "bigquery", "postgres"]
-        version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"]
+        version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"]
     runs-on: ubuntu-latest
     permissions:
       contents: "read"

diff --git a/.github/workflows/publish_docs_on_release.yml b/.github/workflows/publish_docs_on_release.yml
@@ -39,7 +39,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install Python packages
-        run: python -m pip install dbt-snowflake~=1.6.0
+        run: python -m pip install dbt-snowflake~=1.8.0
 
       - name: Test database connection
         run: dbt debug

diff --git a/README.md b/README.md
@@ -46,7 +46,7 @@ See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/
 ```
 packages:
   - package: brooklyn-data/dbt_artifacts
-    version: 2.6.0
+    version: 2.6.4
 ```
 
 :construction_worker: Make sure to fix at least the **minor** version, to avoid issues when a new release is open. See the notes on upgrading below for more detail.

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,7 +1,7 @@
 name: "dbt_artifacts"
-version: "2.6.0"
+version: "2.6.4"
 config-version: 2
-require-dbt-version: [">=1.3.0", "<1.7.0"]
+require-dbt-version: [">=1.3.0", "<1.9.0"]
 profile: "dbt_artifacts"
 
 clean-targets: # folders to be removed by `dbt clean`

diff --git a/integration_test_project/example-env.sh b/integration_test_project/example-env.sh
@@ -18,6 +18,7 @@ export DBT_ENV_SPARK_DRIVER_PATH= # /Library/simba/spark/lib/libsparkodbc_sbu.dy
 export DBT_ENV_SPARK_ENDPOINT= # The endpoint ID from the Databricks HTTP path
 
 # dbt environment variables, change these
+export DBT_VERSION="1_5_0"
 export DBT_CLOUD_PROJECT_ID=
 export DBT_CLOUD_JOB_ID=
 export DBT_CLOUD_RUN_ID=

diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql
@@ -24,26 +24,27 @@
             {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(15)) }}
         from values
         {% for model in models -%}
-                {% do model.pop('raw_code', None) %}
+                {% set model_copy = model.copy() -%}
+                {% do model_copy.pop('raw_code', None) %}
             (
                 '{{ invocation_id }}', {# command_invocation_id #}
-                '{{ model.unique_id }}', {# node_id #}
+                '{{ model_copy.unique_id }}', {# node_id #}
                 '{{ run_started_at }}', {# run_started_at #}
-                '{{ model.database }}', {# database #}
-                '{{ model.schema }}', {# schema #}
-                '{{ model.name }}', {# name #}
-                '{{ tojson(model.depends_on.nodes) | replace('\\', '\\\\') }}', {# depends_on_nodes #}
-                '{{ model.package_name }}', {# package_name #}
-                '{{ model.original_file_path | replace('\\', '\\\\') }}', {# path #}
-                '{{ model.checksum.checksum  | replace('\\', '\\\\') }}', {# checksum #}
-                '{{ model.config.materialized }}', {# materialization #}
-                '{{ tojson(model.tags) }}', {# tags #}
-                '{{ tojson(model.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #}
-                '{{ model.alias }}', {# alias #}
+                '{{ model_copy.database }}', {# database #}
+                '{{ model_copy.schema }}', {# schema #}
+                '{{ model_copy.name }}', {# name #}
+                '{{ tojson(model_copy.depends_on.nodes) | replace('\\', '\\\\') }}', {# depends_on_nodes #}
+                '{{ model_copy.package_name }}', {# package_name #}
+                '{{ model_copy.original_file_path | replace('\\', '\\\\') }}', {# path #}
+                '{{ model_copy.checksum.checksum  | replace('\\', '\\\\') }}', {# checksum #}
+                '{{ model_copy.config.materialized }}', {# materialization #}
+                '{{ tojson(model_copy.tags) }}', {# tags #}
+                '{{ tojson(model_copy.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #}
+                '{{ model_copy.alias }}', {# alias #}
                 {% if var('dbt_artifacts_exclude_all_results', false) %}
                     null
                 {% else %}
-                    '{{ tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #}
+                    '{{ tojson(model_copy) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #}
                 {% endif %}
             )
             {%- if not loop.last %},{%- endif %}
@@ -59,26 +60,27 @@
     {% if models != [] %}
         {% set model_values %}
             {% for model in models -%}
-                {% do model.pop('raw_code', None) %}
+                {% set model_copy = model.copy() -%}
+                {% do model_copy.pop('raw_code', None) %}
                 (
                     '{{ invocation_id }}', {# command_invocation_id #}
-                    '{{ model.unique_id }}', {# node_id #}
+                    '{{ model_copy.unique_id }}', {# node_id #}
                     '{{ run_started_at }}', {# run_started_at #}
-                    '{{ model.database }}', {# database #}
-                    '{{ model.schema }}', {# schema #}
-                    '{{ model.name }}', {# name #}
-                    {{ tojson(model.depends_on.nodes) }}, {# depends_on_nodes #}
-                    '{{ model.package_name }}', {# package_name #}
-                    '{{ model.original_file_path | replace('\\', '\\\\') }}', {# path #}
-                    '{{ model.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #}
-                    '{{ model.config.materialized }}', {# materialization #}
-                    {{ tojson(model.tags) }}, {# tags #}
-                    {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.config.meta)) }}, {# meta #}
-                    '{{ model.alias }}', {# alias #}
+                    '{{ model_copy.database }}', {# database #}
+                    '{{ model_copy.schema }}', {# schema #}
+                    '{{ model_copy.name }}', {# name #}
+                    {{ tojson(model_copy.depends_on.nodes) }}, {# depends_on_nodes #}
+                    '{{ model_copy.package_name }}', {# package_name #}
+                    '{{ model_copy.original_file_path | replace('\\', '\\\\') }}', {# path #}
+                    '{{ model_copy.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #}
+                    '{{ model_copy.config.materialized }}', {# materialization #}
+                    {{ tojson(model_copy.tags) }}, {# tags #}
+                    {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model_copy.config.meta)) }}, {# meta #}
+                    '{{ model_copy.alias }}', {# alias #}
                     {% if var('dbt_artifacts_exclude_all_results', false) %}
                         null
                     {% else %}
-                        {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #}
+                        {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model_copy) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #}
                     {% endif %}
                 )
                 {%- if not loop.last %},{%- endif %}
@@ -94,26 +96,27 @@
     {% if models != [] %}
         {% set model_values %}
             {% for model in models -%}
-                {% do model.pop('raw_code', None) %}
+                {% set model_copy = model.copy() -%}
+                {% do model_copy.pop('raw_code', None) %}
                 (
                     '{{ invocation_id }}', {# command_invocation_id #}
-                    '{{ model.unique_id }}', {# node_id #}
+                    '{{ model_copy.unique_id }}', {# node_id #}
                     '{{ run_started_at }}', {# run_started_at #}
-                    '{{ model.database }}', {# database #}
-                    '{{ model.schema }}', {# schema #}
-                    '{{ model.name }}', {# name #}
-                    '{{ tojson(model.depends_on.nodes) }}', {# depends_on_nodes #}
-                    '{{ model.package_name }}', {# package_name #}
-                    $${{ model.original_file_path | replace('\\', '\\\\') }}$$, {# path #}
-                    '{{ model.checksum.checksum }}', {# checksum #}
-                    '{{ model.config.materialized }}', {# materialization #}
-                    '{{ tojson(model.tags) }}', {# tags #}
-                    $${{ model.config.meta }}$$, {# meta #}
-                    '{{ model.alias }}', {# alias #}
+                    '{{ model_copy.database }}', {# database #}
+                    '{{ model_copy.schema }}', {# schema #}
+                    '{{ model_copy.name }}', {# name #}
+                    '{{ tojson(model_copy.depends_on.nodes) }}', {# depends_on_nodes #}
+                    '{{ model_copy.package_name }}', {# package_name #}
+                    $${{ model_copy.original_file_path | replace('\\', '\\\\') }}$$, {# path #}
+                    '{{ model_copy.checksum.checksum }}', {# checksum #}
+                    '{{ model_copy.config.materialized }}', {# materialization #}
+                    '{{ tojson(model_copy.tags) }}', {# tags #}
+                    $${{ model_copy.config.meta }}$$, {# meta #}
+                    '{{ model_copy.alias }}', {# alias #}
                     {% if var('dbt_artifacts_exclude_all_results', false) %}
                         null
                     {% else %}
-                        $${{ tojson(model) }}$$ {# all_results #}
+                        $${{ tojson(model_copy) }}$$ {# all_results #}
                     {% endif %}
                 )
                 {%- if not loop.last %},{%- endif %}

diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql
@@ -38,26 +38,10 @@
                 '{{ model.thread_id }}', {# thread_id #}
                 '{{ model.status }}', {# status #}
 
-                {% if model.timing != [] %}
-                    {% for stage in model.timing if stage.name == "compile" %}
-                        {% if loop.length == 0 %}
-                            null, {# compile_started_at #}
-                        {% else %}
-                            '{{ stage.started_at }}', {# compile_started_at #}
-                        {% endif %}
-                    {% endfor %}
-
-                    {% for stage in model.timing if stage.name == "execute" %}
-                        {% if loop.length == 0 %}
-                            null, {# query_completed_at #}
-                        {% else %}
-                            '{{ stage.completed_at }}', {# query_completed_at #}
-                        {% endif %}
-                    {% endfor %}
-                {% else %}
-                    null, {# compile_started_at #}
-                    null, {# query_completed_at #}
-                {% endif %}
+                {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
+                {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
+                {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
+                {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}
 
                 {{ model.execution_time }}, {# total_node_runtime #}
                 null, -- rows_affected not available {# Only available in Snowflake #}
@@ -95,26 +79,10 @@
                 '{{ model.thread_id }}', {# thread_id #}
                 '{{ model.status }}', {# status #}
 
-                {% if model.timing != [] %}
-                    {% for stage in model.timing if stage.name == "compile" %}
-                        {% if loop.length == 0 %}
-                            null, {# compile_started_at #}
-                        {% else %}
-                            '{{ stage.started_at }}', {# compile_started_at #}
-                        {% endif %}
-                    {% endfor %}
-
-                    {% for stage in model.timing if stage.name == "execute" %}
-                        {% if loop.length == 0 %}
-                            null, {# query_completed_at #}
-                        {% else %}
-                            '{{ stage.completed_at }}', {# query_completed_at #}
-                        {% endif %}
-                    {% endfor %}
-                {% else %}
-                    null, {# compile_started_at #}
-                    null, {# query_completed_at #}
-                {% endif %}
+                {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
+                {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
+                {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
+                {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}
 
                 {{ model.execution_time }}, {# total_node_runtime #}
                 null, -- rows_affected not available {# Databricks #}
@@ -170,26 +138,10 @@
                 '{{ model.thread_id }}', {# thread_id #}
                 '{{ model.status }}', {# status #}
 
-                {% if model.timing != [] %}
-                    {% for stage in model.timing if stage.name == "compile" %}
-                        {% if loop.length == 0 %}
-                            null, {# compile_started_at #}
-                        {% else %}
-                            '{{ stage.started_at }}', {# compile_started_at #}
-                        {% endif %}
-                    {% endfor %}
-
-                    {% for stage in model.timing if stage.name == "execute" %}
-                        {% if loop.length == 0 %}
-                            null, {# query_completed_at #}
-                        {% else %}
-                            '{{ stage.completed_at }}', {# query_completed_at #}
-                        {% endif %}
-                    {% endfor %}
-                {% else %}
-                    null, {# compile_started_at #}
-                    null, {# query_completed_at #}
-                {% endif %}
+                {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
+                {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
+                {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
+                {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}
 
                 {{ model.execution_time }}, {# total_node_runtime #}
                 try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #}