Feature/enable null urls

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
 
-  - label: ":bricks: Run Tests - Databricks"
+  - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:

.buildkite/scripts/run_models.sh

@@ -18,4 +18,6 @@ cd integration_tests
 dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
-dbt test --target "$db"
+dbt test --target "$db"
+dbt run --vars '{ad_reporting__url_report__using_null_filter: false}' --target "$db" --full-refresh
+dbt test --vars '{ad_reporting__url_report__using_null_filter: false}' --target "$db"

CHANGELOG.md

@@ -37,6 +37,12 @@
     - `dbt.current_timestamp_in_utc_backcompat`
 - Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
 
+## 🎉 Features 🎉
+- For use in the [dbt_ad_reporting package](https://github.com/fivetran/dbt_ad_reporting), users can now allow records having nulls in url fields to be included in the `ad_reporting__url_report` model. See the [dbt_ad_reporting README](https://github.com/fivetran/dbt_ad_reporting) for more details ([#23](https://github.com/fivetran/dbt_pinterest/pull/23)).
+## 🚘 Under the Hood 🚘
+- Disabled the `not_null` test for `pinterest_ads__url_report` when null urls are allowed ([#23](https://github.com/fivetran/dbt_pinterest/pull/23)).
+
+
 # dbt_pinterest v0.6.0
 PR [#21](https://github.com/fivetran/dbt_pinterest/pull/21) includes the following changes:
 ## 🚨 Breaking Changes 🚨

README.md

@@ -66,6 +66,16 @@ vars:
 ## (Optional) Step 4: Additional configurations
 <details><summary>Expand for configurations</summary>
 
+## Enable records to have null urls
+The default behavior for the `pinterest_ads__url_report` end model is to filter out records having null URL fields, however you are able to turn off this filter if necessary. This is done by setting one of the variables below to `True` in your `dbt_project.yml` file. 
+> Note that using the variable `allow_ad_reporting_null_urls` will allow records with null urls for ALL Fivetran ad packages included in your project.
+
+```yml
+vars:
+  allow_pinterest_ads_null_urls: True # Use this variable to allow null urls for dbt_pinterest_ads only. Default is False. 
+  allow_ad_reporting_null_urls: True # Use this variable to allow null urls for ALL Fivetran ad packages included in your project. Default is False. 
+```
+
 ### Passing Through Additional Metrics
 By default, this package will select `clicks`, `impressions`, and `cost` from the source reporting tables to store into the staging models. If you would like to pass through additional metrics to the staging models, add the below configurations to your `dbt_project.yml` file. These variables allow for the pass-through fields to be aliased (`alias`) if desired, but not required. Use the below format for declaring the respective pass-through variables:
 

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: pinterest_integration_tests
+      schema: pinterest_integration_tests_2
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: pinterest_integration_tests
+      schema: pinterest_integration_tests_2
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: pinterest_integration_tests
+      schema: pinterest_integration_tests_2
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: pinterest_integration_tests
+      schema: pinterest_integration_tests_2
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: pinterest_integration_tests
+      schema: pinterest_integration_tests_2
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -4,7 +4,7 @@ profile: 'integration_tests'
 config-version: 2
 
 vars:
-  pinterest_schema: pinterest_integration_tests
+  pinterest_schema: pinterest_integration_tests_2
   pinterest_ad_group_history_identifier: "ad_group_history_data"
   pinterest_campaign_history_identifier: "campaign_history_data"
   pinterest_pin_promotion_history_identifier: "pin_promotion_history_data"

models/pinterest.yml

@@ -11,6 +11,9 @@ models:
             - campaign_id
             - advertiser_id
             - date_day
+          config:
+            # Below is so we don't run this test if the user has allowed urls to have null entries
+            enabled: "{{ var('ad_reporting__url_report__using_null_filter', true) }}"
     columns:
       - name: date_day
         description: "{{ doc('date_day') }}"
@@ -26,6 +29,11 @@ models:
         description: Status of the ad group.
       - name: destination_url
         description: Pin destination URL.
+        tests:
+          - not_null:
+              config:
+                # Below is so we don't run this test if the user has allowed urls to have null entries
+                enabled: "{{ var('ad_reporting__url_report__using_null_filter', true) }}"
       - name: pin_promotion_id
         description: "{{ doc('pin_promotion_id') }}"
         tests:

models/pinterest_ads__url_report.sql

@@ -75,8 +75,10 @@ joined as (
     left join advertisers
         on campaigns.advertiser_id = advertisers.advertiser_id
 
-    -- We only want utm ads to populate this report. Therefore, we filter where url pins are populated.
+    {% if var('ad_reporting__url_report__using_null_filter', True) %}
     where pins.destination_url is not null
+    {% endif %}
+
     {{ dbt_utils.group_by(22) }}
 )