Merge pull request #130 from fivetran/bug/databricks-get-url-parameter

Bug/databricks get url parameter

.buildkite/pipeline.yml

@@ -59,7 +59,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift linkedin apple_search_ads snapchat_ads facebook_ads
 
-  - label: ":bricks: Run Tests - Ads Packages :bookmark: :one:"
+  - label: ":databricks: Run Tests - Ads Packages :bookmark: :one:"
     key: "run_dbt_databricks_ads_1"
     plugins:
       - docker#v3.13.0:
@@ -133,7 +133,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift tiktok_ads twitter_ads amazon_ads reddit_ads
 
-  - label: ":bricks: Run Tests - Ads Packages :bookmark: :two:"
+  - label: ":databricks: Run Tests - Ads Packages :bookmark: :two:"
     key: "run_dbt_databricks_ads_2"
     plugins:
       - docker#v3.13.0:
@@ -210,7 +210,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift google_ads microsoft_ads pinterest
 
-  - label: ":bricks: Run Tests - Ads Packages :bookmark: :three:"
+  - label: ":databricks: Run Tests - Ads Packages :bookmark: :three:"
     key: "run_dbt_databricks_ads_3"
     plugins:
       - docker#v3.13.0:
@@ -287,7 +287,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift linkedin_pages facebook_pages instagram_business twitter_organic youtube_analytics
 
-  - label: ":bricks: Run Tests - Social Packages :iphone:"
+  - label: ":databricks: Run Tests - Social Packages :iphone:"
     key: "run_dbt_databricks_social"
     plugins:
       - docker#v3.13.0:
@@ -364,7 +364,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift google_play apple_store
 
-  - label: ":bricks: Run Tests - App Packages :video_game:"
+  - label: ":databricks: Run Tests - App Packages :video_game:"
     key: "run_dbt_databricks_app"
     plugins:
       - docker#v3.13.0:
@@ -441,7 +441,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift salesforce hubspot
 
-  - label: ":bricks: Run Tests - CRM Packages :salesforce::hubspot:"
+  - label: ":databricks: Run Tests - CRM Packages :salesforce::hubspot:"
     key: "run_dbt_databricks_crm"
     plugins:
       - docker#v3.13.0:
@@ -518,7 +518,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift shopify recurly stripe zuora
 
-  - label: ":bricks: Run Tests - Ecommerce and Subscription Packages :shopping_bags:"
+  - label: ":databricks: Run Tests - Ecommerce and Subscription Packages :shopping_bags:"
     key: "run_dbt_databricks_esub"
     plugins:
       - docker#v3.13.0:
@@ -595,7 +595,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift marketo iterable klaviyo mailchimp pardot
 
-  - label: ":bricks: Run Tests - Email Marketing Packages :email:"
+  - label: ":databricks: Run Tests - Email Marketing Packages :email:"
     key: "run_dbt_databricks_email"
     plugins:
       - docker#v3.13.0:
@@ -673,7 +673,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift quickbooks xero sage_intacct netsuite
 
-  - label: ":bricks: Run Tests - Finance Packages :money_with_wings:"
+  - label: ":databricks: Run Tests - Finance Packages :money_with_wings:"
     key: "run_dbt_databricks_finance"
     plugins:
       - docker#v3.13.0:
@@ -751,7 +751,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift fivetran_log greenhouse lever
 
-  - label: ":bricks: Run Tests - Log and People Packages :people_hugging:"
+  - label: ":databricks: Run Tests - Log and People Packages :people_hugging:"
     key: "run_dbt_databricks_log_people"
     plugins:
       - docker#v3.13.0:
@@ -829,7 +829,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift amplitude mixpanel pendo
 
-  - label: ":bricks: Run Tests - Product Packages :unicorn_face:"
+  - label: ":databricks: Run Tests - Product Packages :unicorn_face:"
     key: "run_dbt_databricks_product"
     plugins:
       - docker#v3.13.0:
@@ -906,7 +906,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift intercom github zendesk asana jira
 
-  - label: ":bricks: Run Tests - Velocity Packages :roller_coaster:"
+  - label: ":databricks: Run Tests - Velocity Packages :roller_coaster:"
     key: "run_dbt_databricks_velocity"
     plugins:
       - docker#v3.13.0:
@@ -984,7 +984,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift ad_reporting
 
-  - label: ":bricks: Run Tests - Ad Reporting Roll Up Package :roller_skate:"
+  - label: ":databricks: Run Tests - Ad Reporting Roll Up Package :roller_skate:"
     key: "run_dbt_databricks_ad_roll"
     plugins:
       - docker#v3.13.0:
@@ -1061,7 +1061,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift app_reporting
 
-  - label: ":bricks: Run Tests - App Reporting Roll Up Package :roller_skate:"
+  - label: ":databricks: Run Tests - App Reporting Roll Up Package :roller_skate:"
     key: "run_dbt_databricks_app_roll"
     plugins:
       - docker#v3.13.0:
@@ -1138,7 +1138,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift social_media_reporting
 
-  - label: ":bricks: Run Tests - Social Reporting Roll Up Package :roller_skate:"
+  - label: ":databricks: Run Tests - Social Reporting Roll Up Package :roller_skate:"
     key: "run_dbt_databricks_social_roll"
     plugins:
       - docker#v3.13.0:
@@ -1215,7 +1215,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_standard_models.sh redshift shopify_holistic_reporting
 
-  - label: ":bricks: Run Tests - Shopify Holistic Reporting Roll Up Package :roller_skate:"
+  - label: ":databricks: Run Tests - Shopify Holistic Reporting Roll Up Package :roller_skate:"
     key: "run_dbt_databricks_shopify_holistic_roll"
     plugins:
       - docker#v3.13.0:

.buildkite/scripts/run_standard_models.sh

@@ -40,6 +40,9 @@ do
         perl -i -pe "s/(schema: |dataset: ).*/\1$value_to_replace/" ~/.dbt/profiles.yml
     elif [ "$package" = "social_media_reporting" ]; then
         perl -i -pe "s/(schema: |dataset: ).*/\1social_media_rollup_integration_tests/" ~/.dbt/profiles.yml
+    elif [ "$package" = "fivetran_log" ]; then
+        value_to_replace=$(grep "fivetran_platform_schema:" dbt_project.yml | awk '{ print $2 }')
+        perl -i -pe "s/(schema: |dataset: ).*/\1$value_to_replace/" ~/.dbt/profiles.yml
     else
         value_to_replace=$(grep ""$package"_schema:" dbt_project.yml | awk '{ print $2 }')
         perl -i -pe "s/(schema: |dataset: ).*/\1$value_to_replace/" ~/.dbt/profiles.yml

CHANGELOG.md

@@ -1,3 +1,8 @@
+# dbt_fivetran_utils v0.4.9
+[PR #130](https://github.com/fivetran/dbt_fivetran_utils/pull/130) includes the following updates.
+## Feature Updates
+- Added macro `extract_url_parameter` to create special logic for Databricks instances not supported by `dbt_utils.get_url_parameter()`. The macro uses `dbt_utils.get_url_parameter()` for default, non-Databricks targets. See [README](https://github.com/fivetran/dbt_fivetran_utils/blob/releases/v0.4.latest/README.md#extract_url_parameter-source) for more details.
+
 # dbt_fivetran_utils v0.4.8
 [PR #127](https://github.com/fivetran/dbt_fivetran_utils/pull/127) includes the following updates.
 ## Feature Updates

README.md

@@ -58,6 +58,7 @@ dispatch:
   - [Cross-database compatibility](#cross-database-compatibility)
     - [array\_agg (source)](#array_agg-source)
     - [ceiling (source)](#ceiling-source)
+    - [extract\_url\_parameter (source)](#extract_url_parameter-source)
     - [first\_value (source)](#first_value-source)
     - [json\_extract (source)](#json_extract-source)
     - [json\_parse (source)](#json_parse-source)
@@ -171,6 +172,18 @@ than, or equal to, the specified numeric expression. The ceiling macro is compat
 **Args:**
 * `num` (required): The integer field you wish to apply the ceiling function.
 
+----
+### extract_url_parameter ([source](macros/extract_url_parameter.sql))
+This macro extracts a url parameter from a column containing a url. It is an expansion of `dbt_utils.get_url_parameter()` to add support for Databricks SQL. 
+
+**Usage:**
+```sql
+{{ fivetran_utils.extract_url_parameter(field="url_field", url_parameter="utm_source") }}
+```
+**Args:**
+* `field` (required): The name of the column containing the url.
+* `url_parameter` (required): The parameter you want to extract. 
+
 ----
 ### first_value ([source](macros/first_value.sql))
 This macro returns the value_expression for the first row in the current window frame with cross db functionality. This macro ignores null values. The default first_value calculation within the macro is the `first_value` function. The Redshift first_value calculation is the `first_value` function, with the inclusion of a frame_clause `{{ partition_field }} rows unbounded preceding`.
@@ -186,7 +199,6 @@ This macro returns the value_expression for the first row in the current window
 * `order`             (optional): The order of which you want to partition the window frame. The order argument by default is `asc`. If you wish to get the last_value, you may change the argument to `desc`.
 
 ----
-
 ### json_extract ([source](macros/json_extract.sql))
 This macro allows for cross database use of the json extract function. The json extract allows the return of data from a json object.
 The data is returned by the path you provide as the argument. The json_extract macro is compatible with BigQuery, Redshift, Postgres, and Snowflake.

integration_tests/packages.yml

@@ -2,12 +2,8 @@ packages:
   - package: fivetran/ad_reporting
     version: [">=1.0.0", "<2.0.0"]
 
-  # - package: fivetran/shopify_holistic_reporting
-  #   version: [">=0.1.0", "<1.0.0"]
-  # put this back when package versions updated in shopify_holistic_reporting
-  - git: https://github.com/fivetran/dbt_shopify_holistic_reporting.git
-    revision: test/update-shopify-versions
-    warn-unpinned: false
+  - package: fivetran/shopify_holistic_reporting
+    version: [">=0.1.0", "<1.0.0"]
 
   - package: fivetran/social_media_reporting
     version: [">=0.1.0", "<1.0.0"]
@@ -19,7 +15,7 @@ packages:
     version: [">=0.1.0", "<1.0.0"]
 
   - package: fivetran/salesforce
-    version: [">=0.1.0", "<1.0.0"]
+    version: [">=1.0.0", "<2.0.0"]
   - package: fivetran/hubspot
     version: [">=0.1.0", "<1.0.0"]
 
@@ -55,7 +51,7 @@ packages:
     version: [">=0.1.0", "<1.0.0"]
 
   - package: fivetran/fivetran_log
-    version: [">=0.1.0", "<1.0.0"]
+    version: [">=1.0.0", "<2.0.0"]
   - package: fivetran/lever
     version: [">=0.1.0", "<1.0.0"]
   - package: fivetran/greenhouse
@@ -76,9 +72,5 @@ packages:
     version: [">=0.1.0", "<1.0.0"]
   - package: fivetran/zendesk
     version: [">=0.1.0", "<1.0.0"]
-  # - package: fivetran/intercom
-  #   version: [">=0.1.0", "<1.0.0"]
-    # put this back when package versions updated in intercom_source
-  - git: https://github.com/fivetran/dbt_intercom.git
-    revision: test/calogica_version_update
-    warn-unpinned: false
+  - package: fivetran/intercom
+    version: [">=0.1.0", "<1.0.0"]

integration_tests/requirements.txt

@@ -4,6 +4,4 @@ dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
-
-oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437
+dbt-databricks>=1.6.0,<2.0.0

macros/extract_url_parameter.sql

@@ -0,0 +1,20 @@
+{% macro extract_url_parameter(field, url_parameter) -%}
+
+{{ adapter.dispatch('extract_url_parameter', 'fivetran_utils') (field, url_parameter) }}
+
+{% endmacro %}
+
+
+{% macro default__extract_url_parameter(field, url_parameter) -%}
+
+{{ dbt_utils.get_url_parameter(field, url_parameter) }}
+
+{%- endmacro %}
+
+
+{% macro spark__extract_url_parameter(field, url_parameter) -%}
+
+{%- set formatted_url_parameter = "'" + url_parameter + "=([^&]+)'" -%}
+nullif(regexp_extract({{ field }}, {{ formatted_url_parameter }}, 1), '')
+
+{%- endmacro %}