diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4a32060..4ab4d35 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -59,7 +59,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift linkedin apple_search_ads snapchat_ads facebook_ads - - label: ":bricks: Run Tests - Ads Packages :bookmark: :one:" + - label: ":databricks: Run Tests - Ads Packages :bookmark: :one:" key: "run_dbt_databricks_ads_1" plugins: - docker#v3.13.0: @@ -133,7 +133,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift tiktok_ads twitter_ads amazon_ads reddit_ads - - label: ":bricks: Run Tests - Ads Packages :bookmark: :two:" + - label: ":databricks: Run Tests - Ads Packages :bookmark: :two:" key: "run_dbt_databricks_ads_2" plugins: - docker#v3.13.0: @@ -210,7 +210,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift google_ads microsoft_ads pinterest - - label: ":bricks: Run Tests - Ads Packages :bookmark: :three:" + - label: ":databricks: Run Tests - Ads Packages :bookmark: :three:" key: "run_dbt_databricks_ads_3" plugins: - docker#v3.13.0: @@ -287,7 +287,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift linkedin_pages facebook_pages instagram_business twitter_organic youtube_analytics - - label: ":bricks: Run Tests - Social Packages :iphone:" + - label: ":databricks: Run Tests - Social Packages :iphone:" key: "run_dbt_databricks_social" plugins: - docker#v3.13.0: @@ -364,7 +364,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift google_play apple_store - - label: ":bricks: Run Tests - App Packages :video_game:" + - label: ":databricks: Run Tests - App Packages :video_game:" key: "run_dbt_databricks_app" plugins: - docker#v3.13.0: @@ -441,7 +441,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift salesforce hubspot - - label: ":bricks: Run Tests - CRM Packages :salesforce::hubspot:" + - label: ":databricks: Run Tests - CRM Packages :salesforce::hubspot:" key: "run_dbt_databricks_crm" plugins: - docker#v3.13.0: @@ -518,7 +518,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift shopify recurly stripe zuora - - label: ":bricks: Run Tests - Ecommerce and Subscription Packages :shopping_bags:" + - label: ":databricks: Run Tests - Ecommerce and Subscription Packages :shopping_bags:" key: "run_dbt_databricks_esub" plugins: - docker#v3.13.0: @@ -595,7 +595,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift marketo iterable klaviyo mailchimp pardot - - label: ":bricks: Run Tests - Email Marketing Packages :email:" + - label: ":databricks: Run Tests - Email Marketing Packages :email:" key: "run_dbt_databricks_email" plugins: - docker#v3.13.0: @@ -673,7 +673,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift quickbooks xero sage_intacct netsuite - - label: ":bricks: Run Tests - Finance Packages :money_with_wings:" + - label: ":databricks: Run Tests - Finance Packages :money_with_wings:" key: "run_dbt_databricks_finance" plugins: - docker#v3.13.0: @@ -751,7 +751,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift fivetran_log greenhouse lever - - label: ":bricks: Run Tests - Log and People Packages :people_hugging:" + - label: ":databricks: Run Tests - Log and People Packages :people_hugging:" key: "run_dbt_databricks_log_people" plugins: - docker#v3.13.0: @@ -829,7 +829,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift amplitude mixpanel pendo - - label: ":bricks: Run Tests - Product Packages :unicorn_face:" + - label: ":databricks: Run Tests - Product Packages :unicorn_face:" key: "run_dbt_databricks_product" plugins: - docker#v3.13.0: @@ -906,7 +906,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift intercom github zendesk asana jira - - label: ":bricks: Run Tests - Velocity Packages :roller_coaster:" + - label: ":databricks: Run Tests - Velocity Packages :roller_coaster:" key: "run_dbt_databricks_velocity" plugins: - docker#v3.13.0: @@ -984,7 +984,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift ad_reporting - - label: ":bricks: Run Tests - Ad Reporting Roll Up Package :roller_skate:" + - label: ":databricks: Run Tests - Ad Reporting Roll Up Package :roller_skate:" key: "run_dbt_databricks_ad_roll" plugins: - docker#v3.13.0: @@ -1061,7 +1061,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift app_reporting - - label: ":bricks: Run Tests - App Reporting Roll Up Package :roller_skate:" + - label: ":databricks: Run Tests - App Reporting Roll Up Package :roller_skate:" key: "run_dbt_databricks_app_roll" plugins: - docker#v3.13.0: @@ -1138,7 +1138,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift social_media_reporting - - label: ":bricks: Run Tests - Social Reporting Roll Up Package :roller_skate:" + - label: ":databricks: Run Tests - Social Reporting Roll Up Package :roller_skate:" key: "run_dbt_databricks_social_roll" plugins: - docker#v3.13.0: @@ -1215,7 +1215,7 @@ steps: commands: | bash .buildkite/scripts/run_standard_models.sh redshift shopify_holistic_reporting - - label: ":bricks: Run Tests - Shopify Holistic Reporting Roll Up Package :roller_skate:" + - label: ":databricks: Run Tests - Shopify Holistic Reporting Roll Up Package :roller_skate:" key: "run_dbt_databricks_shopify_holistic_roll" plugins: - docker#v3.13.0: diff --git a/.buildkite/scripts/run_standard_models.sh b/.buildkite/scripts/run_standard_models.sh index 3bce7b2..a80a71d 100644 --- a/.buildkite/scripts/run_standard_models.sh +++ b/.buildkite/scripts/run_standard_models.sh @@ -40,6 +40,9 @@ do perl -i -pe "s/(schema: |dataset: ).*/\1$value_to_replace/" ~/.dbt/profiles.yml elif [ "$package" = "social_media_reporting" ]; then perl -i -pe "s/(schema: |dataset: ).*/\1social_media_rollup_integration_tests/" ~/.dbt/profiles.yml + elif [ "$package" = "fivetran_log" ]; then + value_to_replace=$(grep "fivetran_platform_schema:" dbt_project.yml | awk '{ print $2 }') + perl -i -pe "s/(schema: |dataset: ).*/\1$value_to_replace/" ~/.dbt/profiles.yml else value_to_replace=$(grep ""$package"_schema:" dbt_project.yml | awk '{ print $2 }') perl -i -pe "s/(schema: |dataset: ).*/\1$value_to_replace/" ~/.dbt/profiles.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 5330c49..6f9e0f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# dbt_fivetran_utils v0.4.9 +[PR #130](https://github.com/fivetran/dbt_fivetran_utils/pull/130) includes the following updates. +## Feature Updates +- Added macro `extract_url_parameter` to create special logic for Databricks instances not supported by `dbt_utils.get_url_parameter()`. The macro uses `dbt_utils.get_url_parameter()` for default, non-Databricks targets. See [README](https://github.com/fivetran/dbt_fivetran_utils/blob/releases/v0.4.latest/README.md#extract_url_parameter-source) for more details. + # dbt_fivetran_utils v0.4.8 [PR #127](https://github.com/fivetran/dbt_fivetran_utils/pull/127) includes the following updates. ## Feature Updates diff --git a/README.md b/README.md index 923ce1d..cfa3c72 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ dispatch: - [Cross-database compatibility](#cross-database-compatibility) - [array\_agg (source)](#array_agg-source) - [ceiling (source)](#ceiling-source) + - [extract\_url\_parameter (source)](#extract_url_parameter-source) - [first\_value (source)](#first_value-source) - [json\_extract (source)](#json_extract-source) - [json\_parse (source)](#json_parse-source) @@ -171,6 +172,18 @@ than, or equal to, the specified numeric expression. The ceiling macro is compat **Args:** * `num` (required): The integer field you wish to apply the ceiling function. +---- +### extract_url_parameter ([source](macros/extract_url_parameter.sql)) +This macro extracts a url parameter from a column containing a url. It is an expansion of `dbt_utils.get_url_parameter()` to add support for Databricks SQL. + +**Usage:** +```sql +{{ fivetran_utils.extract_url_parameter(field="url_field", url_parameter="utm_source") }} +``` +**Args:** +* `field` (required): The name of the column containing the url. +* `url_parameter` (required): The parameter you want to extract. + ---- ### first_value ([source](macros/first_value.sql)) This macro returns the value_expression for the first row in the current window frame with cross db functionality. This macro ignores null values. The default first_value calculation within the macro is the `first_value` function. The Redshift first_value calculation is the `first_value` function, with the inclusion of a frame_clause `{{ partition_field }} rows unbounded preceding`. @@ -186,7 +199,6 @@ This macro returns the value_expression for the first row in the current window * `order` (optional): The order of which you want to partition the window frame. The order argument by default is `asc`. If you wish to get the last_value, you may change the argument to `desc`. ---- - ### json_extract ([source](macros/json_extract.sql)) This macro allows for cross database use of the json extract function. The json extract allows the return of data from a json object. The data is returned by the path you provide as the argument. The json_extract macro is compatible with BigQuery, Redshift, Postgres, and Snowflake. diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml index f15766b..1f95659 100644 --- a/integration_tests/packages.yml +++ b/integration_tests/packages.yml @@ -2,12 +2,8 @@ packages: - package: fivetran/ad_reporting version: [">=1.0.0", "<2.0.0"] - # - package: fivetran/shopify_holistic_reporting - # version: [">=0.1.0", "<1.0.0"] - # put this back when package versions updated in shopify_holistic_reporting - - git: https://github.com/fivetran/dbt_shopify_holistic_reporting.git - revision: test/update-shopify-versions - warn-unpinned: false + - package: fivetran/shopify_holistic_reporting + version: [">=0.1.0", "<1.0.0"] - package: fivetran/social_media_reporting version: [">=0.1.0", "<1.0.0"] @@ -19,7 +15,7 @@ packages: version: [">=0.1.0", "<1.0.0"] - package: fivetran/salesforce - version: [">=0.1.0", "<1.0.0"] + version: [">=1.0.0", "<2.0.0"] - package: fivetran/hubspot version: [">=0.1.0", "<1.0.0"] @@ -55,7 +51,7 @@ packages: version: [">=0.1.0", "<1.0.0"] - package: fivetran/fivetran_log - version: [">=0.1.0", "<1.0.0"] + version: [">=1.0.0", "<2.0.0"] - package: fivetran/lever version: [">=0.1.0", "<1.0.0"] - package: fivetran/greenhouse @@ -76,9 +72,5 @@ packages: version: [">=0.1.0", "<1.0.0"] - package: fivetran/zendesk version: [">=0.1.0", "<1.0.0"] - # - package: fivetran/intercom - # version: [">=0.1.0", "<1.0.0"] - # put this back when package versions updated in intercom_source - - git: https://github.com/fivetran/dbt_intercom.git - revision: test/calogica_version_update - warn-unpinned: false \ No newline at end of file + - package: fivetran/intercom + version: [">=0.1.0", "<1.0.0"] \ No newline at end of file diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index b7702d6..810bae1 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,6 +4,4 @@ dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 - -oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437 \ No newline at end of file +dbt-databricks>=1.6.0,<2.0.0 \ No newline at end of file diff --git a/macros/extract_url_parameter.sql b/macros/extract_url_parameter.sql new file mode 100644 index 0000000..4597803 --- /dev/null +++ b/macros/extract_url_parameter.sql @@ -0,0 +1,20 @@ +{% macro extract_url_parameter(field, url_parameter) -%} + +{{ adapter.dispatch('extract_url_parameter', 'fivetran_utils') (field, url_parameter) }} + +{% endmacro %} + + +{% macro default__extract_url_parameter(field, url_parameter) -%} + +{{ dbt_utils.get_url_parameter(field, url_parameter) }} + +{%- endmacro %} + + +{% macro spark__extract_url_parameter(field, url_parameter) -%} + +{%- set formatted_url_parameter = "'" + url_parameter + "=([^&]+)'" -%} +nullif(regexp_extract({{ field }}, {{ formatted_url_parameter }}, 1), '') + +{%- endmacro %} \ No newline at end of file