From ec666c9160dde7b09ac9c6c5e5d5f12e36e46a2a Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 21:28:30 +0100 Subject: [PATCH 01/18] Updating package dependendcies --- packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages.yml b/packages.yml index b4b5f5b..1b1203f 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - - package: fivetran/facebook_ads_creative_history - version: [">=0.2.0","<0.3.0"] \ No newline at end of file +- package: fivetran/facebook_ads_creative_history + version: [">=0.2.0", "<0.3.0"] From e94865c845dea195b881bcf40c074590fb5791ae Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 21:28:31 +0100 Subject: [PATCH 02/18] Updating require-dbt-version --- dbt_project.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index d86e6e4..305b7b4 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,17 +1,13 @@ - name: 'facebook_ads' -version: '0.2.2' +version: '0.3.0' config-version: 2 - -require-dbt-version: [">=0.18.0", "<0.20.0"] - +require-dbt-version: [">=0.20.0"] models: facebook_ads: +schema: facebook_ads +materialized: table intermediate: +materialized: view - vars: facebook_ads: account_history: "{{ ref('stg_facebook_ads__account_history') }}" From 4dfd4f4e3dfbcc29dc4a7d98aea61c410358cdd2 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 21:28:31 +0100 Subject: [PATCH 03/18] Updating dbt version in requirements.txt --- integration_tests/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index ca89302..b06ff52 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.19.1 -dbt-spark==0.19.1 -dbt-spark[PyHive]==0.19.1 \ No newline at end of file +dbt==0.20.0 +dbt-spark==0.20.0 +dbt-spark[PyHive]==0.20.0 From 071008eca2082f7eb2e51a3a8b44eefb255d395d Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 21:29:26 +0100 Subject: [PATCH 04/18] Update dbt_project.yml --- integration_tests/dbt_project.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2f2ffc4..6ad9db7 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -46,4 +46,8 @@ seeds: account_id: "{{ 'int64' if target.name == 'bigquery' else 'bigint' }}" models: - +materialized: table \ No newline at end of file + +materialized: table + +dispatch: + - macro_namespace: dbt_utils + search_order: ['spark_utils', 'dbt_utils'] From 42b6038367c32bef10067ac4b6820a4fecb154e5 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 21:30:45 +0100 Subject: [PATCH 05/18] Update packages.yml --- packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.yml b/packages.yml index 1b1203f..0c92a35 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - package: fivetran/facebook_ads_creative_history - version: [">=0.2.0", "<0.3.0"] + version: [">=0.3.0", "<0.4.0"] From f6dcf21030681dffefad1f97c1948b557f9c6f19 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Tue, 27 Jul 2021 23:09:52 -0500 Subject: [PATCH 06/18] dbt 20 readme and minor edits --- README.md | 18 ++++++++++++++++++ dbt_project.yml | 2 -- integration_tests/dbt_project.yml | 2 +- packages.yml | 4 ++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dbbf7fe..8e74121 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![dbt logo and version](https://img.shields.io/static/v1?logo=dbt&label=dbt-version&message=0.20.x&color=orange) # Facebook Ads This package models Facebook Ads data from [Fivetran's connector](https://fivetran.com/docs/applications/facebook-ads). It uses data in the format described by [this ERD](https://fivetran.com/docs/applications/facebook-ads#schemainformation). @@ -19,6 +20,14 @@ This package contains transformation models, designed to work simultaneously wit Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions, or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. +Include in your `packages.yml` + +```yaml +packages: + - package: fivetran/facebook_ads + version: [">=0.3.0", "<0.4.0"] +``` + ## Configuration By default, this package will look for your Facebook Ads data in the `facebook_ads` schema of your [target database](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile). If this is not where your Facebook Ads data is, please add the following configuration to your `dbt_project.yml` file: @@ -55,6 +64,15 @@ models: This package has been tested on BigQuery, Snowflake, Redshift, Postgres, and Databricks. +### Databricks Dispatch Configuration +dbt `v0.20.0` introduced a new project-level dispatch configuration that enables an "override" setting for all dispatched macros. If you are using a Databricks destination with this package you will need to add the below (or a variation of the below) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively. +```yml +# dbt_project.yml + +dispatch: + - macro_namespace: dbt_utils + se + ## Contributions Additional contributions to this package are very welcome! Please create issues or open PRs against `master`. Check out [this post](https://discourse.getdbt.com/t/contributing-to-a-dbt-package/657) on the best workflow for contributing to a package. diff --git a/dbt_project.yml b/dbt_project.yml index 305b7b4..0c906b3 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -17,5 +17,3 @@ vars: campaign_history: "{{ ref('stg_facebook_ads__campaign_history') }}" creative_history: "{{ ref('stg_facebook_ads__creative_history') }}" url_tag: "{{ ref('stg_facebook_ads__url_tag') }}" - dbt_utils_dispatch_list: ['spark_utils', 'fivetran_utils'] - fivetran_utils_dispatch_list: ['spark_utils'] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 6ad9db7..d0997ab 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'facebook_ads_integration_tests' -version: '0.2.2' +version: '0.3.0' profile: 'integration_tests' config-version: 2 diff --git a/packages.yml b/packages.yml index 0c92a35..31b1cee 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: -- package: fivetran/facebook_ads_creative_history - version: [">=0.3.0", "<0.4.0"] + - package: fivetran/facebook_ads_creative_history + version: [">=0.3.0", "<0.4.0"] From 178cc6f31b0c8a04bee63ba2b418c9838874f6d4 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Wed, 28 Jul 2021 08:49:34 -0500 Subject: [PATCH 07/18] readme edits --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e74121..a0dfe1e 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,8 @@ dbt `v0.20.0` introduced a new project-level dispatch configuration that enables dispatch: - macro_namespace: dbt_utils - se + search_order: ['spark_utils', 'dbt_utils'] +``` ## Contributions From 4b197beeab56624c5c500f5061f36408e88cb54e Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 3 Aug 2021 15:20:57 -0500 Subject: [PATCH 08/18] Update integration_tests/requirements.txt --- integration_tests/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index b06ff52..1b919b2 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.20.0 -dbt-spark==0.20.0 -dbt-spark[PyHive]==0.20.0 +dbt~=0.20.0 +dbt-spark~=0.20.0 +dbt-spark[PyHive]~=0.20.0 From 162aaac7e733c6f905484a4b567b3b2ed8b83565 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 3 Aug 2021 15:25:49 -0500 Subject: [PATCH 09/18] Update integration_tests/requirements.txt --- integration_tests/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 1b919b2..e22e926 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt~=0.20.0 -dbt-spark~=0.20.0 -dbt-spark[PyHive]~=0.20.0 +dbt==0.20.1 +dbt-spark==0.20.1 +dbt-spark[PyHive]==0.20.1 From 5a3dc58af054bfc1968d89979c4de1097f6aa237 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 3 Aug 2021 15:27:17 -0500 Subject: [PATCH 10/18] Update integration_tests/requirements.txt --- integration_tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index e22e926..ed6ec40 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.20.1 +dbt==0.20.0 dbt-spark==0.20.1 dbt-spark[PyHive]==0.20.1 From 44d5c582de6baa60a275d281a7b9b4d996b7e574 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 3 Aug 2021 15:29:06 -0500 Subject: [PATCH 11/18] Update integration_tests/requirements.txt --- integration_tests/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index ed6ec40..193d1c7 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,2 @@ dbt==0.20.0 -dbt-spark==0.20.1 dbt-spark[PyHive]==0.20.1 From 0fd1606b392b7bce2592b7433bc34460a0af4b74 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 3 Aug 2021 15:31:52 -0500 Subject: [PATCH 12/18] Update integration_tests/requirements.txt --- integration_tests/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 193d1c7..40b8fef 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,2 +1,3 @@ -dbt==0.20.0 +dbt==0.20.1rc1 +dbt-spark==0.20.1 dbt-spark[PyHive]==0.20.1 From 8e0c4c62ec8b8daa95f0c20c5d77dab0e91a48ec Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 3 Aug 2021 15:40:55 -0500 Subject: [PATCH 13/18] Update integration_tests/requirements.txt --- integration_tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 40b8fef..5186d56 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.20.1rc1 +dbt-core==0.20.1rc1 dbt-spark==0.20.1 dbt-spark[PyHive]==0.20.1 From 4f9ab169224ee39c652458a2b1877e037db66019 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 3 Aug 2021 17:06:54 -0400 Subject: [PATCH 14/18] Update requirements.txt --- integration_tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 5186d56..40b8fef 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt-core==0.20.1rc1 +dbt==0.20.1rc1 dbt-spark==0.20.1 dbt-spark[PyHive]==0.20.1 From 96bd2d6e15e7c6abc5f1dcbc5c204074ced2d76a Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 4 Aug 2021 16:08:52 -0500 Subject: [PATCH 15/18] Update integration_tests/requirements.txt --- integration_tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 40b8fef..1ba72fd 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.20.1rc1 +dbt==0.20.1-rc1 dbt-spark==0.20.1 dbt-spark[PyHive]==0.20.1 From 9d2c228872d540eaae10a11cceb91fb007b85e29 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 4 Aug 2021 16:10:22 -0500 Subject: [PATCH 16/18] Update integration_tests/requirements.txt --- integration_tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 1ba72fd..40b8fef 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.20.1-rc1 +dbt==0.20.1rc1 dbt-spark==0.20.1 dbt-spark[PyHive]==0.20.1 From f08a9ab653921763340acd3e4fa5dc45e021a5e3 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 4 Aug 2021 16:13:30 -0500 Subject: [PATCH 17/18] Update integration_tests/requirements.txt --- integration_tests/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 40b8fef..dc77891 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ dbt==0.20.1rc1 -dbt-spark==0.20.1 -dbt-spark[PyHive]==0.20.1 +dbt-spark==0.20.1rc1 +dbt-spark[PyHive]==0.20.1rc1 From f76d6b80dc2eb37465cd58b0c17b65f7292f52c5 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 4 Aug 2021 17:11:09 -0500 Subject: [PATCH 18/18] Update sample.profiles.yml --- integration_tests/ci/sample.profiles.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index bf1de23..0241770 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -38,7 +38,7 @@ integration_tests: spark: type: spark method: http - schema: facebook_ads_integration_tests + schema: facebook_ads_integration_tests_2 host: "{{ env_var('CI_SPARK_DBT_HOST') }}" organization: "{{ env_var('CI_SPARK_DBT_ORGANIZATION') }}" token: "{{ env_var('CI_SPARK_DBT_TOKEN') }}" @@ -57,4 +57,4 @@ integration_tests: schema: facebook_ads_integration_tests threads: 8 keepalives_idle: 0 - sslmode: prefer \ No newline at end of file + sslmode: prefer