Merge pull request #38 from fivetran/release/v0.6.0

version upgrade

CHANGELOG.md

@@ -1,12 +1,18 @@
+# dbt_ad_reporting v0.6.0
+TikTok Ads has been added as a dependency and is enabled by default. Be sure to disable the models via the README if you do not have a TikTok Ads connector.
+
+## Feature Additions
+- Added the TikTok Ads package to be included in the final `ad_reporting` model. ([#36](https://github.com/fivetran/dbt_ad_reporting/pull/36))
+
 # dbt_ad_reporting v0.6.0-b1
 🎉 dbt v1.0.0 Compatibility Pre Release 🎉 An official dbt v1.0.0 compatible version of the package will be released once existing feature/bug PRs are merged.
+
 ## 🚨 Breaking Changes 🚨
 - Adjusts the `require-dbt-version` to now be within the range [">=1.0.0", "<2.0.0"]. Additionally, the package has been updated for dbt v1.0.0 compatibility. If you are using a dbt version <1.0.0, you will need to upgrade in order to leverage the latest version of the package.
   - For help upgrading your package, I recommend reviewing this GitHub repo's Release Notes on what changes have been implemented since your last upgrade.
   - For help upgrading your dbt project to dbt v1.0.0, I recommend reviewing dbt-labs [upgrading to 1.0.0 docs](https://docs.getdbt.com/docs/guides/migration-guide/upgrading-to-1-0-0) for more details on what changes must be made.
 - Upgrades the package dependencies to refer to the latest individual ad package versions. Additionally, the latest individual ad package versions have a dependency on the latest `dbt_fivetran_utils`. Further, the latest `dbt_fivetran_utils` package also has a dependency on `dbt_utils` [">=0.8.0", "<0.9.0"].
   - Please note, if you are installing a version of `dbt_utils` in your `packages.yml` that is not in the range above then you will encounter a package dependency error.
-
 # dbt_ad_reporting v0.5.1
 ## Feature Additions
 - Added the `account_name` and `account_id` to the `stg_snapchat_ads` model. This will allow for the relevant Snapchat Ads account information to flow downstream into the final `ad_reporting` model. ([#30](https://github.com/fivetran/dbt_ad_reporting/pull/30))

README.md

@@ -12,6 +12,7 @@ Currently, this package supports the following ad connector types:
 * [Pinterest Ads](https://github.com/fivetran/dbt_pinterest)
 * [Twitter Ads](https://github.com/fivetran/dbt_twitter)
 * [Snapchat Ads](https://github.com/fivetran/dbt_snapchat_ads)
+* [TikTok Ads](https://github.com/fivetran/dbt_tiktok_ads)
 
 ## Models
 
@@ -29,7 +30,8 @@ Include in your `packages.yml`
 ```yaml
 packages:
   - package: fivetran/ad_reporting
-    version: 0.6.0-b1
+    version: [">=0.6.0", "<0.7.0"]
+
 ```
 
 ## Configuration
@@ -52,6 +54,7 @@ vars:
   ad_reporting__twitter_ads_enabled: False
   ad_reporting__facebook_ads_enabled: False
   ad_reporting__snapchat_ads_enabled: False
+  ad_reporting__tiktok_ads_enabled: False
 ```
 
 Next, you must disable the models in the unwanted connector's related package, which has its own configuration. Disable the relevant models under the models section of your `dbt_project.yml` file by setting the `enabled` value to `false`. 
@@ -103,6 +106,12 @@ models:
     enabled: false
   snapchat_ads_source:
     enabled: false
+  
+  # disable both tiktok ads models if not using tiktok ads
+  tiktok_ads:
+    enabled: false
+  tiktok_ads_source:
+    enabled: false
 ```
 ### Google Ads and Adwords API Configuration
 This package allows users to leverage either the Adwords API or the Google Ads API if you have enabled the Google Ads connector. You will be able to determine which API your Google Ads connector is using by navigating within your Fivetran UI to the `setup` tab -> `edit connection details` link -> and reference the `API configuration` used. You will want to refer to the respective configuration steps below based off the API used by your connector. 
@@ -149,6 +158,8 @@ vars:
     google_ads_database: your_database_name 
     snapchat_schema: snapchat_ads
     snapchat_database: your_database_name 
+    tiktok_ads_schema: tiktok_ads
+    tiktok_ads_database: your_database_name 
 ```
 
 For more configuration information, see the relevant connectors ([listed above](https://github.com/fivetran/dbt_ad_reporting/edit/master/README.md#adreporting)).
@@ -197,6 +208,11 @@ models:
     +schema: snapchat_ads
   snapchat_ads_source:
     +schema: snapchat_ads_source
+  
+  tiktok_ads:
+    +schema: tiktok_ads
+  tiktok_ads_source:
+    +schema: tiktok_ads_source
 
 ```
 

dbt_project.yml

@@ -12,6 +12,7 @@ vars:
   ad_reporting__twitter_ads_enabled: true
   ad_reporting__facebook_ads_enabled: true
   ad_reporting__snapchat_ads_enabled: true
+  ad_reporting__tiktok_ads_enabled: true
 
 models:
   ad_reporting:
@@ -52,4 +53,9 @@ models:
   snapchat_ads:
     enabled: true
   snapchat_ads_source:
-    enabled: true
+    enabled: true
+
+  tiktok_ads:
+    enabled: true
+  tiktok_ads_source:
+    enabled: true

integration_tests/dbt_project.yml

@@ -64,8 +64,18 @@ vars:
     creative_history: "{{ ref('snapchat_creative_history_data') }}"
     creative_url_tag_history: "{{ ref('snapchat_creative_url_tag_history_data') }}"
 
+  tiktok_ads_source:
+    ad_history: "{{ ref('tiktok_ad_history_data') }}"
+    ad_report_hourly: "{{ ref('tiktok_ad_report_hourly_data') }}"
+    ad_group_history: "{{ ref('tiktok_adgroup_history_data') }}"
+    ad_group_hourly: "{{ ref('tiktok_adgroup_report_hourly_data') }}"
+    campaign_history: "{{ ref('tiktok_campaign_history_data') }}"
+    campaign_report_hourly: "{{ ref('tiktok_campaign_report_hourly_data') }}"
+    advertiser: "{{ ref('tiktok_advertiser_data') }}"
+
 seeds:
   ad_reporting_integration_tests:
+    +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}"
 
     ### Microsoft Seeds
 

integration_tests/seeds/tiktok_ad_history_data.csv

@@ -0,0 +1,5 @@
+ad_id,updated_at,advertiser_id,adgroup_id,campaign_id,create_time,ad_name,call_to_action,status,opt_status,ad_text,video_id,app_name,open_url,landing_page_url,display_name,profile_image,impression_tracking_url,click_tracking_url,playable_url,is_aco,is_creative_authorized,is_new_structure,image_ids,_fivetran_synced
+1,2021-09-15 04:26:28.000 +0000,1,1,1,2021-08-27 14:13:57.000 +0000,ad name one,SIGN_UP,AD_STATUS_DELIVERY_OK,DISABLE,subscribe,v10033g50000c4kf54jc77ucvou6fv0g,,,https://www.stuffthatworks.health/,health,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:16.797 +0000
+2,2021-09-15 03:19:25.000 +0000,2,2,2,2021-09-13 18:54:11.000 +0000,ad name two,SIGN_UP,AD_STATUS_DELIVERY_OK,ENABLE,join us,v10033g50000c4vnvmbc77u58bhlklvg,,,https://swaggystocks.com/dashboard/home,stocks,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:16.641 +0000
+3,2021-09-15 03:54:51.000 +0000,3,3,3,2021-07-26 16:28:39.000 +0000,ad name three,INTERESTED,AD_STATUS_DELIVERY_OK,ENABLE,like and share,v10033g50000c3qbo6jc77uaacd9hvv0,,,https://finviz.com/screener.ashx,more stocks,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:18.918 +0000
+4,2021-09-15 04:26:26.000 +0000,4,4,4,2021-08-05 21:02:33.000 +0000,ad name four,SUBSCRIBE,AD_STATUS_DELIVERY_OK,DISABLE,get stuck in,v10033g50000c3qbo6jc77ucp113o3qg,,,linkedin.com,linkedin,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:18.897 +0000

integration_tests/seeds/tiktok_ad_report_hourly_data.csv

@@ -0,0 +1,5 @@
+ad_id,stat_time_hour,cost_per_conversion,real_time_conversion,cpc,video_play_actions,conversion_rate,video_views_p_75,result,video_views_p_50,impressions,comments,real_time_cost_per_result,conversion,real_time_result,video_views_p_100,shares,real_time_conversion_rate,cost_per_secondary_goal_result,secondary_goal_result_rate,clicks,cost_per_1000_reached,video_views_p_25,reach,real_time_cost_per_conversion,profile_visits_rate,average_video_play,profile_visits,cpm,ctr,video_watched_2_s,follows,result_rate,video_watched_6_s,secondary_goal_result,cost_per_result,average_video_play_per_user,real_time_result_rate,spend,likes,_fivetran_synced
+1,2021-09-15 10:00:00,0,0,0,22,0,2,0,2,23,0,0,0,0,0,0,0,-,-,0,0,22,0,0,0,2.22,0,32.22,0,2,0,0,22,-,0,0,0,2.22,0,2021-09-15 15:56:34.862 +0000
+2,2021-09-15 10:00:00,0,0,0,33,0,0,0,0,22,0,0,0,0,0,0,0,-,-,0,0,3,0,0,0,2.22,0,2.22,0,2,0,0,2,-,0,0,0,0.22,0,2021-09-15 15:56:34.953 +0000
+3,2021-09-15 10:00:00,0,0,0,2,0,0,0,0,2,0,0,0,0,0,0,0,-,-,0,0,0,0,0,0,2.22,0,0,0,0,0,0,0,-,0,0,0,0,0,2021-09-15 15:56:34.923 +0000
+4,2021-09-15 10:00:00,0.22,2,0.22,22,222,2,2,2,22,0,0.22,2,2,2,0,200,-,-,2,0,2,0,0.22,0,2.22,0,22.22,2.22,2,0,100,2,-,0.22,0,222,0.22,2,2021-09-15 15:56:34.950 +0000

integration_tests/seeds/tiktok_adgroup_history_data.csv

@@ -0,0 +1,5 @@
+adgroup_id,updated_at,advertiser_id,campaign_id,create_time,adgroup_name,placement_type,profile_image,landing_page_url,display_name,app_type,app_download_url,app_name,external_action,deep_external_action,creative_material_mode,audience_type,gender,android_osv,ios_osv,budget_mode,schedule_type,dayparting,optimize_goal,cpv_video_duration,pacing,billing_event,bid_type,deep_bid_type,impression_tracking_url,click_tracking_url,status,opt_status,statistic_type,video_download,open_url,open_url_type,fallback_type,budget,bid,conversion_bid,deep_cpabid,schedule_start_time,schedule_end_time,app_id,pixel_id,enable_inventory_filter,is_hfss,is_new_structure,category,is_comment_disable,skip_learning_phase,frequency,frequency_schedule,action_days,audience,excluded_audience,location,interest_category_v_2,pangle_block_app_list_id,action_categories,placement,keywords,age,languages,operation_system,connection_type,carriers,video_actions,package,_fivetran_synced
+1,2021-09-15 02:44:56.000 +0000,111,222,2021-08-05 01:34:25.000 +0000,Ad group name one,PLACEMENT_TYPE_NORMAL,,,,,,,ON_WEB_REGISTER,,DYNAMIC,,GENDER_UNLIMITED,,,BUDGET_MODE_DAY,SCHEDULE_FROM_NOW,,CONVERT,,PACING_MODE_SMOOTH,OCPM,BID_TYPE_NO_BID,,,,ADGROUP_STATUS_DELIVERY_OK,ENABLE,,ALLOW_DOWNLOAD,,,,500,0,0,0,2021-08-05 02:30:23.000 +0000,2031-08-03 02:30:23.000 +0000,,543543543,FALSE,FALSE,TRUE,0,1,1,,,,[ 124938917 ],[ 124938917 ],[ 124938917 ],[ 124938917 ],[],[],"[ ""PLACEMENT_TIKTOK"" ]",,"[ ""AGE_25_34"", ""AGE_35_44"", ""AGE_45_54"", ""AGE_55_100"", ""AGE_18_24"" ]","[ ""en"" ]",[],[],[],[],,2021-09-15 02:47:55.652 +0000
+2,2021-09-15 00:52:08.000 +0000,222,333,2021-08-16 20:40:16.000 +0000,Ad group name two,PLACEMENT_TYPE_NORMAL,,,,,,,ON_WEB_REGISTER,,DYNAMIC,,GENDER_UNLIMITED,,,BUDGET_MODE_INFINITE,SCHEDULE_FROM_NOW,,CONVERT,,PACING_MODE_SMOOTH,OCPM,BID_TYPE_NO_BID,,,,ADGROUP_STATUS_DELIVERY_OK,ENABLE,,ALLOW_DOWNLOAD,,,,0,0,0,0,2021-08-17 13:00:08.000 +0000,2031-08-15 13:00:08.000 +0000,,543543543,FALSE,FALSE,TRUE,0,1,1,,,,[ 124938917 ],[ 124938917 ],[ 124938917 ],[ 124938917 ],[],[],"[ ""PLACEMENT_TIKTOK"" ]",,"[ ""AGE_25_34"", ""AGE_35_44"", ""AGE_45_54"", ""AGE_55_100"", ""AGE_18_24"" ]","[ ""en"" ]",[],[],[],[],,2021-09-15 02:47:55.643 +0000
+3,2021-09-15 02:44:55.000 +0000,333,444,2021-09-13 18:44:14.000 +0000,Ad group name three,PLACEMENT_TYPE_NORMAL,,,,,,,ON_WEB_REGISTER,,CUSTOM,,GENDER_UNLIMITED,,,BUDGET_MODE_DAY,SCHEDULE_FROM_NOW,,CONVERT,,PACING_MODE_SMOOTH,OCPM,BID_TYPE_NO_BID,,,,ADGROUP_STATUS_DELIVERY_OK,ENABLE,,ALLOW_DOWNLOAD,,,,900,0,0,0,2021-09-13 18:44:05.000 +0000,2031-09-11 18:44:05.000 +0000,,654645654,FALSE,FALSE,TRUE,0,1,1,,,,[ 124938917 ],[ 124938917 ],[ 124938917 ],[ 124938917 ],[],[],"[ ""PLACEMENT_TIKTOK"" ]",,"[ ""AGE_25_34"", ""AGE_35_44"", ""AGE_45_54"", ""AGE_55_100"", ""AGE_18_24"" ]","[ ""en"" ]",[],[],[],[],,2021-09-15 02:47:55.635 +0000
+4,2021-09-15 02:44:55.000 +0000,444,555,2021-09-13 19:21:33.000 +0000,Ad group name four,PLACEMENT_TYPE_NORMAL,,,,,,,ON_WEB_REGISTER,,DYNAMIC,,GENDER_UNLIMITED,,,BUDGET_MODE_DAY,SCHEDULE_FROM_NOW,,CONVERT,,PACING_MODE_SMOOTH,OCPM,BID_TYPE_NO_BID,,,,ADGROUP_STATUS_DELIVERY_OK,ENABLE,,ALLOW_DOWNLOAD,,,,500,0,0,0,2021-09-13 20:14:52.000 +0000,2031-09-11 20:14:52.000 +0000,,867876867,FALSE,FALSE,TRUE,0,1,1,,,,[ 124938917 ],[ 124938917 ],[ 124938917 ],[ 124938917 ],[],[],"[ ""PLACEMENT_TIKTOK"" ]",,"[ ""AGE_25_34"", ""AGE_35_44"", ""AGE_45_54"", ""AGE_55_100"", ""AGE_18_24"" ]","[ ""en"" ]",[],[],[],[],,2021-09-15 02:47:55.633 +0000

integration_tests/seeds/tiktok_adgroup_report_hourly_data.csv

@@ -0,0 +1,5 @@
+adgroup_id,stat_time_hour,cost_per_conversion,real_time_conversion,cpc,video_play_actions,conversion_rate,video_views_p_75,result,video_views_p_50,impressions,comments,real_time_cost_per_result,conversion,real_time_result,video_views_p_100,shares,real_time_conversion_rate,cost_per_secondary_goal_result,secondary_goal_result_rate,clicks,cost_per_1000_reached,video_views_p_25,reach,real_time_cost_per_conversion,profile_visits_rate,average_video_play,profile_visits,cpm,ctr,video_watched_2_s,follows,result_rate,video_watched_6_s,secondary_goal_result,cost_per_result,average_video_play_per_user,real_time_result_rate,spend,likes,_fivetran_synced
+1,2021-09-15 5:00:00,0,0,1.77,555,0,33,0,38,32,0,0,0,0,3,0,0,-,-,2,18.44,150,333,0,0,5,0,17.99,1.22,333,0,0,133,-,0,4.66,0,7.6,3,2021-09-15 15:57:22.614 +0000
+2,2021-09-15 7:00:00,1.33,7,0.76,1111,39,33,5,33,3235,0,4.123,5,3,44,0,43.22,-,-,34,0,150,0,1.75,0,3.33,0,13.66,1.44,333,0,36.77,233,-,2.3342,0,38.88,17.22,3,2021-09-15 15:57:22.606 +0000
+3,2021-09-15 7:00:00,3.23,7,1,2222,33,33,7,443,3235,0,2.353,5,7,55,0,22,-,-,33,0,333,0,4.44,0,4.22,0,11.88,1.55,334,0,34,233,-,3.6212,0,22,25.33,4,2021-09-15 15:57:22.616 +0000
+4,2021-09-15 4:00:00,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,2021-09-15 15:57:22.616 +0000

integration_tests/seeds/tiktok_advertiser_data.csv

@@ -0,0 +1,3 @@
+id,name,address,company,contacter,country,currency,description,email,industry,license_no,license_url,promotion_area,reason,role,status,telephone,timezone,balance,create_time,language,phone_number,_fivetran_synced
+1,US,1600 pennsylvania avenue,White House,,US,USD,hello,louis@vuitton.com,123,,,111,,ROLE_ADVERTISER,STATUS_ENABLE,1,Etc/GMT+5,32.12,2021-08-27 18:13:41.000 +0000,en,string,2021-09-15 15:53:18.938 +0000
+2,Canada,21 w wacker st,Blanco House,,CAN,CAN,world,jeff@bezos.com,321,,,0,,ROLE_ADVERTISER,STATUS_ENABLE,1,Etc/GMT+5,321.22,2021-03-09 20:55:37.000 +0000,en,string,2021-09-15 15:53:07.621 +0000

integration_tests/seeds/tiktok_campaign_history_data.csv

@@ -0,0 +1,5 @@
+campaign_id,updated_at,advertiser_id,campaign_name,campaign_type,budget,budget_mode,opt_status,objective_type,is_new_structure,split_test_variable,create_time,_fivetran_synced
+1,2021-09-15 00:52:08.000 +0000,1,campaign name one,REGULAR_CAMPAIGN,450,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:40:15.000 +0000,2021-09-15 02:47:54.737 +0000
+2,2021-09-15 01:33:37.000 +0000,2,campaign name two,REGULAR_CAMPAIGN,300,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:29:06.000 +0000,2021-09-15 02:47:54.737 +0000
+3,2021-09-14 02:53:48.000 +0000,3,campaign name three,REGULAR_CAMPAIGN,450,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:40:15.000 +0000,2021-09-14 03:01:03.005 +0000
+4,2021-09-14 02:59:01.000 +0000,4,campaign name four,REGULAR_CAMPAIGN,300,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:29:06.000 +0000,2021-09-14 03:01:03.006 +0000

integration_tests/seeds/tiktok_campaign_report_hourly_data.csv

@@ -0,0 +1,5 @@
+campaign_id,stat_time_hour,cost_per_conversion,real_time_conversion,cpc,video_play_actions,conversion_rate,video_views_p_75,result,video_views_p_50,impressions,comments,real_time_cost_per_result,conversion,real_time_result,video_views_p_100,shares,real_time_conversion_rate,cost_per_secondary_goal_result,secondary_goal_result_rate,clicks,cost_per_1000_reached,video_views_p_25,reach,real_time_cost_per_conversion,profile_visits_rate,average_video_play,profile_visits,cpm,ctr,video_watched_2_s,follows,result_rate,video_watched_6_s,secondary_goal_result,cost_per_result,average_video_play_per_user,real_time_result_rate,spend,likes,_fivetran_synced
+1,2021-09-15 5:00:00,0,0,1.88,32322,0,222,222,222,22222,0,3.3235,0,23423,33,0,0,-,-,22,2.234,233,22413,0,0,2.22,0,2.35,0.33,2222,0,22.35,322,-,35.32,2.13,22.22,222.12,22,2021-09-15 15:58:05.904 +0000
+2,2021-09-15 4:00:00,0,0,0,4,0,0,0,0,4,0,0,0,0,0,0,0,-,-,0,22,1,4,0,0,3.334,0,22,0,2,0,0,2,-,3,2.35,0,0.22,2,2021-09-15 15:58:05.905 +0000
+3,2021-09-15 8:00:00,2.3,22,2.2,222222,222,222,22,787,12222,0,4.12312,22,33,222,2,22.2,-,-,222,0,2335,0,2.55,0,3.52,0,22.22,0.232,3322,0,33.22,2222,-,2.2222,0,22.2,222.22,22,2021-09-15 15:58:05.903 +0000
+4,2021-09-15 10:00:00,0,0,0,15,0,1,0,1,22,0,0,0,0,1,0,0,-,-,0,0,2,0,0,0,3.324,0,22,0,3,0,0,4,-,0,0,0,0.232,22,2021-09-15 15:58:05.903 +0000

macros/get_staging_files.sql

@@ -30,6 +30,10 @@
     {% set _ = staging_file.append(ref('stg_snapchat_ads')) %}
     {% endif %}
 
+    {% if var('ad_reporting__tiktok_ads_enabled') %} 
+    {% set _ = staging_file.append(ref('stg_tiktok_ads')) %}
+    {% endif %}
+
 
     {{ return(staging_file) }}
 

models/stg_tiktok_ads.sql

@@ -0,0 +1,38 @@
+{{ config(enabled=var('ad_reporting__tiktok_ads_enabled')) }}
+
+with base as (
+
+    select *
+    from {{ ref('tiktok_ads__ad_adapter')}}
+
+), fields as (
+
+    select
+        advertiser_name as account_name,
+        cast(advertiser_id as {{ dbt_utils.type_string() }}) as account_id,
+        cast(date_day as date) as date_day,
+        base_url,
+        url_host,
+        url_path,
+        utm_source,
+        utm_medium,
+        utm_campaign,
+        utm_content,
+        utm_term,
+        cast(campaign_id as {{ dbt_utils.type_string() }}) as campaign_id,
+        campaign_name,
+        cast(ad_group_id as {{ dbt_utils.type_string() }}) as ad_group_id,
+        ad_group_name as ad_group_name,
+        'TikTok Ads' as platform,
+        sum(coalesce(clicks, 0)) as clicks,
+        sum(coalesce(impressions, 0)) as impressions,
+        sum(coalesce(spend, 0)) as spend
+
+    from base
+    {{ dbt_utils.group_by(16) }}
+
+
+)
+
+select *
+from fields

packages.yml

@@ -11,6 +11,9 @@ packages:
 - package: fivetran/google_ads
   version: [">=0.5.0", "<0.6.0"]
 
+- package: fivetran/tiktok_ads
+  version: [">=0.1.0", "<0.2.0"]
+
 - package: fivetran/twitter_ads
   version: [">=0.4.0", "<0.5.0"]