Merge pull request #21 from fivetran/feature/v2-updates

Ad reporting v2 updates

CHANGELOG.md

@@ -1,3 +1,35 @@
+# dbt_linkedin v0.5.0
+
+PR [#21](https://github.com/fivetran/dbt_linkedin/pull/21) includes the following changes:
+
+## 🚨 Breaking Changes 🚨
+- **ALL** models and **ALL** variables now have the prefix `linkedin_ads_*`. They previously were prepended with `linkedin_*`. This includes the required schema and database variables. We made this change to better discern between Linkedin Ads and [Linkedin Pages](https://github.com/fivetran/dbt_linkedin_pages/tree/main).
+- The following models have been renamed:
+  - `linkedin__account_ad_report` -> `linkedin_ads__account_report`
+  - `linkedin__campaign_ad_report` -> `linkedin_ads__campaign_report`
+  - `linkedin__campaign_group_ad_report` -> `linkedin_ads__campaign_group_report`
+- The `linkedin__ad_adapter` model has been renamed and refactored into two separate models:
+  - `linkedin_ads__url_report`: Each record in this table represents the daily performance at the url level.
+  - `linkedin_ads__creative_report`: Each record in this table represents the daily performance at the creative level.
+- The declaration of passthrough variables within your root `dbt_project.yml` has changed. To allow for more flexibility and better tracking of passthrough columns, you will now want to define passthrough columns in the following format:
+```yml
+vars:
+  linkedin_ads__campaign_passthrough_metrics: # this will pass through fields to the account, campaign, and campaign group report models. it pulls from `ad_analytics_by_campaign`
+    - name: "my_field_to_include" # Required: Name of the field within the source.
+      alias: "field_alias" # Optional: If you wish to alias the field within the staging model.
+  linkedin_ads__creative_passthrough_metrics: # this will pass through fields to the creative and url report models.  it pulls from `ad_analytics_by_creative`
+    - name: "my_field_to_include"
+      alias: "field_alias"
+```
+- Staging models are now by default written within a schema titled (`<target_schema>` + `_linkedin_ads_source`) in your destination. Previously, this was titled (`<target_schema>` + `_stg_linkedin`).
+
+## 🎉 Feature Enhancements 🎉
+- README updates for easier navigation and use of the package.
+- Addition of identifier variables for each of the source tables to allow for further flexibility in source table direction within the dbt project.
+- Addition of new columns to `_report` models.
+- More complete table and column documentation.
+- More robust schema tests.
+
 # dbt_linkedin v0.4.0
 🎉 dbt v1.0.0 Compatibility 🎉
 ## 🚨 Breaking Changes 🚨
@@ -8,4 +40,4 @@
   - Please note, if you are installing a version of `dbt_utils` in your `packages.yml` that is not in the range above then you will encounter a package dependency error.
 
 # dbt_linkedin v0.1.0 -> v0.3.0
-Refer to the relevant release notes on the Github repository for specific details for the previous releases. Thank you!
+Refer to the relevant release notes on the Github repository for specific details for the previous releases. Thank you!

dbt_project.yml

@@ -1,16 +1,20 @@
 name: 'linkedin'
-version: '0.4.0'
+version: '0.5.0'
 config-version: 2
 require-dbt-version: [">=1.0.0", "<2.0.0"]
+
 vars:
   linkedin:
-    ad_analytics_by_creative: "{{ ref('stg_linkedin__ad_analytics_by_creative') }}"
-    creative_history: "{{ ref('stg_linkedin__creative_history') }}"
-    campaign_history: "{{ ref('stg_linkedin__campaign_history') }}"
-    campaign_group_history: "{{ ref('stg_linkedin__campaign_group_history') }}"
-    account_history: "{{ ref('stg_linkedin__account_history') }}"
-  linkedin__passthrough_metrics: []
+    ad_analytics_by_creative: "{{ ref('stg_linkedin_ads__ad_analytics_by_creative') }}"
+    creative_history: "{{ ref('stg_linkedin_ads__creative_history') }}"
+    campaign_history: "{{ ref('stg_linkedin_ads__campaign_history') }}"
+    campaign_group_history: "{{ ref('stg_linkedin_ads__campaign_group_history') }}"
+    account_history: "{{ ref('stg_linkedin_ads__account_history') }}"
+    ad_analytics_by_campaign: "{{ ref('stg_linkedin_ads__ad_analytics_by_campaign') }}"
+  linkedin_ads__creative_passthrough_metrics: []
+  linkedin_ads__campaign_passthrough_metrics: []
+
 models:
   linkedin:
     +materialized: table
-    +schema: linkedin
+    +schema: linkedin_ads

integration_tests/dbt_project.yml

@@ -1,23 +1,25 @@
 name: 'linkedin_integration_tests'
-version: '0.3.0'
+version: '0.5.0'
 profile: 'integration_tests'
 config-version: 2
 
 vars:
   linkedin_source:
-        account_history: "{{ ref('linkedin_account_history_data') }}"
-        ad_analytics_by_creative: "{{ ref('linkedin_ad_analytics_by_creative_data') }}"
-        campaign_group_history: "{{ ref('linkedin_campaign_group_history_data') }}"
-        campaign_history: "{{ ref('linkedin_campaign_history_data') }}"
-        creative_history: "{{ ref('linkedin_creative_history_data') }}"
+    linkedin_ads_schema: linkedin_integration_tests
+    account_history: "{{ ref('linkedin_account_history_data') }}"
+    ad_analytics_by_creative: "{{ ref('linkedin_ad_analytics_by_creative_data') }}"
+    ad_analytics_by_campaign: "{{ ref('linkedin_ad_analytics_by_campaign_data') }}"
+    campaign_group_history: "{{ ref('linkedin_campaign_group_history_data') }}"
+    campaign_history: "{{ ref('linkedin_campaign_history_data') }}"
+    creative_history: "{{ ref('linkedin_creative_history_data') }}"
 
 seeds:
   linkedin_integration_tests:
     linkedin_creative_history_data:
       +column_types:
         created_time: timestamp
         last_modified_time: timestamp
-        click_uri: "{{ 'string' if target.type == 'bigquery' else 'varchar' }}"
+        click_uri: "{{ 'string' if target.type in ('bigquery', 'spark') else 'varchar' }}"
     linkedin_ad_analytics_by_creative_data:
       +column_types:
         day: timestamp
@@ -33,10 +35,7 @@ seeds:
       +column_types:
         created_time: timestamp
         last_modified_time: timestamp
-    linkedin_creative_history_data:
-      +column_types:
-        created_time: timestamp
-        last_modified_time: timestamp
+
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']

integration_tests/requirements.txt

@@ -1,6 +1,6 @@
-dbt-snowflake~=1.0.0
-dbt-bigquery~=1.0.0
-dbt-redshift~=1.0.0
-dbt-postgres~=1.0.0
-dbt-spark~=1.0.0
-dbt-spark[PyHive]~=1.0.0
+dbt-snowflake>=1.0.0
+dbt-bigquery>=1.0.0
+dbt-redshift>=1.0.0
+dbt-postgres>=1.0.0
+dbt-spark>=1.0.0
+dbt-spark[PyHive]>=1.0.0

integration_tests/seeds/linkedin_ad_analytics_by_campaign_data.csv

@@ -0,0 +1,21 @@
+campaign_id,day,_fivetran_synced,action_clicks,ad_unit_clicks,approximate_unique_impressions,card_clicks,card_impressions,clicks,comment_likes,comments,company_page_clicks,conversion_value_in_local_currency,cost_in_local_currency,cost_in_usd,external_website_conversions,external_website_post_click_conversions,external_website_post_view_conversions,follows,full_screen_plays,impressions,landing_page_clicks,lead_generation_mail_contact_info_shares,lead_generation_mail_interested_clicks,likes,one_click_lead_form_opens,one_click_leads,opens,other_engagements,shares,text_url_clicks,total_engagements,video_completions,video_first_quartile_completions,video_midpoint_completions,video_starts,video_third_quartile_completions,video_views,viral_card_clicks,viral_card_impressions,viral_clicks,viral_comment_likes,viral_comments,viral_company_page_clicks,viral_external_website_conversions,viral_external_website_post_click_conversions,viral_external_website_post_view_conversions,viral_follows,viral_full_screen_plays,viral_impressions,viral_landing_page_clicks,viral_likes,viral_one_click_lead_form_opens,viral_one_click_leads,viral_other_engagements,viral_shares,viral_total_engagements,viral_video_completions,viral_video_first_quartile_completions,viral_video_midpoint_completions,viral_video_starts,viral_video_third_quartile_completions,viral_video_views
+148633856,2020-04-06 00:00:00,2021-04-06 12:21:11.686,,,0,,0,18,0,,,0,0,,0,,,,,,,0,0,,,,18,0,,,18,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+148633856,2020-04-03 00:00:00,2021-04-06 12:21:11.686,,,0,,0,18,0,,,0,0,,0,,,,,,,0,0,,,,18,0,,,18,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+148633856,2020-03-28 00:00:00,2021-04-06 12:21:11.686,1,,125,,0,51,0,0,,0,50.2,50.2,0,,0,,,125,1,0,0,,0,,51,0,0,,52,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-23 00:00:00,2021-04-06 12:21:11.686,1,,133,,0,52,0,0,,0,50.31,50.31,0,,0,,,143,1,0,0,,0,,52,0,0,,53,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-27 00:00:00,2021-04-06 12:21:11.686,0,,119,,0,59,0,0,,0,50.43,50.43,0,,0,,,126,0,0,0,,0,,59,0,0,,59,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-31 00:00:00,2021-04-06 12:21:11.686,2,,125,,0,85,0,0,,0,50.07,50.07,0,,0,,,125,2,0,0,,0,,85,0,0,,87,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-26 00:00:00,2021-04-06 12:21:11.686,2,0,67,0,0,37,0,0,0,0,30.12,30.12,0,0,0,0,0,75,3,0,0,0,0,0,37,0,0,1,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-24 00:00:00,2021-04-06 12:21:11.686,0,0,143,0,0,52,0,0,0,0,50.31,50.31,0,0,0,0,0,143,1,0,0,0,0,0,52,0,0,1,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-25 00:00:00,2021-04-06 12:21:11.686,1,0,124,0,0,56,0,0,0,0,50.05,50.05,0,0,0,0,0,125,2,0,0,0,0,0,56,0,0,1,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-29 00:00:00,2021-04-06 12:21:11.685,2,0,125,0,0,68,0,0,0,0,50.16,50.16,0,0,0,0,0,125,3,0,0,0,0,0,68,0,0,1,71,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-04-02 00:00:00,2021-04-06 12:21:11.686,0,0,122,0,0,74,0,0,0,0,48.8,48.8,0,0,0,0,0,122,2,0,0,0,0,0,74,0,0,2,76,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-04-01 00:00:00,2021-04-06 12:21:11.686,0,0,120,0,0,79,0,0,0,0,50.13,50.13,0,0,0,0,0,125,2,0,0,0,0,0,79,0,0,2,81,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-30 00:00:00,2021-04-06 12:21:11.686,3,0,125,0,0,82,0,0,0,0,50.04,50.04,0,0,0,0,0,125,4,0,0,0,0,0,82,0,0,1,86,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+174096954,2021-05-25 00:00:00,2021-06-24 14:43:55.448,,,0,,0,23,0,,,0,0,,0,,,,,,,0,0,,,,23,0,,,23,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+174096954,2021-05-20 00:00:00,2021-06-19 14:38:29.197,,,0,,0,35,0,,,0,0,,0,,,,,,,0,0,,,,35,0,,,35,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+167276694,2020-10-22 00:00:00,2021-04-06 08:04:06.348,,,0,,0,38,0,,,0,0,,0,,,,,,,0,0,,,,38,0,,,38,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+174096954,2021-05-19 00:00:00,2021-06-18 14:46:41.820,,,0,,0,43,0,,,0,0,,0,,,,,,,0,0,,,,43,0,,,43,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+167276694,2020-11-03 00:00:00,2021-04-06 08:04:06.348,,,0,,0,15,0,,,0,0,,0,,,,,,,0,0,,,,15,0,,,15,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+165867684,2020-07-31 00:00:00,2021-04-06 17:16:49.608,,,0,,0,17,0,,,0,0,,0,,,,,,,0,0,,,,17,0,,,17,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+165867684,2020-08-03 00:00:00,2021-04-06 17:16:49.608,,,0,,0,17,0,,,0,0,,0,,,,,,,0,0,,,,17,0,,,17,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0

integration_tests/seeds/linkedin_campaign_history_data.csv

@@ -4,25 +4,25 @@ id,last_modified_time,account_id,campaign_group_id,created_time,name,version_tag
 132866546,2019-09-14 17:52:40.697,507515057,600268686,2018-07-27 20:51:22,lRq7HdKfzTQkOokgrw809Q==,33
 147624636,2020-03-31 23:49:40.206,507515057,600268686,2020-02-17 08:55:44,vqkXsHSguUTB/tNx2xICkQ==,73
 147624646,2020-03-31 23:49:40.206,507515057,600268686,2020-02-17 08:56:24,p878CnpgR+9mdq58o3nkhg==,64
-147624706,2020-03-31 23:49:40.206,507515057,600268686,2020-02-17 08:57:41,kUcfickNRhqYy57PLNwqTQ==,66
-162286513,2020-07-14 11:54:05,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,125
+148633856,2020-03-31 23:49:40.206,507515057,600268686,2020-02-17 08:57:41,kUcfickNRhqYy57PLNwqTQ==,66
+148633856,2020-07-14 11:54:05,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,125
 162286513,2020-10-19 08:00:30,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,170
 162286513,2020-10-19 15:20:48,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,171
 162286513,2020-10-18 10:07:35,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,167
 162286513,2020-10-16 16:44:41,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,163
-162286513,2020-10-17 23:08:54,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,166
+174096954,2020-10-17 23:08:54,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,166
 162286513,2020-10-17 02:47:20,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,164
 162286513,2020-08-07 09:26:07,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,160
 162286513,2020-10-22 21:56:41,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,175
 162286513,2020-08-06 21:10:53,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,145
-162286513,2020-07-21 00:57:53,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,141
+167276694,2020-07-21 00:57:53,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,141
 162286513,2020-07-13 20:22:54,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,124
 162286513,2020-08-06 17:12:01,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,144
 162286513,2020-10-16 07:40:36,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,162
 162286513,2020-10-17 11:38:02,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,165
 162286513,2020-10-15 16:48:15,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,161
 162286513,2020-10-18 20:13:44,507515057,600268686,2020-05-06 20:41:33,+A+12nfTw2I6vA3PNV0GDw==,168
-162286313,2020-07-13 19:55:36,507515057,600268686,2020-05-06 20:37:36,XjFwQcqOj/a3ca8uS3EUzA==,126
+165867684,2020-07-13 19:55:36,507515057,600268686,2020-05-06 20:37:36,XjFwQcqOj/a3ca8uS3EUzA==,126
 162286313,2020-08-24 21:06:55,507515057,600268686,2020-05-06 20:37:36,XjFwQcqOj/a3ca8uS3EUzA==,206
 162286313,2020-10-16 16:20:40,507515057,600268686,2020-05-06 20:37:36,XjFwQcqOj/a3ca8uS3EUzA==,226
 162286313,2020-10-17 04:59:30,507515057,600268686,2020-05-06 20:37:36,XjFwQcqOj/a3ca8uS3EUzA==,227