Merge pull request #28 from fivetran/feature/linkedin-api-updates-2023

Feature/linkedin api updates 2023

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -66,6 +66,7 @@ steps:
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_CATALOG"
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"

.gitignore

@@ -1 +1,5 @@
+target/
+dbt_modules/
+logs/
+.DS_Store
 dbt_packages/

CHANGELOG.md

@@ -1,3 +1,26 @@
+# dbt_linkedin v0.7.0
+## 🚨 Breaking Changes 🚨
+Due to Linkedin Ads API [change in January 2023](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/recent-changes?view=li-lms-2022-12#january-2023), there have been updates in the Linkedin Ads Fivetran Connector and therefore, updates to this Linkedin package. 
+
+The following fields have been completely deprecated in the `stg_linkedin_ads__creative_history` model ([dbt_linkedin_source PR #48](https://github.com/fivetran/dbt_linkedin_source/pull/48)) and therefore removed from the below respective models in this package([#28](https://github.com/fivetran/dbt_linkedin/pull/28)):
+- `type` (removed from `*_creative_report`)
+- `call_to_action_label_type` (removed from `*_creative_report`)
+- `version_tag` (removed from `*_creative_report` and `*_url_report`)
+
+## Updates
+[PR #28](https://github.com/fivetran/dbt_linkedin/pull/28) introduces the below changes:
+- `linkedin_ads__creative_report` now leverages `report.creative_id` instead of `creative.creative_id`. 
+- `linkedin_ads__campaign_report` now leverages `report.campaign_id` instead of `campaign.campaign_id`.
+- `linkedin_ads__url_report` now leverages `report.creative_id` instead of `creative.creative_id`.
+
+## Under the hood
+- `integration_tests/seed/linkedin_creative_history_data` has been updated to reflect new fields and deprecated field updates
+- Legacy fields have been updated respectively in the connector and `dbt_linkedin_source v0.7.0` includes modifications that could affect the below fields within the `linkedin_ads__creative_history` model:
+  - `last_modified_at`
+  - `created_at`
+  - `status`
+
+For more information, please refer to [dbt_linkedin_source PR #48](https://github.com/fivetran/dbt_linkedin_source/pull/48).
 # dbt_linkedin v0.6.1
 
 ## Bugfixes

README.md

@@ -55,7 +55,7 @@ Include the following Linkedin Ads package version in your `packages.yml` file:
 # packages.yml
 packages:
   - package: fivetran/linkedin
-    version: [">=0.6.0", "<0.7.0"]
+    version: [">=0.7.0", "<0.8.0"]
 ```
 
 ## Step 3: Define database and schema variables
@@ -136,7 +136,7 @@ This dbt package is dependent on the following dbt packages. Please be aware tha
 ```yml
 packages:
     - package: fivetran/linkedin_source
-      version: [">=0.6.0", "<0.7.0"]
+      version: [">=0.7.0", "<0.8.0"]
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]
     - package: dbt-labs/dbt_utils

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'linkedin'
-version: '0.6.1'
+version: '0.7.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: linkedin_integration_tests_2
+      schema: linkedin_integration_tests_4
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: linkedin_integration_tests_2
+      schema: linkedin_integration_tests_4
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: linkedin_integration_tests_2
+      schema: linkedin_integration_tests_4
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: linkedin_integration_tests_2
+      schema: linkedin_integration_tests_4
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: linkedin_integration_tests_2
+      schema: linkedin_integration_tests_4
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,25 +1,30 @@
 name: 'linkedin_integration_tests'
-version: '0.6.1'
+version: '0.7.0'
 profile: 'integration_tests'
 config-version: 2
 
 vars:
-  linkedin_source:
-    linkedin_ads_schema: linkedin_integration_tests_2
-    account_history: "{{ ref('linkedin_account_history_data') }}"
-    ad_analytics_by_creative: "{{ ref('linkedin_ad_analytics_by_creative_data') }}"
-    ad_analytics_by_campaign: "{{ ref('linkedin_ad_analytics_by_campaign_data') }}"
-    campaign_group_history: "{{ ref('linkedin_campaign_group_history_data') }}"
-    campaign_history: "{{ ref('linkedin_campaign_history_data') }}"
-    creative_history: "{{ ref('linkedin_creative_history_data') }}"
+    linkedin_ads_schema: linkedin_integration_tests_4
+    linkedin_ads_account_history_identifier: "linkedin_account_history_data"
+    linkedin_ads_ad_analytics_by_creative_identifier: "linkedin_ad_analytics_by_creative_data"
+    linkedin_ads_campaign_group_history_identifier: "linkedin_campaign_group_history_data"
+    linkedin_ads_campaign_history_identifier: "linkedin_campaign_history_data"
+    linkedin_ads_creative_history_identifier: "linkedin_creative_history_data"
+    linkedin_ads_ad_analytics_by_campaign_identifier: "linkedin_ad_analytics_by_campaign_data"
+
+
 
 seeds:
   linkedin_integration_tests:
     linkedin_creative_history_data:
       +column_types:
         created_time: timestamp
         last_modified_time: timestamp
+        created_at: timestamp
+        last_modified_at: timestamp
         click_uri: "{{ 'string' if target.type in ('bigquery','spark','databricks') else 'varchar' }}"
+        intended_status: "{{ 'string' if target.type in ('bigquery','spark','databricks') else 'varchar' }}"
+        status: "{{ 'string' if target.type in ('bigquery','spark','databricks') else 'varchar' }}"
     linkedin_ad_analytics_by_creative_data:
       +column_types:
         day: timestamp
@@ -35,7 +40,7 @@ seeds:
       +column_types:
         created_time: timestamp
         last_modified_time: timestamp
-
+        
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']

integration_tests/seeds/linkedin_ad_analytics_by_campaign_data.csv

@@ -1,21 +1,21 @@
-campaign_id,day,_fivetran_synced,action_clicks,ad_unit_clicks,approximate_unique_impressions,card_clicks,card_impressions,clicks,comment_likes,comments,company_page_clicks,conversion_value_in_local_currency,cost_in_local_currency,cost_in_usd,external_website_conversions,external_website_post_click_conversions,external_website_post_view_conversions,follows,full_screen_plays,impressions,landing_page_clicks,lead_generation_mail_contact_info_shares,lead_generation_mail_interested_clicks,likes,one_click_lead_form_opens,one_click_leads,opens,other_engagements,shares,text_url_clicks,total_engagements,video_completions,video_first_quartile_completions,video_midpoint_completions,video_starts,video_third_quartile_completions,video_views,viral_card_clicks,viral_card_impressions,viral_clicks,viral_comment_likes,viral_comments,viral_company_page_clicks,viral_external_website_conversions,viral_external_website_post_click_conversions,viral_external_website_post_view_conversions,viral_follows,viral_full_screen_plays,viral_impressions,viral_landing_page_clicks,viral_likes,viral_one_click_lead_form_opens,viral_one_click_leads,viral_other_engagements,viral_shares,viral_total_engagements,viral_video_completions,viral_video_first_quartile_completions,viral_video_midpoint_completions,viral_video_starts,viral_video_third_quartile_completions,viral_video_views
-148633856,2020-04-06 00:00:00,2021-04-06 12:21:11.686,,,0,,0,18,0,,,0,0,,0,,,,,,,0,0,,,,18,0,,,18,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-148633856,2020-04-03 00:00:00,2021-04-06 12:21:11.686,,,0,,0,18,0,,,0,0,,0,,,,,,,0,0,,,,18,0,,,18,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-148633856,2020-03-28 00:00:00,2021-04-06 12:21:11.686,1,,125,,0,51,0,0,,0,50.2,50.2,0,,0,,,125,1,0,0,,0,,51,0,0,,52,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
-148633856,2020-03-23 00:00:00,2021-04-06 12:21:11.686,1,,133,,0,52,0,0,,0,50.31,50.31,0,,0,,,143,1,0,0,,0,,52,0,0,,53,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
-148633856,2020-03-27 00:00:00,2021-04-06 12:21:11.686,0,,119,,0,59,0,0,,0,50.43,50.43,0,,0,,,126,0,0,0,,0,,59,0,0,,59,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
-148633856,2020-03-31 00:00:00,2021-04-06 12:21:11.686,2,,125,,0,85,0,0,,0,50.07,50.07,0,,0,,,125,2,0,0,,0,,85,0,0,,87,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
-148633856,2020-03-26 00:00:00,2021-04-06 12:21:11.686,2,0,67,0,0,37,0,0,0,0,30.12,30.12,0,0,0,0,0,75,3,0,0,0,0,0,37,0,0,1,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-03-24 00:00:00,2021-04-06 12:21:11.686,0,0,143,0,0,52,0,0,0,0,50.31,50.31,0,0,0,0,0,143,1,0,0,0,0,0,52,0,0,1,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-03-25 00:00:00,2021-04-06 12:21:11.686,1,0,124,0,0,56,0,0,0,0,50.05,50.05,0,0,0,0,0,125,2,0,0,0,0,0,56,0,0,1,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-03-29 00:00:00,2021-04-06 12:21:11.685,2,0,125,0,0,68,0,0,0,0,50.16,50.16,0,0,0,0,0,125,3,0,0,0,0,0,68,0,0,1,71,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-04-02 00:00:00,2021-04-06 12:21:11.686,0,0,122,0,0,74,0,0,0,0,48.8,48.8,0,0,0,0,0,122,2,0,0,0,0,0,74,0,0,2,76,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-04-01 00:00:00,2021-04-06 12:21:11.686,0,0,120,0,0,79,0,0,0,0,50.13,50.13,0,0,0,0,0,125,2,0,0,0,0,0,79,0,0,2,81,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-03-30 00:00:00,2021-04-06 12:21:11.686,3,0,125,0,0,82,0,0,0,0,50.04,50.04,0,0,0,0,0,125,4,0,0,0,0,0,82,0,0,1,86,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-174096954,2021-05-25 00:00:00,2021-06-24 14:43:55.448,,,0,,0,23,0,,,0,0,,0,,,,,,,0,0,,,,23,0,,,23,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-174096954,2021-05-20 00:00:00,2021-06-19 14:38:29.197,,,0,,0,35,0,,,0,0,,0,,,,,,,0,0,,,,35,0,,,35,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-167276694,2020-10-22 00:00:00,2021-04-06 08:04:06.348,,,0,,0,38,0,,,0,0,,0,,,,,,,0,0,,,,38,0,,,38,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-174096954,2021-05-19 00:00:00,2021-06-18 14:46:41.820,,,0,,0,43,0,,,0,0,,0,,,,,,,0,0,,,,43,0,,,43,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-167276694,2020-11-03 00:00:00,2021-04-06 08:04:06.348,,,0,,0,15,0,,,0,0,,0,,,,,,,0,0,,,,15,0,,,15,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-165867684,2020-07-31 00:00:00,2021-04-06 17:16:49.608,,,0,,0,17,0,,,0,0,,0,,,,,,,0,0,,,,17,0,,,17,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-165867684,2020-08-03 00:00:00,2021-04-06 17:16:49.608,,,0,,0,17,0,,,0,0,,0,,,,,,,0,0,,,,17,0,,,17,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+campaign_id,day,action_clicks,ad_unit_clicks,approximate_unique_impressions,card_clicks,card_impressions,clicks,comment_likes,comments,company_page_clicks,conversion_value_in_local_currency,cost_in_local_currency,cost_in_usd,external_website_conversions,external_website_post_click_conversions,external_website_post_view_conversions,follows,full_screen_plays,impressions,landing_page_clicks,lead_generation_mail_contact_info_shares,lead_generation_mail_interested_clicks,likes,one_click_lead_form_opens,one_click_leads,opens,other_engagements,shares,text_url_clicks,total_engagements,video_completions,video_first_quartile_completions,video_midpoint_completions,video_starts,video_third_quartile_completions,video_views,viral_card_clicks,viral_card_impressions,viral_clicks,viral_comment_likes,viral_comments,viral_company_page_clicks,viral_external_website_conversions,viral_external_website_post_click_conversions,viral_external_website_post_view_conversions,viral_follows,viral_full_screen_plays,viral_impressions,viral_landing_page_clicks,viral_likes,viral_one_click_lead_form_opens,viral_one_click_leads,viral_other_engagements,viral_shares,viral_total_engagements,viral_video_completions,viral_video_first_quartile_completions,viral_video_midpoint_completions,viral_video_starts,viral_video_third_quartile_completions,viral_video_views
+148633856,2020-04-06 00:00:00,,,0,,0,18,0,,,0,0,,0,,,,,,,0,0,,,,18,0,,,18,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+148633856,2020-04-03 00:00:00,,,0,,0,18,0,,,0,0,,0,,,,,,,0,0,,,,18,0,,,18,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+148633856,2020-03-28 00:00:00,1,,125,,0,51,0,0,,0,50.2,50.2,0,,0,,,125,1,0,0,,0,,51,0,0,,52,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-23 00:00:00,1,,133,,0,52,0,0,,0,50.31,50.31,0,,0,,,143,1,0,0,,0,,52,0,0,,53,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-27 00:00:00,0,,119,,0,59,0,0,,0,50.43,50.43,0,,0,,,126,0,0,0,,0,,59,0,0,,59,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-31 00:00:00,2,,125,,0,85,0,0,,0,50.07,50.07,0,,0,,,125,2,0,0,,0,,85,0,0,,87,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-26 00:00:00,2,0,67,0,0,37,0,0,0,0,30.12,30.12,0,0,0,0,0,75,3,0,0,0,0,0,37,0,0,1,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-24 00:00:00,0,0,143,0,0,52,0,0,0,0,50.31,50.31,0,0,0,0,0,143,1,0,0,0,0,0,52,0,0,1,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-25 00:00:00,1,0,124,0,0,56,0,0,0,0,50.05,50.05,0,0,0,0,0,125,2,0,0,0,0,0,56,0,0,1,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-29 00:00:00,2,0,125,0,0,68,0,0,0,0,50.16,50.16,0,0,0,0,0,125,3,0,0,0,0,0,68,0,0,1,71,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-04-02 00:00:00,0,0,122,0,0,74,0,0,0,0,48.8,48.8,0,0,0,0,0,122,2,0,0,0,0,0,74,0,0,2,76,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-04-01 00:00:00,0,0,120,0,0,79,0,0,0,0,50.13,50.13,0,0,0,0,0,125,2,0,0,0,0,0,79,0,0,2,81,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-30 00:00:00,3,0,125,0,0,82,0,0,0,0,50.04,50.04,0,0,0,0,0,125,4,0,0,0,0,0,82,0,0,1,86,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+174096954,2021-05-25 00:00:00,,,0,,0,23,0,,,0,0,,0,,,,,,,0,0,,,,23,0,,,23,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+174096954,2021-05-20 00:00:00,,,0,,0,35,0,,,0,0,,0,,,,,,,0,0,,,,35,0,,,35,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+167276694,2020-10-22 00:00:00,,,0,,0,38,0,,,0,0,,0,,,,,,,0,0,,,,38,0,,,38,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+174096954,2021-05-19 00:00:00,,,0,,0,43,0,,,0,0,,0,,,,,,,0,0,,,,43,0,,,43,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+167276694,2020-11-03 00:00:00,,,0,,0,15,0,,,0,0,,0,,,,,,,0,0,,,,15,0,,,15,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+165867684,2020-07-31 00:00:00,,,0,,0,17,0,,,0,0,,0,,,,,,,0,0,,,,17,0,,,17,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+165867684,2020-08-03 00:00:00,,,0,,0,17,0,,,0,0,,0,,,,,,,0,0,,,,17,0,,,17,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0