api updates as of 2/3

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -66,6 +66,7 @@ steps:
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_CATALOG"
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"

.gitignore

@@ -1 +1,5 @@
+target/
+dbt_modules/
+logs/
+.DS_Store
 dbt_packages/

CHANGELOG.md

@@ -1,3 +1,24 @@
+# dbt_linkedin_source v0.7.0
+## 🚨 Breaking Changes 🚨
+Due to Linkedin Ads API [change in January 2023](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/recent-changes?view=li-lms-2022-12#january-2023), there have been updates in the Linkedin Ads Fivetran Connector and therefore, updates to this Linkedin package. 
+
+The following fields have been completely deprecated in the `stg_linkedin_ads__creative_history` model ([PR #48](https://github.com/fivetran/dbt_linkedin_source/pull/48)):
+- `type`
+- `call_to_action_label_type`
+- `version_tag`
+
+## Updates
+- The following legacy fields have been updated respectively in the connector and [PR #48](https://github.com/fivetran/dbt_linkedin_source/pull/48) includes the below modifications:
+  - `last_modified_time` has been updated to `last_modified_at`
+  - `created_time` has been updated to `created_at`
+  - `status` has been updated to `intended_status`
+- `src_linkedin.yml` have been updated to reflect new definitions for the above updated fields.
+- Removing unique column testing from `stg_linkedin__creative_history` as a result of the recent [API version update](https://fivetran.com/docs/applications/linkedin-ads/changelog#january2023) that impacted the `CREATIVE_HISTORY` table. We were recently made aware of an edge case that results in duplicate records for a given `Creative ID` due to a primary key change (`last_modified_time` to `last_modified_at`). Duplicate data will appear if a creative was deleted from the LinkedIn Ads platform during the API update process -- the likelihood of this happening is small and it would only impact deleted creatives.
+
+## Under the Hood
+- `integration_tests/seeds/linkedin_creative_history_data` has been updated to reflect new fields and deprecated fields
+- `_fivetran_synced` field removed from seed data for `linkedin_ad_analytics_by_campaign_data` integration testing as it is not used in this package's models
+
 # dbt_linkedin_source v0.6.0
 
 ## 🚨 Breaking Changes 🚨:

README.md

@@ -42,7 +42,7 @@ Include the following linkedin_source package version in your `packages.yml` fil
 # packages.yml
 packages:
   - package: fivetran/linkedin_source
-    version: [">=0.6.0", "<0.7.0"]
+    version: [">=0.7.0", "<0.8.0"]
 ```
 
 ## Step 3: Define database and schema variables

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'linkedin_source'
-version: '0.6.0'
+version: '0.7.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: linkedin_source_integration_tests
+      schema: linkedin_source_integration_tests_1
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: linkedin_source_integration_tests
+      schema: linkedin_source_integration_tests_1
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: linkedin_source_integration_tests
+      schema: linkedin_source_integration_tests_1
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: linkedin_source_integration_tests
+      schema: linkedin_source_integration_tests_1
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: linkedin_source_integration_tests
+      schema: linkedin_source_integration_tests_1
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'linkedin_source_integration_tests'
-version: '0.6.0'
+version: '0.7.0'
 profile: 'integration_tests'
 config-version: 2
 
@@ -12,15 +12,19 @@ vars:
     linkedin_ads_creative_history_identifier: "linkedin_creative_history_data"
     linkedin_ads_ad_analytics_by_campaign_identifier: "linkedin_ad_analytics_by_campaign_data"
 
-    linkedin_ads_schema: linkedin_source_integration_tests
+    linkedin_ads_schema: linkedin_source_integration_tests_1
 
 seeds:
   linkedin_source_integration_tests:
     linkedin_creative_history_data:
       +column_types:
         created_time: timestamp
         last_modified_time: timestamp
+        created_at: timestamp
+        last_modified_at: timestamp
         click_uri: "{{ 'string' if target.type in ('bigquery','spark','databricks') else 'varchar' }}"
+        intended_status: "{{ 'string' if target.type in ('bigquery','spark','databricks') else 'varchar' }}"
+        status: "{{ 'string' if target.type in ('bigquery','spark','databricks') else 'varchar' }}"
     linkedin_ad_analytics_by_creative_data:
       +column_types:
         day: timestamp

integration_tests/seeds/linkedin_ad_analytics_by_campaign_data.csv

@@ -1,21 +1,21 @@
-campaign_id,day,_fivetran_synced,action_clicks,ad_unit_clicks,approximate_unique_impressions,card_clicks,card_impressions,clicks,comment_likes,comments,company_page_clicks,conversion_value_in_local_currency,cost_in_local_currency,cost_in_usd,external_website_conversions,external_website_post_click_conversions,external_website_post_view_conversions,follows,full_screen_plays,impressions,landing_page_clicks,lead_generation_mail_contact_info_shares,lead_generation_mail_interested_clicks,likes,one_click_lead_form_opens,one_click_leads,opens,other_engagements,shares,text_url_clicks,total_engagements,video_completions,video_first_quartile_completions,video_midpoint_completions,video_starts,video_third_quartile_completions,video_views,viral_card_clicks,viral_card_impressions,viral_clicks,viral_comment_likes,viral_comments,viral_company_page_clicks,viral_external_website_conversions,viral_external_website_post_click_conversions,viral_external_website_post_view_conversions,viral_follows,viral_full_screen_plays,viral_impressions,viral_landing_page_clicks,viral_likes,viral_one_click_lead_form_opens,viral_one_click_leads,viral_other_engagements,viral_shares,viral_total_engagements,viral_video_completions,viral_video_first_quartile_completions,viral_video_midpoint_completions,viral_video_starts,viral_video_third_quartile_completions,viral_video_views
-148633856,2020-04-06 00:00:00,2021-04-06 12:21:11.686,,,0,,0,18,0,,,0,0,,0,,,,,,,0,0,,,,18,0,,,18,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-148633856,2020-04-03 00:00:00,2021-04-06 12:21:11.686,,,0,,0,18,0,,,0,0,,0,,,,,,,0,0,,,,18,0,,,18,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-148633856,2020-03-28 00:00:00,2021-04-06 12:21:11.686,1,,125,,0,51,0,0,,0,50.2,50.2,0,,0,,,125,1,0,0,,0,,51,0,0,,52,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
-148633856,2020-03-23 00:00:00,2021-04-06 12:21:11.686,1,,133,,0,52,0,0,,0,50.31,50.31,0,,0,,,143,1,0,0,,0,,52,0,0,,53,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
-148633856,2020-03-27 00:00:00,2021-04-06 12:21:11.686,0,,119,,0,59,0,0,,0,50.43,50.43,0,,0,,,126,0,0,0,,0,,59,0,0,,59,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
-148633856,2020-03-31 00:00:00,2021-04-06 12:21:11.686,2,,125,,0,85,0,0,,0,50.07,50.07,0,,0,,,125,2,0,0,,0,,85,0,0,,87,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
-148633856,2020-03-26 00:00:00,2021-04-06 12:21:11.686,2,0,67,0,0,37,0,0,0,0,30.12,30.12,0,0,0,0,0,75,3,0,0,0,0,0,37,0,0,1,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-03-24 00:00:00,2021-04-06 12:21:11.686,0,0,143,0,0,52,0,0,0,0,50.31,50.31,0,0,0,0,0,143,1,0,0,0,0,0,52,0,0,1,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-03-25 00:00:00,2021-04-06 12:21:11.686,1,0,124,0,0,56,0,0,0,0,50.05,50.05,0,0,0,0,0,125,2,0,0,0,0,0,56,0,0,1,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-03-29 00:00:00,2021-04-06 12:21:11.685,2,0,125,0,0,68,0,0,0,0,50.16,50.16,0,0,0,0,0,125,3,0,0,0,0,0,68,0,0,1,71,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-04-02 00:00:00,2021-04-06 12:21:11.686,0,0,122,0,0,74,0,0,0,0,48.8,48.8,0,0,0,0,0,122,2,0,0,0,0,0,74,0,0,2,76,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-04-01 00:00:00,2021-04-06 12:21:11.686,0,0,120,0,0,79,0,0,0,0,50.13,50.13,0,0,0,0,0,125,2,0,0,0,0,0,79,0,0,2,81,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-148633856,2020-03-30 00:00:00,2021-04-06 12:21:11.686,3,0,125,0,0,82,0,0,0,0,50.04,50.04,0,0,0,0,0,125,4,0,0,0,0,0,82,0,0,1,86,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-174096954,2021-05-25 00:00:00,2021-06-24 14:43:55.448,,,0,,0,23,0,,,0,0,,0,,,,,,,0,0,,,,23,0,,,23,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-174096954,2021-05-20 00:00:00,2021-06-19 14:38:29.197,,,0,,0,35,0,,,0,0,,0,,,,,,,0,0,,,,35,0,,,35,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-167276694,2020-10-22 00:00:00,2021-04-06 08:04:06.348,,,0,,0,38,0,,,0,0,,0,,,,,,,0,0,,,,38,0,,,38,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-174096954,2021-05-19 00:00:00,2021-06-18 14:46:41.820,,,0,,0,43,0,,,0,0,,0,,,,,,,0,0,,,,43,0,,,43,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-167276694,2020-11-03 00:00:00,2021-04-06 08:04:06.348,,,0,,0,15,0,,,0,0,,0,,,,,,,0,0,,,,15,0,,,15,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-165867684,2020-07-31 00:00:00,2021-04-06 17:16:49.608,,,0,,0,17,0,,,0,0,,0,,,,,,,0,0,,,,17,0,,,17,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
-165867684,2020-08-03 00:00:00,2021-04-06 17:16:49.608,,,0,,0,17,0,,,0,0,,0,,,,,,,0,0,,,,17,0,,,17,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+campaign_id,day,action_clicks,ad_unit_clicks,approximate_unique_impressions,card_clicks,card_impressions,clicks,comment_likes,comments,company_page_clicks,conversion_value_in_local_currency,cost_in_local_currency,cost_in_usd,external_website_conversions,external_website_post_click_conversions,external_website_post_view_conversions,follows,full_screen_plays,impressions,landing_page_clicks,lead_generation_mail_contact_info_shares,lead_generation_mail_interested_clicks,likes,one_click_lead_form_opens,one_click_leads,opens,other_engagements,shares,text_url_clicks,total_engagements,video_completions,video_first_quartile_completions,video_midpoint_completions,video_starts,video_third_quartile_completions,video_views,viral_card_clicks,viral_card_impressions,viral_clicks,viral_comment_likes,viral_comments,viral_company_page_clicks,viral_external_website_conversions,viral_external_website_post_click_conversions,viral_external_website_post_view_conversions,viral_follows,viral_full_screen_plays,viral_impressions,viral_landing_page_clicks,viral_likes,viral_one_click_lead_form_opens,viral_one_click_leads,viral_other_engagements,viral_shares,viral_total_engagements,viral_video_completions,viral_video_first_quartile_completions,viral_video_midpoint_completions,viral_video_starts,viral_video_third_quartile_completions,viral_video_views
+148633856,2020-04-06 00:00:00,,,0,,0,18,0,,,0,0,,0,,,,,,,0,0,,,,18,0,,,18,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+148633856,2020-04-03 00:00:00,,,0,,0,18,0,,,0,0,,0,,,,,,,0,0,,,,18,0,,,18,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+148633856,2020-03-28 00:00:00,1,,125,,0,51,0,0,,0,50.2,50.2,0,,0,,,125,1,0,0,,0,,51,0,0,,52,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-23 00:00:00,1,,133,,0,52,0,0,,0,50.31,50.31,0,,0,,,143,1,0,0,,0,,52,0,0,,53,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-27 00:00:00,0,,119,,0,59,0,0,,0,50.43,50.43,0,,0,,,126,0,0,0,,0,,59,0,0,,59,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-31 00:00:00,2,,125,,0,85,0,0,,0,50.07,50.07,0,,0,,,125,2,0,0,,0,,85,0,0,,87,0,,0,,,0,0,,0,,,0,,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0,,0
+148633856,2020-03-26 00:00:00,2,0,67,0,0,37,0,0,0,0,30.12,30.12,0,0,0,0,0,75,3,0,0,0,0,0,37,0,0,1,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-24 00:00:00,0,0,143,0,0,52,0,0,0,0,50.31,50.31,0,0,0,0,0,143,1,0,0,0,0,0,52,0,0,1,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-25 00:00:00,1,0,124,0,0,56,0,0,0,0,50.05,50.05,0,0,0,0,0,125,2,0,0,0,0,0,56,0,0,1,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-29 00:00:00,2,0,125,0,0,68,0,0,0,0,50.16,50.16,0,0,0,0,0,125,3,0,0,0,0,0,68,0,0,1,71,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-04-02 00:00:00,0,0,122,0,0,74,0,0,0,0,48.8,48.8,0,0,0,0,0,122,2,0,0,0,0,0,74,0,0,2,76,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-04-01 00:00:00,0,0,120,0,0,79,0,0,0,0,50.13,50.13,0,0,0,0,0,125,2,0,0,0,0,0,79,0,0,2,81,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+148633856,2020-03-30 00:00:00,3,0,125,0,0,82,0,0,0,0,50.04,50.04,0,0,0,0,0,125,4,0,0,0,0,0,82,0,0,1,86,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+174096954,2021-05-25 00:00:00,,,0,,0,23,0,,,0,0,,0,,,,,,,0,0,,,,23,0,,,23,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+174096954,2021-05-20 00:00:00,,,0,,0,35,0,,,0,0,,0,,,,,,,0,0,,,,35,0,,,35,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+167276694,2020-10-22 00:00:00,,,0,,0,38,0,,,0,0,,0,,,,,,,0,0,,,,38,0,,,38,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+174096954,2021-05-19 00:00:00,,,0,,0,43,0,,,0,0,,0,,,,,,,0,0,,,,43,0,,,43,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+167276694,2020-11-03 00:00:00,,,0,,0,15,0,,,0,0,,0,,,,,,,0,0,,,,15,0,,,15,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+165867684,2020-07-31 00:00:00,,,0,,0,17,0,,,0,0,,0,,,,,,,0,0,,,,17,0,,,17,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0
+165867684,2020-08-03 00:00:00,,,0,,0,17,0,,,0,0,,0,,,,,,,0,0,,,,17,0,,,17,,,,,,0,,,0,,,,,0,,0,,0,,0,0,,0,,,0,0,0,,,0