feature/remove-deprecated-fields

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
 
-  - label: ":bricks: Run Tests - Databricks"
+  - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:
@@ -69,5 +69,6 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks

CHANGELOG.md

@@ -1,6 +1,15 @@
-# dbt_recurly_source v0.UPDATE.UPDATE
+# dbt_recurly_source v0.3.0
+[PR #16](https://github.com/fivetran/dbt_recurly_source/pull/16) introduces the following updates:
 
- ## Under the Hood:
+## Breaking Changes
+- Removed unused fields from `stg_recurly__subscription_history`:
+  - `converted_at`
+  - `has_started_with_gift`
+- These fields were removed since they are planned for future deprecation in the Fivetran Connector and are not utilized in the downstream transformation package.
+
+
+[PR #14](https://github.com/fivetran/dbt_recurly_source/pull/14) includes the following updates:
+## Under the Hood:
 - Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job.
 - Updated the pull request [templates](/.github).
 

README.md

@@ -41,7 +41,7 @@ If you are **not** using the [Recurly transformation package](https://github.com
 ```yaml
 packages:
   - package: fivetran/recurly_source
-    version: [">=0.2.0", "<0.3.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.3.0", "<0.4.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 ### Step 3: Define database and schema variables
 By default, this package runs using your destination and the `recurly` schema. If this is not where your recurly data is (for example, if your recurly schema is named `recurly_fivetran`), add the following configuration to your root `dbt_project.yml` file:

dbt_project.yml

@@ -1,5 +1,5 @@
 name: "recurly_source"
-version: "0.2.1"
+version: "0.3.0"
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: recurly_source_integrations_tests
+      schema: recurly_source_integration_tests
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: recurly_source_integrations_tests
+      schema: recurly_source_integration_tests
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: recurly_source_integrations_tests
+      schema: recurly_source_integration_tests
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: recurly_source_integrations_tests
+      schema: recurly_source_integration_tests
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: recurly_source_integrations_tests
-      threads: 2
+      schema: recurly_source_integration_tests
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
-      type: databricks
+      type: databricks

integration_tests/dbt_project.yml

@@ -1,10 +1,10 @@
 name: 'recurly_source_integration_tests'
-version: '0.2.1'
+version: '0.3.0'
 profile: 'integration_tests'
 config-version: 2
 
 vars:
-  recurly_schema: recurly_source_integrations_tests
+  recurly_schema: recurly_source_integration_tests
   recurly_source:
     account_balance_history_identifier: "account_balance_history_data"
     account_history_identifier: "account_history_data"

integration_tests/seeds/subscription_history_data.csv

@@ -1,6 +1,6 @@
-_fivetran_synced,id,created_at,updated_at,activated_at,canceled_at,expires_at,account_id,plan_id,object,uuid,state,current_period_started_at,current_period_ends_at,current_term_started_at,current_term_ends_at,trial_started_at,trial_ends_at,remaining_billing_cycles,total_billing_cycles,renewal_billing_cycles,auto_renew,paused_at,remaining_pause_cycles,currency,unit_amount,quantity,add_ons_total,subtotal,collection_method,expiration_reason,started_with_gift
-2022-07-28 14:59:08,99b1bc4097d247c10c89dc71d41f8046,2019-03-26 09:28:17,2019-03-26 09:37:26,2019-03-26 09:28:17,2019-03-26 09:37:26,2019-03-26 09:37:26,4458762633d912919d7ff692cab13b69,1c85c5b774e9eabf29b9b439b8ee4584,subscription,d2033e289f1823674fd6cb332db6bf5f,expired,2019-03-26 09:28:17,2019-04-26 09:28:17,,2019-04-26 09:28:17,,,,,,true,,,USD,99.0,1,0.0,99.0,manual,canceled,false
-2022-07-28 14:59:08,b4110f7673aa3e121abf4ffaad157f62,2019-03-26 09:25:03,2019-03-26 09:27:19,2019-03-26 09:25:03,2019-03-26 09:27:19,2019-03-26 09:27:19,4458762633d912919d7ff692cab13b69,1c85c5b774e9eabf29b9b439b8ee4584,subscription,1f5496fb1572e464194d1f6d41a91707,expired,2019-03-26 09:25:03,2019-04-26 09:25:03,,2019-04-26 09:25:03,,,,,,true,,,USD,99.0,1,0.0,99.0,automatic,canceled,false
-2022-08-11 08:21:03,2900cfb1daef789cc21ea100ff6af6e9,2015-02-11 08:20:39,2022-08-11 08:20:39,2015-02-11 08:20:38,,,51bccf87e2571558b3da93688e17469f,1c85c5b774e9eabf29b9b439b8ee4584,subscription,81fe13285947f866c70b8102aa8f231d,active,2022-08-11 08:20:38,2022-09-11 08:20:38,2022-08-11 08:20:38,2022-09-11 08:20:38,,,0,1,1,true,,,USD,99.0,1,0.0,99.0,automatic,,false
-2022-07-28 14:59:08,2900cfb1daef789cc21ea100ff6af6e9,2015-02-11 08:20:39,2022-07-11 08:23:42,2015-02-11 08:20:38,,,51bccf87e2571558b3da93688e17469f,1c85c5b774e9eabf29b9b439b8ee4584,subscription,81fe13285947f866c70b8102aa8f231d,active,2022-07-11 08:20:38,2022-08-11 08:20:38,2022-07-11 08:20:38,2022-08-11 08:20:38,,,0,1,1,true,,,USD,99.0,1,0.0,99.0,automatic,,false
-2022-08-11 08:21:03,bcd3f1f36bd98895b5336e067049364b,2015-02-13 09:31:52,2022-07-20 09:32:53,2015-02-13 09:31:52,,,4458762633d912919d7ff692cab13b69,455b18e9a39e782da54de395edb6b566,subscription,971038e8a1a99aed7d9972903e0b5219,active,2022-07-20 09:31:52,2022-08-20 09:31:52,2022-07-20 09:31:52,2022-08-20 09:31:52,2015-02-13 09:31:52,2015-02-20 09:31:52,0,1,1,true,,,USD,2999.0,1,2597.0,5596.0,automatic,,false
+_fivetran_synced,id,created_at,updated_at,activated_at,canceled_at,expires_at,account_id,plan_id,object,uuid,state,current_period_started_at,current_period_ends_at,current_term_started_at,current_term_ends_at,trial_started_at,trial_ends_at,remaining_billing_cycles,total_billing_cycles,renewal_billing_cycles,auto_renew,paused_at,remaining_pause_cycles,currency,unit_amount,quantity,add_ons_total,subtotal,collection_method,expiration_reason
+2022-07-28 14:59:08,99b1bc4097d247c10c89dc71d41f8046,2019-03-26 09:28:17,2019-03-26 09:37:26,2019-03-26 09:28:17,2019-03-26 09:37:26,2019-03-26 09:37:26,4458762633d912919d7ff692cab13b69,1c85c5b774e9eabf29b9b439b8ee4584,subscription,d2033e289f1823674fd6cb332db6bf5f,expired,2019-03-26 09:28:17,2019-04-26 09:28:17,,2019-04-26 09:28:17,,,,,,true,,,USD,99.0,1,0.0,99.0,manual,canceled
+2022-07-28 14:59:08,b4110f7673aa3e121abf4ffaad157f62,2019-03-26 09:25:03,2019-03-26 09:27:19,2019-03-26 09:25:03,2019-03-26 09:27:19,2019-03-26 09:27:19,4458762633d912919d7ff692cab13b69,1c85c5b774e9eabf29b9b439b8ee4584,subscription,1f5496fb1572e464194d1f6d41a91707,expired,2019-03-26 09:25:03,2019-04-26 09:25:03,,2019-04-26 09:25:03,,,,,,true,,,USD,99.0,1,0.0,99.0,automatic,canceled
+2022-08-11 08:21:03,2900cfb1daef789cc21ea100ff6af6e9,2015-02-11 08:20:39,2022-08-11 08:20:39,2015-02-11 08:20:38,,,51bccf87e2571558b3da93688e17469f,1c85c5b774e9eabf29b9b439b8ee4584,subscription,81fe13285947f866c70b8102aa8f231d,active,2022-08-11 08:20:38,2022-09-11 08:20:38,2022-08-11 08:20:38,2022-09-11 08:20:38,,,0,1,1,true,,,USD,99.0,1,0.0,99.0,automatic,
+2022-07-28 14:59:08,2900cfb1daef789cc21ea100ff6af6e9,2015-02-11 08:20:39,2022-07-11 08:23:42,2015-02-11 08:20:38,,,51bccf87e2571558b3da93688e17469f,1c85c5b774e9eabf29b9b439b8ee4584,subscription,81fe13285947f866c70b8102aa8f231d,active,2022-07-11 08:20:38,2022-08-11 08:20:38,2022-07-11 08:20:38,2022-08-11 08:20:38,,,0,1,1,true,,,USD,99.0,1,0.0,99.0,automatic,
+2022-08-11 08:21:03,bcd3f1f36bd98895b5336e067049364b,2015-02-13 09:31:52,2022-07-20 09:32:53,2015-02-13 09:31:52,,,4458762633d912919d7ff692cab13b69,455b18e9a39e782da54de395edb6b566,subscription,971038e8a1a99aed7d9972903e0b5219,active,2022-07-20 09:31:52,2022-08-20 09:31:52,2022-07-20 09:31:52,2022-08-20 09:31:52,2015-02-13 09:31:52,2015-02-20 09:31:52,0,1,1,true,,,USD,2999.0,1,2597.0,5596.0,automatic,

macros/get_subscription_history_columns.sql

@@ -7,7 +7,6 @@
     {"name": "updated_at", "datatype": dbt.type_timestamp()},
     {"name": "activated_at", "datatype": dbt.type_timestamp()},
     {"name": "canceled_at", "datatype": dbt.type_timestamp()},
-    {"name": "converted_at", "datatype": dbt.type_timestamp()},
     {"name": "expires_at", "datatype": dbt.type_timestamp()},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "plan_id", "datatype": dbt.type_string()},
@@ -32,8 +31,7 @@
     {"name": "add_ons_total", "datatype": dbt.type_int()},
     {"name": "subtotal", "datatype": dbt.type_int()},
     {"name": "collection_method", "datatype": dbt.type_string()},
-    {"name": "expiration_reason", "datatype": dbt.type_string()},
-    {"name": "started_with_gift", "datatype": "boolean"} 
+    {"name": "expiration_reason", "datatype": dbt.type_string()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('recurly_subscription_pass_through_columns')) }}

models/src_recurly.yml

@@ -761,8 +761,6 @@ sources:
               Defaults to the plan's total_billing_cycles.
           - name: shipping_address_id
             description: Unique id assigned to shipping address.
-          - name: started_with_gift
-            description: Did the subscription start with a gift? Was it paid with at least one gift card credit?
           - name: state
             description: >
               The current state of the subscription.

models/stg_recurly.yml

@@ -733,8 +733,6 @@ models:
         description: >
           Default: "automatic"
           Enum: "automatic", "manual"
-      - name: converted_at
-        description: When the subscription has converted from trial.
       - name: created_at
         description: When the subscription was created.
       - name: currency
@@ -760,8 +758,6 @@ models:
         description: >
           Whether the subscription renews at the end of its term.
           Default: true
-      - name: has_started_with_gift
-        description: Did the subscription start with a gift? Was it paid with at least one gift card credit?
       - name: is_most_recent_record
         description: Is this the most recent record of this subscription? 
       - name: object

models/stg_recurly__subscription_history.sql

@@ -26,7 +26,6 @@ final as (
         add_ons_total,
         cast(canceled_at as {{ dbt.type_timestamp() }}) as canceled_at,
         collection_method,
-        cast(converted_at as {{ dbt.type_timestamp() }}) as converted_at,
         cast(created_at as {{ dbt.type_timestamp() }}) as created_at,
         currency,
         cast(current_period_ends_at as {{ dbt.type_timestamp() }}) as current_period_ended_at,
@@ -36,7 +35,6 @@ final as (
         expiration_reason,
         cast(expires_at as {{ dbt.type_timestamp() }}) as expires_at,
         auto_renew as has_auto_renew,
-        started_with_gift as has_started_with_gift,
         row_number() over (partition by id order by current_period_started_at desc) = 1 as is_most_recent_record,
         object,
         cast(paused_at as {{ dbt.type_timestamp() }}) as paused_at,