diff --git a/dbt_project.yml b/dbt_project.yml index c50bb518..ac7dba28 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'stripe' -version: '0.13.0' +version: '0.14.0' require-dbt-version: [">=1.3.0", "<2.0.0"] models: stripe: @@ -17,6 +17,7 @@ vars: charge: "{{ ref('stg_stripe__charge') }}" customer: "{{ ref('stg_stripe__customer') }}" dispute: "{{ ref('stg_stripe__dispute') }}" + discount: "{{ ref('stg_stripe__discount') }}" fee: "{{ ref('stg_stripe__fee') }}" invoice: "{{ ref('stg_stripe__invoice') }}" invoice_line_item: "{{ ref('stg_stripe__invoice_line_item') }}" @@ -25,6 +26,7 @@ vars: payment_method: "{{ ref('stg_stripe__payment_method') }}" payout: "{{ ref('stg_stripe__payout') }}" price_plan: "{{ ref('stg_stripe__price_plan') }}" + product: "{{ ref('stg_stripe__product') }}" refund: "{{ ref('stg_stripe__refund') }}" subscription: "{{ ref('stg_stripe__subscription') }}" transfer: "{{ ref('stg_stripe__transfer') }}" diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 1836e112..be39544b 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: stripe_integrations_tests_11 + schema: stripe_integrations_tests_14 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: stripe_integrations_tests_11 + schema: stripe_integrations_tests_14 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: stripe_integrations_tests_11 + schema: stripe_integrations_tests_14 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: stripe_integrations_tests_11 + schema: stripe_integrations_tests_14 threads: 8 databricks: catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: stripe_integrations_tests_11 + schema: stripe_integrations_tests_14 threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index edec8013..45efcea5 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'stripe_integration_tests' -version: '0.13.0' +version: '0.14.0' profile: 'integration_tests' @@ -10,7 +10,7 @@ models: +schema: "stripe_{{ var('directed_schema','dev') }}" vars: - stripe_schema: stripe_integrations_tests_11 + stripe_schema: stripe_integrations_tests_14 stripe_source: stripe_account_identifier: "account_data" stripe_group_identifier: "group_data" @@ -18,6 +18,7 @@ vars: stripe_card_identifier: "card_data" stripe_charge_identifier: "charge_data" stripe_customer_identifier: "customer_data" + stripe_discount_identifier: "discount_data" stripe_dispute_identifier: "dispute_data" stripe_fee_identifier: "fee_data" stripe_invoice_identifier: "invoice_data" @@ -28,6 +29,7 @@ vars: stripe_payout_identifier: "payout_data" stripe_plan_identifier: "plan_data" stripe_price_identifier: "price_data" + stripe_product_identifier: "product_data" stripe_refund_identifier: "refund_data" stripe_subscription_history_identifier: "subscription_history_data" stripe_subscription_identifier: "subscription_data" @@ -47,6 +49,7 @@ seeds: voided_at: timestamp canceled_at: timestamp invoice_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}" + end: timestamp balance_transaction_data: +column_types: available_on: timestamp @@ -61,6 +64,15 @@ seeds: +column_types: destination_bank_account_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}" destination_card_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}" + discount_data: + +enabled: "{{ true if target.type not in ('snowflake', 'postgres') else false }}" + discount_data_snowflake: + +alias: discount_data + +enabled: "{{ true if target.type == 'snowflake' else false }}" + discount_data_postgres: + +alias: discount_data + +enabled: "{{ true if target.type == 'postgres' else false }}" + dispatch: - macro_namespace: dbt_utils diff --git a/integration_tests/seeds/discount_data.csv b/integration_tests/seeds/discount_data.csv new file mode 100644 index 00000000..04ab5445 --- /dev/null +++ b/integration_tests/seeds/discount_data.csv @@ -0,0 +1,11 @@ +id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,end,invoice_id,invoice_item_id,promotion_code,start,subscription_id +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a1b99DZlKRVG8Gn5eb6dced,2024-03-19 12:32:16.476000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OvvZQDZlKRVG8GnByl08x0P,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1cd03bDZlKRVG8Gn694a7916,2024-02-19 12:32:46.371000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OlPk3DZlKRVG8GnkWTFA0go,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_15a605DZlKRVG8Gncf06fd6b,2024-02-16 18:45:08.587000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OaAyXDZlKRVG8GnQo19CbEo,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1d3987DZlKRVG8Gnb404c3ca,2023-11-19 08:18:47.498000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1432ddDZlKRVG8Gna92778f6,2024-04-19 07:45:08.824000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1P7ALFDZlKRVG8GnTUCtI40b,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a71eaDZlKRVG8Gn3909c3a4,2024-02-16 18:33:16.340000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_146059DZlKRVG8Gn63a45e70,2024-05-19 12:34:05.168000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1PI2dvDZlKRVG8Gn4I34kEqR,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,SUBSCRIPTION,sub_1LjcrrDZlKRVG8Gnv1Bjosun,2024-05-19 06:31:36.758000,,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1NcPsdDZlKRVG8GnU0gsnD2b,CUSTOMER,cus_OPEJRxhitWa90P,2023-10-17 14:18:42.776000,,,,ioLCPhMr,,cus_OPEJRxhitWa90P,,,,,2023-08-07 09:37:39.000000, +di_1NcPt3DZlKRVG8GnqM2ZJTve,CUSTOMER,cus_OPEKG00sHdsUs8,2024-06-06 12:32:41.114000,,,,Oodskd5H,,cus_OPEKG00sHdsUs8,,,,,2023-08-07 09:38:05.000000, \ No newline at end of file diff --git a/integration_tests/seeds/discount_data_postgres.csv b/integration_tests/seeds/discount_data_postgres.csv new file mode 100644 index 00000000..1519ee76 --- /dev/null +++ b/integration_tests/seeds/discount_data_postgres.csv @@ -0,0 +1,11 @@ +id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,""end"",invoice_id,invoice_item_id,promotion_code,""start"",subscription_id +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a1b99DZlKRVG8Gn5eb6dced,2024-03-19 12:32:16.476000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1OvvZQDZlKRVG8GnByl08x0P,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1cd03bDZlKRVG8Gn694a7916,2024-02-19 12:32:46.371000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",in_1OlPk3DZlKRVG8GnkWTFA0go,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_15a605DZlKRVG8Gncf06fd6b,2024-02-16 18:45:08.587000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1OaAyXDZlKRVG8GnQo19CbEo,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1d3987DZlKRVG8Gnb404c3ca,2023-11-19 08:18:47.498000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1432ddDZlKRVG8Gna92778f6,2024-04-19 07:45:08.824000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1P7ALFDZlKRVG8GnTUCtI40b,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a71eaDZlKRVG8Gn3909c3a4,2024-02-16 18:33:16.340000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_146059DZlKRVG8Gn63a45e70,2024-05-19 12:34:05.168000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1PI2dvDZlKRVG8Gn4I34kEqR,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,SUBSCRIPTION,sub_1LjcrrDZlKRVG8Gnv1Bjosun,2024-05-19 06:31:36.758000,,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1NcPsdDZlKRVG8GnU0gsnD2b,CUSTOMER,cus_OPEJRxhitWa90P,2023-10-17 14:18:42.776000,,,,ioLCPhMr,,cus_OPEJRxhitWa90P,"2022-09-27 05:50:07.000000",,,,"2023-08-07 09:37:39.000000", +di_1NcPt3DZlKRVG8GnqM2ZJTve,CUSTOMER,cus_OPEKG00sHdsUs8,2024-06-06 12:32:41.114000,,,,Oodskd5H,,cus_OPEKG00sHdsUs8,,,,,"2023-08-07 09:38:05.000000", \ No newline at end of file diff --git a/integration_tests/seeds/discount_data_snowflake.csv b/integration_tests/seeds/discount_data_snowflake.csv new file mode 100644 index 00000000..14f18e10 --- /dev/null +++ b/integration_tests/seeds/discount_data_snowflake.csv @@ -0,0 +1,11 @@ +id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,"""END""",invoice_id,invoice_item_id,promotion_code,"""START""",subscription_id +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a1b99DZlKRVG8Gn5eb6dced,2024-03-19 12:32:16.476000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OvvZQDZlKRVG8GnByl08x0P,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1cd03bDZlKRVG8Gn694a7916,2024-02-19 12:32:46.371000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OlPk3DZlKRVG8GnkWTFA0go,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_15a605DZlKRVG8Gncf06fd6b,2024-02-16 18:45:08.587000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OaAyXDZlKRVG8GnQo19CbEo,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1d3987DZlKRVG8Gnb404c3ca,2023-11-19 08:18:47.498000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1432ddDZlKRVG8Gna92778f6,2024-04-19 07:45:08.824000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1P7ALFDZlKRVG8GnTUCtI40b,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a71eaDZlKRVG8Gn3909c3a4,2024-02-16 18:33:16.340000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_146059DZlKRVG8Gn63a45e70,2024-05-19 12:34:05.168000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1PI2dvDZlKRVG8Gn4I34kEqR,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1LjcrrDZlKRVG8GnP7REhm8t,SUBSCRIPTION,sub_1LjcrrDZlKRVG8Gnv1Bjosun,2024-05-19 06:31:36.758000,,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun +di_1NcPsdDZlKRVG8GnU0gsnD2b,CUSTOMER,cus_OPEJRxhitWa90P,2023-10-17 14:18:42.776000,,,,ioLCPhMr,,cus_OPEJRxhitWa90P,,,,,2023-08-07 09:37:39.000000, +di_1NcPt3DZlKRVG8GnqM2ZJTve,CUSTOMER,cus_OPEKG00sHdsUs8,2024-06-06 12:32:41.114000,,,,Oodskd5H,,cus_OPEKG00sHdsUs8,,,,,2023-08-07 09:38:05.000000, \ No newline at end of file diff --git a/integration_tests/tests/consistency/consistency_line_item_enhanced copy.sql b/integration_tests/tests/consistency/consistency_line_item_enhanced copy.sql new file mode 100644 index 00000000..e98eaf14 --- /dev/null +++ b/integration_tests/tests/consistency/consistency_line_item_enhanced copy.sql @@ -0,0 +1,31 @@ +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +with prod as ( + select * + from {{ target.schema }}_stripe_prod.stripe__line_item_enhanced +), + +dev as ( + select * + from {{ target.schema }}_stripe_dev.stripe__line_item_enhanced +), + +final as ( + -- test will fail if any rows from prod are not found in dev + (select * from prod + except distinct + select * from dev) + + union all -- union since we only care if rows are produced + + -- test will fail if any rows from dev are not found in prod + (select * from dev + except distinct + select * from prod) + ) + +select * +from final \ No newline at end of file diff --git a/integration_tests/tests/consistency/consistency_line_item_enhanced.sql b/integration_tests/tests/consistency/consistency_line_item_enhanced.sql new file mode 100644 index 00000000..0eb51584 --- /dev/null +++ b/integration_tests/tests/consistency/consistency_line_item_enhanced.sql @@ -0,0 +1,21 @@ +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +-- this test is to make sure the rows counts are the same between versions +with prod as ( + select count(*) as prod_rows + from {{ target.schema }}_stripe_prod.stripe__line_item_enhanced +), + +dev as ( + select count(*) as dev_rows + from {{ target.schema }}_stripe_dev.stripe__line_item_enhanced +) + +-- test will return values and fail if the row counts don't match +select * +from prod +join dev + on prod.prod_rows != dev.dev_rows \ No newline at end of file diff --git a/integration_tests/tests/integrity/integrity_line_item_enhanced.sql b/integration_tests/tests/integrity/integrity_line_item_enhanced.sql new file mode 100644 index 00000000..e46df4f5 --- /dev/null +++ b/integration_tests/tests/integrity/integrity_line_item_enhanced.sql @@ -0,0 +1,21 @@ +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +-- this test is to make sure there is no fanout between the spine and the daily_overview +with stg_invoice_line_item as ( + select count(*) as line_item_count + from {{ target.schema }}_stripe_dev.stg_stripe__invoice_line_item +), + +line_item_enhanced as ( + select count(*) as daily_overview_count + from {{ target.schema }}_stripe_dev.stripe__line_item_enhanced +) + +-- test will return values and fail if the row counts don't match +select * +from stg_invoice_line_item +join line_item_enhanced + on stg_invoice_line_item.line_item_count != line_item_enhanced.daily_overview_count \ No newline at end of file diff --git a/models/common_data_models/stripe__common_data_models.yml b/models/common_data_models/stripe__common_data_models.yml new file mode 100644 index 00000000..dc5b6685 --- /dev/null +++ b/models/common_data_models/stripe__common_data_models.yml @@ -0,0 +1,79 @@ +version: 2 + +models: + - name: stripe__line_item_enhanced + description: Add description + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - header_id + - line_item_id + columns: + - name: header_id + description: Id of corresponding invoice. + - name: line_item_id + description: Invoice line item id. + - name: line_item_index + description: Unique index of each line item id for each invoice. + - name: record_type + description: header or line_item + - name: created_at + description: When the invoice was created + - name: header_status + description: Status of the invoice. + - name: billing_type + description: Type of invoice line item. + - name: currency + description: Currency used in the invoice. + - name: product_id + description: Product ID + - name: product_name + description: Product name + - name: product_type + description: Product type + - name: transaction_type + description: Balance transaction type + - name: quantity + description: The number of units of this line item. + - name: unit_amount + description: Amount per unit. + - name: discount_amount + description: The discount amount associated with this invoice. + - name: tax_amount + description: The tax amount associated with this invoice. + - name: total_amount + description: Invoice total + - name: payment_id + description: Payment intent ID + - name: payment_method + description: Payment method type + - name: payment_method_id + description: Payment method ID + - name: payment_at + description: Charge created date + - name: fee_amount + description: Balance transaction fee + - name: refund_amount + description: Refund amount associated with this invoice. + - name: subscription_id + description: Subscription associated with this invoice. + - name: subscription_period_started_at + description: Subscription's current period start + - name: subscription_period_ended_at + description: Subscription's current period end + - name: subscription_status + description: Subscription status + - name: customer_id + description: Customer associated with this invoice. + - name: customer_level + description: Whether 'account' or 'customer'. For Stripe, the level is 'customer'. + - name: customer_name + description: Customer name + - name: customer_company + description: Associated account's company if applicable. + - name: customer_email + description: Customer email + - name: customer_city + description: Customer city + - name: customer_country + description: Customer country \ No newline at end of file diff --git a/models/common_data_models/stripe__line_item_enhanced.sql b/models/common_data_models/stripe__line_item_enhanced.sql new file mode 100644 index 00000000..b59d916e --- /dev/null +++ b/models/common_data_models/stripe__line_item_enhanced.sql @@ -0,0 +1,261 @@ +{{ config(enabled=var('stripe__using_invoices', True)) }} + +with invoice_line_item as ( + + select * + from {{ var('invoice_line_item') }} + +), invoice as ( + + select * + from {{ var('invoice') }} + +{% if var('stripe__using_subscriptions', True) %} +), subscription as ( + + select * + from {{ var('subscription') }} + +), price_plan as ( + + select * + from {{ var('price_plan') }} + +), product as ( + + select * + from {{ var('product') }} + +{% endif %} + +), payment_intent as ( + + select * + from {{ var('payment_intent') }} + +), payment_method as ( + + select * + from {{ var('payment_method')}} + +), fee as ( + + select * + from {{ var('fee') }} + +), account as ( + + select * + from {{ var('account') }} + +), balance_transaction as ( + + select * + from {{ var('balance_transaction') }} + +), charge as ( + + select * + from {{ var('charge') }} + +), discount as ( + + select + invoice_id, + sum(amount) as total_discount_amount + from {{ var('discount') }} + group by 1 + +), refund as ( + + select * + from {{ var('refund') }} + +), customer as ( + + select * + from {{ var('customer') }} + +), enhanced as ( + + select + invoice_line_item.invoice_id as header_id, + cast(invoice_line_item.invoice_line_item_id as {{ dbt.type_string() }}) as line_item_id, + row_number() over (partition by invoice_line_item.invoice_id order by invoice_line_item.amount desc) as line_item_index, + invoice.created_at as created_at, + cast(invoice_line_item.currency as {{ dbt.type_string() }}) as currency, + cast(invoice.status as {{ dbt.type_string() }}) as header_status, + cast(price_plan.product_id as {{ dbt.type_string() }}) as product_id, -- The ID of the product this price is associated with. https://docs.stripe.com/api/invoices/line_item#invoice_line_item_object-price-product + cast(product.name as {{ dbt.type_string() }}) as product_name, + cast(balance_transaction.type as {{ dbt.type_string() }}) as transaction_type, + cast(invoice_line_item.type as {{ dbt.type_string() }}) as billing_type, + cast(product.type as {{ dbt.type_string() }}) as product_type, + cast(invoice_line_item.quantity as {{ dbt.type_numeric() }}) as quantity, + cast((invoice_line_item.amount/invoice_line_item.quantity) as {{ dbt.type_numeric() }}) as unit_amount, + cast(discount.total_discount_amount as {{ dbt.type_numeric() }}) as discount_amount, + cast(invoice.tax as {{ dbt.type_numeric() }}) as tax_amount, + cast(invoice.total as {{ dbt.type_numeric() }}) as total_amount, + cast(payment_intent.payment_intent_id as {{ dbt.type_string() }}) as payment_id, + cast(payment_method.payment_method_id as {{ dbt.type_string() }}) as payment_method_id, + cast(payment_method.type as {{ dbt.type_string() }}) as payment_method, + cast(charge.created_at as {{ dbt.type_timestamp() }}) as payment_at, + cast(balance_transaction.fee as {{ dbt.type_numeric() }}) as fee_amount, + cast(refund.amount as {{ dbt.type_numeric() }}) as refund_amount, + cast(invoice.subscription_id as {{ dbt.type_string() }}) as subscription_id, + + {% if var('stripe__using_subscriptions', True) %} + + cast(subscription.current_period_start as {{ dbt.type_timestamp() }}) as subscription_period_started_at, + cast(subscription.current_period_end as {{ dbt.type_timestamp() }}) as subscription_period_ended_at, + cast(subscription.status as {{ dbt.type_string() }}) as subscription_status, + + {% else %} + + cast(null as {{ dbt.type_timestamp() }}) as subscription_period_started_at, + cast(null as {{ dbt.type_timestamp() }}) as subscription_period_ended_at, + cast(null as {{ dbt.type_string() }}) as subscription_status, + + {% endif %} + + cast(invoice.customer_id as {{ dbt.type_string() }}) as customer_id, + 'customer' as customer_level, + cast(customer.customer_name as {{ dbt.type_string() }}) as customer_name, + cast(connected_account.company_name as {{ dbt.type_string() }}) as customer_company, + cast(customer.email as {{ dbt.type_string() }}) as customer_email, + cast(customer.customer_address_city as {{ dbt.type_string() }}) as customer_city, + cast(customer.customer_address_country as {{ dbt.type_string() }}) as customer_country + + from invoice_line_item + + left join invoice + on invoice.invoice_id = invoice_line_item.invoice_id + + left join charge + on invoice.charge_id = charge.charge_id + and invoice.invoice_id = charge.invoice_id + + left join balance_transaction + on charge.balance_transaction_id = balance_transaction.balance_transaction_id + + left join discount + on invoice.invoice_id = discount.invoice_id + + left join refund + on balance_transaction.balance_transaction_id = refund.balance_transaction_id + + left join account connected_account + on balance_transaction.connected_account_id = connected_account.account_id + + left join payment_intent + on charge.payment_intent_id = payment_intent.payment_intent_id + + left join payment_method + on charge.payment_method_id = payment_method.payment_method_id + + left join customer + on invoice.customer_id = customer.customer_id + + {% if var('stripe__using_subscriptions', True) %} + + left join subscription + on invoice.subscription_id = subscription.subscription_id + + left join price_plan + + {% if var('stripe__using_price', stripe_source.does_table_exist('price')) %} + on invoice_line_item.price_id = price_plan.price_plan_id + {% else %} + on invoice_line_item.plan_id = price_plan.price_plan_id + {% endif %} + + left join product + on price_plan.product_id = product.product_id + {% endif %} + +), final as ( + +-- invoice_line_item_level + select + header_id, + line_item_id, + line_item_index, + 'line_item' as record_type, + created_at, + currency, + header_status, + product_id, + product_name, + cast(null as {{ dbt.type_string() }}) as transaction_type, + billing_type, + product_type, + quantity, + unit_amount, + cast(null as {{ dbt.type_numeric() }}) as discount_amount, + cast(null as {{ dbt.type_numeric() }}) as tax_amount, + total_amount, + payment_id, + payment_method_id, + payment_method, + payment_at, + cast(null as {{ dbt.type_numeric() }}) as fee_amount, + cast(null as {{ dbt.type_numeric() }}) as refund_amount, + subscription_id, + subscription_period_started_at, + subscription_period_ended_at, + subscription_status, + customer_id, + customer_level, + customer_name, + customer_company, + customer_email, + customer_city, + customer_country + from enhanced + + union all + + -- create records for fields only pertinent at the invoice level + select + header_id, + cast(null as {{ dbt.type_string() }}) as line_item_id, + line_item_index, + 'header' as record_type, + created_at, + currency, + header_status, + cast(null as {{ dbt.type_string() }}) as product_id, + cast(null as {{ dbt.type_string() }}) as product_name, + transaction_type, + cast(null as {{ dbt.type_string() }}) as billing_type, + cast(null as {{ dbt.type_string() }}) as product_type, + cast(null as {{ dbt.type_float() }}) as quantity, + cast(null as {{ dbt.type_float() }}) as unit_amount, + discount_amount, + tax_amount, + cast(null as {{ dbt.type_float() }}) as total_amount, + payment_id, + payment_method_id, + payment_method, + payment_at, + fee_amount, + refund_amount, + subscription_id, + subscription_period_started_at, + subscription_period_ended_at, + subscription_status, + customer_id, + customer_level, + customer_name, + customer_company, + customer_email, + customer_city, + customer_country + from enhanced + where line_item_index = 1 + and discount_amount is not null or tax_amount is not null or fee_amount is not null or refund_amount is not null +) + +select * +from final + + diff --git a/packages.yml b/packages.yml index d16b8066..4ed751b2 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,7 @@ packages: -- package: fivetran/stripe_source - version: [">=0.11.0", "<0.12.0"] \ No newline at end of file +# - package: fivetran/stripe_source +# version: [">=0.11.0", "<0.12.0"] + +- git: https://github.com/fivetran/dbt_stripe_source.git + revision: feature/normalized-billing-model + warn-unpinned: false \ No newline at end of file