From 55c52f6aabb070d94c6e72fd03ba9aa552ea229b Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 15 Dec 2022 10:32:46 -0800 Subject: [PATCH 01/13] some docs --- macros/get_order_columns.sql | 2 -- models/src_shopify.yml | 33 +++++++++++++++++++++++++++++- models/stg_shopify.yml | 35 ++++++++++++++++++++++++++++++-- models/stg_shopify__customer.sql | 17 ++++++++++------ models/stg_shopify__order.sql | 11 +++++----- 5 files changed, 81 insertions(+), 17 deletions(-) diff --git a/macros/get_order_columns.sql b/macros/get_order_columns.sql index c3b9a16..abae5fe 100644 --- a/macros/get_order_columns.sql +++ b/macros/get_order_columns.sql @@ -20,7 +20,6 @@ {"name": "total_weight", "datatype": dbt.type_numeric()}, {"name": "total_tip_received", "datatype": dbt.type_float()}, {"name": "landing_site_base_url", "datatype": dbt.type_string()}, - {"name": "landing_site_ref", "datatype": dbt.type_string()}, {"name": "location_id", "datatype": dbt.type_numeric()}, {"name": "name", "datatype": dbt.type_string()}, {"name": "note", "datatype": dbt.type_string()}, @@ -80,7 +79,6 @@ {"name": "checkout_id", "datatype": dbt.type_int()}, {"name": "client_details_user_agent", "datatype": dbt.type_string()}, {"name": "customer_locale", "datatype": dbt.type_string()}, - {"name": "device_id", "datatype": dbt.type_string()}, {"name": "order_status_url", "datatype": dbt.type_string()}, {"name": "presentment_currency", "datatype": dbt.type_string()}, {"name": "confirmed", "datatype": "boolean"} diff --git a/models/src_shopify.yml b/models/src_shopify.yml index 7ab0171..d00af69 100644 --- a/models/src_shopify.yml +++ b/models/src_shopify.yml @@ -143,6 +143,21 @@ sources: description: The date and time (ISO 8601 format) when the order was last modified. - name: user_id description: The ID of the user logged into Shopify POS who processed the order, if applicable. + - name: checkout_token + description: A unique value when referencing the checkout that's associated with the order. + - name: confirmed + description: Whether inventory has been reserved for the order. + - name: customer_locale + description: A two-letter or three-letter language code, optionally followed by a region modifier. + - name: checkout_id + description: ID of the order's checkout. + - name: order_status_url + description: The URL pointing to the order status web page, if applicable. + - name: _fivetran_deleted + description: "{{ doc('_fivetran_deleted') }}" + - name: total_tip_received + description: The sum of all the tips in the order in the shop currency. + - name: customer identifier: "{{ var('shopify_customer_identifier', 'customer') }}" description: Each record represents a customer in Shopify. @@ -150,7 +165,7 @@ sources: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: accepts_marketing - description: Whether the customer has consented to receive marketing material via email. + description: Whether the customer has consented to receive marketing material via email. Deprecated and will be coalesced with `email_marketing_consent_state`. - name: created_at description: The date and time when the customer was created. - name: default_address_id @@ -177,6 +192,22 @@ sources: description: The date and time when the customer information was last updated. - name: verified_email description: Whether the customer has verified their email address. + - name: email_marketing_consent_state + description: The current email marketing state for the customer. New version of `accepts_marketing` field. + - name: email_marketing_consent_opt_in_level + description: The marketing subscription opt-in level, as described in the M3AAWG Sender Best Common Practices, that the customer gave when they consented to receive marketing material by email. New version of `marketing_opt_in_level` field. + - name: email_marketing_consent_consent_updated_at + description: The date and time when the customer consented to receive marketing material by email. If no date is provided, then the date and time when the consent information was sent is used. New version of `accepts_marketing_updated_at` field. + - name: accepts_marketing_updated_at + description: Deprecated. The package will coalesce with `email_marketing_consent_consent_updated_at`. + - name: marketing_opt_in_level + description: Deprecated. The package will coalesce with `email_marketing_consent_opt_in_level`. + - name: lifetime_duration + description: The amount of time since the customer was first added to the store. + - name: _fivetran_deleted + description: "{{ doc('_fivetran_deleted') }}" + - name: currency + description: The three-letter code (ISO 4217 format) for the currency that the customer used when they paid for their last order. Defaults to the shop currency. Returns the shop currency for test orders. - name: order_line identifier: "{{ var('shopify_order_line_identifier', 'order_line') }}" description: Each record represents a line item for an order in Shopify. diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index 5e6357a..7b899c9 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -11,8 +11,14 @@ models: columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - - name: has_accepted_marketing - description: Whether the customer has consented to receive marketing material via email. + - name: marketing_consent_state + description: > + Field indicating if the customer has consented to receive marketing material via email. Coalescing of the new `email_marketing_consent_state` + field and the deprected `accepts_marketing` field. Records with the old field will be marked with '(legacy)'. + - name: marketing_opt_in_level + description: The marketing subscription opt-in level, as described in the M3AAWG Sender Best Common Practices, that the customer gave when they consented to receive marketing material by email. + - name: marketing_consent_updated_at + description: The date and time when the customer consented to receive marketing material by email. If no date is provided, then the date and time when the consent information was sent is used. - name: created_at description: The date and time when the customer was created. - name: default_address_id @@ -39,6 +45,10 @@ models: description: The date and time when the customer information was last updated. - name: is_verified_email description: Whether the customer has verified their email address. + - name: lifetime_duration + description: The amount of time since the customer was first added to the store. + - name: currency + description: The three-letter code (ISO 4217 format) for the currency that the customer used when they paid for their last order. Defaults to the shop currency. Returns the shop currency for test orders. - name: stg_shopify__order_line_refund description: Each record represents a line item from an order in Shopify. @@ -126,6 +136,10 @@ models: description: "{{ doc('_fivetran_synced') }}" - name: app_id description: The ID of the app that created the order. + - name: checkout_id + description: ID of the order's checkout. + - name: order_status_url + description: The URL pointing to the order status web page, if applicable. - name: billing_address_address_1 description: The street address of the billing address. - name: billing_address_address_2 @@ -260,6 +274,23 @@ models: description: The index associated with the order. - name: pre_tax_price description: The total pre tax price of the order. + - name: checkout_token + description: A unique value when referencing the checkout that's associated with the order. + - name: is_inventory_confirmed + description: Whether inventory has been reserved for the order. + - name: customer_locale + description: A two-letter or three-letter language code, optionally followed by a region modifier. + - name: is_deleted + description: "{{ doc('_fivetran_deleted') }}" + - name: total_tip_received + description: The sum of all the tips in the order in the shop currency. + - name: client_details_user_agent + description: Details of the browsing client, including software and operating versions. + - name: total_tax_set + - name: total_shipping_price_set + - name: total_discounts_set + - name: subtotal_price_set + - name: presentment_currency - name: stg_shopify__product description: Each record represents a product in Shopify. diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql index 8a564fa..54d9740 100644 --- a/models/stg_shopify__customer.sql +++ b/models/stg_shopify__customer.sql @@ -28,24 +28,29 @@ fields as ( final as ( select - accepts_marketing as has_accepted_marketing, - email_marketing_consent_state, - default_address_id, + id as customer_id, email, first_name, - id as customer_id, last_name, orders_count, + default_address_id, phone, state as account_state, tax_exempt as is_tax_exempt, total_spent, verified_email as is_verified_email, note, - coalesce(marketing_opt_in_level, email_marketing_consent_opt_in_level) as marketing_opt_in_level, lifetime_duration, currency, - cast( coalesce(accepts_marketing_updated_at, email_marketing_consent_consent_updated_at) as {{ dbt.type_timestamp() }}) as accepts_marketing_updated_at, + case + when email_marketing_consent_state is null then + case + when accepts_marketing is null then null + when accepts_marketing then 'subscribed (legacy)' + else 'not_subscribed (legacy)' end + else lower(email_marketing_consent_state) end as marketing_consent_state, + lower(coalesce(email_marketing_consent_opt_in_level, marketing_opt_in_level)) as marketing_opt_in_level, + cast( coalesce(email_marketing_consent_consent_updated_at, accepts_marketing_updated_at) as {{ dbt.type_timestamp() }}) as marketing_consent_updated_at, cast(updated_at as {{ dbt.type_timestamp() }}) as updated_at, cast(created_at as {{ dbt.type_timestamp() }}) as created_at, cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced, diff --git a/models/stg_shopify__order.sql b/models/stg_shopify__order.sql index 9480adf..ee065df 100644 --- a/models/stg_shopify__order.sql +++ b/models/stg_shopify__order.sql @@ -47,7 +47,6 @@ final as ( total_weight, total_tip_received, landing_site_base_url, - landing_site_ref, location_id, name, note, @@ -99,23 +98,23 @@ final as ( shipping_address_province, shipping_address_province_code, shipping_address_zip, - test as is_test_order, + {# test as is_test_order, #} token, - _fivetran_synced, - _fivetran_deleted, + _fivetran_deleted as is_deleted, app_id, checkout_id, client_details_user_agent, customer_locale, - device_id, order_status_url, presentment_currency, - confirmed, + confirmed as is_inventory_confirmed, + cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced, source_relation {{ fivetran_utils.fill_pass_through_columns('order_pass_through_columns') }} from fields + where not coalesce(test, false) ) select * From 7d46536be1b0f9c610cdcb520c609bc2a5b63f41 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 28 Dec 2022 16:58:51 -0800 Subject: [PATCH 02/13] docs and style ipdates --- CHANGELOG.md | 37 +++++++++++++++ integration_tests/dbt_project.yml | 46 +++++++++---------- .../seeds/shopify_order_adjustment_data.csv | 10 ++-- .../seeds/shopify_order_data.csv | 6 +-- .../seeds/shopify_product_variant_data.csv | 10 ++-- .../seeds/shopify_refund_data.csv | 10 ++-- .../seeds/shopify_transaction_data.csv | 10 ++-- models/src_shopify.yml | 22 +++++++++ models/stg_shopify__order.sql | 8 ++-- models/stg_shopify__order_line.sql | 3 +- 10 files changed, 109 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01fb853..0fd0d82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,41 @@ ## 🎉 Documentation and Feature Updates - Updated README documentation updates for easier navigation and setup of the dbt package - Included `shopify_[source_table_name]_identifier` variable for additional flexibility within the package when source tables are named differently. +- The following fields have been added to their respective staging models: + - `stg_shopify__order`: + - `total_discounts_set` + - `total_line_items_price_set` + - `total_price_usd` + - `total_price_set` + - `total_tax_set` + - `total_tip_received` + - `is_deleted` + - `app_id` + - `checkout_id` + - `client_details_user_agent` + - `customer_locale` + - `order_status_url` + - `presentment_currency` + - `is_inventory_confirmed` + - `stg_shopify__customer`: + - `note` + - `lifetime_duration` + - `currency` + - `marketing_consent_state` (coalescing of `email_marketing_consent_state` and deprecated `accepts_marketing` field) + - `marketing_opt_in_level` (coalescing of `email_marketing_consent_opt_in_level` and deprecated `marketing_opt_in_level` field) + - `marketing_consent_updated_at` (coalescing of `email_marketing_consent_consent_updated_at` and deprecated `accepts_marketing_updated_at` field) + - `stg_shopify__order_line_refund`: + - `subtotal_set` + - `total_tax_set` + - `stg_shopify__order_line`: + - `pre_tax_price_set` + - `price_set` + - `tax_code` + - `total_discount_set` + - `variant_title` + - `variant_inventory_management` + - `vendor` + - `properties` ## 🚨 Breaking Changes 🚨: [PR #36](https://github.com/fivetran/dbt_shopify_source/pull/36) includes the following breaking changes: @@ -39,6 +74,8 @@ - `dbt.current_timestamp_backcompat` - `dbt.current_timestamp_in_utc_backcompat` - Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`. +- The following fields have been removed as they have been deprecated: + - ## Updates: [PR #40](https://github.com/fivetran/dbt_shopify_source/pull/40) includes the following changes: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 22d0668..0106a2e 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -15,30 +15,28 @@ vars: shopify_product_variant_identifier: "shopify_product_variant_data" shopify_refund_identifier: "shopify_refund_data" shopify_transaction_identifier: "shopify_transaction_data" - shopify_schema: shopify_source_integration_tests_1 - - abandoned_checkout_source: "{{ ref('shopify_abandoned_checkout_data') }}" - collection_product_source: "{{ ref('shopify_collection_product_data') }}" - collection_source: "{{ ref('shopify_collection_data') }}" - customer_tag_source: "{{ ref('shopify_customer_tag_data') }}" - discount_code_source: "{{ ref('shopify_discount_code_data') }}" - fulfillment_source: "{{ ref('shopify_fulfillment_data') }}" - inventory_item_source: "{{ ref('shopify_inventory_item_data') }}" - inventory_level_source: "{{ ref('shopify_inventory_level_data') }}" - location_source: "{{ ref('shopify_location_data') }}" - metafield_source: "{{ ref('shopify_metafield_data') }}" - order_note_attribute_source: "{{ ref('shopify_order_note_attribute_data') }}" - order_shipping_line_source: "{{ ref('shopify_order_shipping_line_data') }}" - order_shipping_tax_line_source: "{{ ref('shopify_order_shipping_tax_line_data') }}" - order_tag_source: "{{ ref('shopify_order_tag_data') }}" - order_url_tag_source: "{{ ref('shopify_order_url_tag_data') }}" - price_rule_source: "{{ ref('shopify_price_rule_data') }}" - product_image_source: "{{ ref('shopify_product_image_data') }}" - product_tag_source: "{{ ref('shopify_product_tag_data') }}" - shop_source: "{{ ref('shopify_shop_data') }}" - tender_transaction_source: "{{ ref('shopify_tender_transaction_data') }}" - abandoned_checkout_discount_code_source: "{{ ref('shopify_abandoned_checkout_discount_code_data') }}" - order_discount_code_source: "{{ ref('shopify_order_discount_code_data') }}" + shopify_abandoned_checkout_identifier: "shopify_abandoned_checkout_data" + shopify_collection_product_identifier: "shopify_collection_product_data" + shopify_collection_identifier: "shopify_collection_data" + shopify_customer_tag_identifier: "shopify_customer_tag_data" + shopify_discount_code_identifier: "shopify_discount_code_data" + shopify_fulfillment_identifier: "shopify_fulfillment_data" + shopify_inventory_item_identifier: "shopify_inventory_item_data" + shopify_inventory_level_identifier: "shopify_inventory_level_data" + shopify_location_identifier: "shopify_location_data" + shopify_metafield_identifier: "shopify_metafield_data" + shopify_order_note_attribute_identifier: "shopify_order_note_attribute_data" + shopify_order_shipping_line_identifier: "shopify_order_shipping_line_data" + shopify_order_shipping_tax_line_identifier: "shopify_order_shipping_tax_line_data" + shopify_order_tag_identifier: "shopify_order_tag_data" + shopify_order_url_tag_identifier: "shopify_order_url_tag_data" + shopify_price_rule_identifier: "shopify_price_rule_data" + shopify_product_image_identifier: "shopify_product_image_data" + shopify_product_tag_identifier: "shopify_product_tag_data" + shopify_shop_identifier: "shopify_shop_data" + shopify_tender_transaction_identifier: "shopify_tender_transaction_data" + shopify_abandoned_checkout_discount_code_identifier: "shopify_abandoned_checkout_discount_code_data" + shopify_order_discount_code_identifier: "shopify_order_discount_code_data" dispatch: - macro_namespace: dbt_utils diff --git a/integration_tests/seeds/shopify_order_adjustment_data.csv b/integration_tests/seeds/shopify_order_adjustment_data.csv index 8f6edd9..191f585 100644 --- a/integration_tests/seeds/shopify_order_adjustment_data.csv +++ b/integration_tests/seeds/shopify_order_adjustment_data.csv @@ -1,6 +1,6 @@ id,order_id,refund_id,amount,tax_amount,kind,reason,amount_set,tax_amount_set,_fivetran_synced -109271056455,2712175083591,675617407047,-465,0,shipping_refund,Shipping refund,,,2020-11-14 07:52:56.522 +0000 -109277085767,2773486501959,675634708551,-95,0,shipping_refund,Shipping refund,,,2020-11-14 07:54:41.682 +0000 -109245956167,2771757826119,675548168263,-27,-1.6,shipping_refund,Shipping refund,,,2020-11-14 07:44:24.602 +0000 -109248118855,2771329908807,675555016775,-35,0,shipping_refund,Shipping refund,,,2020-11-14 07:45:11.536 +0000 -109275742279,2773429682247,675632644167,-515,0,refund_discrepancy,Refund discrepancy,,,2020-11-14 07:54:31.054 +0000 \ No newline at end of file +109271056455,2712175083591,675617407047,-465,0,shipping_refund,Shipping refund,,,2020-11-14 07:52:56.522 +109277085767,2773486501959,675634708551,-95,0,shipping_refund,Shipping refund,,,2020-11-14 07:54:41.682 +109245956167,2771757826119,675548168263,-27,-1.6,shipping_refund,Shipping refund,,,2020-11-14 07:44:24.602 +109248118855,2771329908807,675555016775,-35,0,shipping_refund,Shipping refund,,,2020-11-14 07:45:11.536 +109275742279,2773429682247,675632644167,-515,0,refund_discrepancy,Refund discrepancy,,,2020-11-14 07:54:31.054 \ No newline at end of file diff --git a/integration_tests/seeds/shopify_order_data.csv b/integration_tests/seeds/shopify_order_data.csv index 319131e..03d9320 100644 --- a/integration_tests/seeds/shopify_order_data.csv +++ b/integration_tests/seeds/shopify_order_data.csv @@ -1,4 +1,4 @@ id,note,email,taxes_included,currency,subtotal_price,total_tax,total_price,created_at,updated_at,name,shipping_address_name,shipping_address_first_name,shipping_address_last_name,shipping_address_company,shipping_address_phone,shipping_address_address_1,shipping_address_address_2,shipping_address_city,shipping_address_country,shipping_address_country_code,shipping_address_province,shipping_address_province_code,shipping_address_zip,shipping_address_latitude,shipping_address_longitude,billing_address_name,billing_address_first_name,billing_address_last_name,billing_address_company,billing_address_phone,billing_address_address_1,billing_address_address_2,billing_address_city,billing_address_country,billing_address_country_code,billing_address_province,billing_address_province_code,billing_address_zip,billing_address_latitude,billing_address_longitude,customer_id,location_id,user_id,number,order_number,financial_status,fulfillment_status,processed_at,processing_method,referring_site,cancel_reason,cancelled_at,closed_at,total_discounts,total_line_items_price,total_weight,source_name,browser_ip,buyer_accepts_marketing,token,cart_token,checkout_token,test,landing_site_base_url,_fivetran_synced,total_shipping_price_set -2674098602081,71509c29301d2cc14e37ecb53f735608,021cb20b5c78751fc7ddc091b6b69b3e,TRUE,GBP,2.8,0,2.8,2020-09-11 19:35:42.000 ,2020-09-11 19:35:46.000 ,d1743fc58a1e4d78769eaac49994a994,8b121314a4d97bc9dc15bfba8518ec88,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,d6f4a399883df85d9d4b3a02bf6e738a,bc9b8576178dcd886639ba718f1d45c8,ac08c606d455cde42980f980524a8038,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,d41d8cd98f00b204e9800998ecf8427e,,00079ce435afddc28205639142773870,d97319f64674c02595f2989019970fc8,c08dae474c5d4d3326fd6764d2a0ebe6,8b121314a4d97bc9dc15bfba8518ec88,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,d6f4a399883df85d9d4b3a02bf6e738a,bc9b8576178dcd886639ba718f1d45c8,ac08c606d455cde42980f980524a8038,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,d41d8cd98f00b204e9800998ecf8427e,,00079ce435afddc28205639142773870,d97319f64674c02595f2989019970fc8,c08dae474c5d4d3326fd6764d2a0ebe6,3589760876641,,,4135,5135,paid,,2020-09-11 19:35:42.000 ,,,,,,2.8,5.6,0,294517,,TRUE,0f9c2880de17f71511eee5542c29b999,,,FALSE,,2020-09-12 00:15:10.199,one -2669516488801,,dce90c7b4e52e045e5975836aff49cf1,TRUE,GBP,2.8,0,3.79,2020-09-09 23:01:54.000 ,2020-09-10 15:38:26.000 ,4fcb884b5b46413bae526a6e7e49d706,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,3584045351009,,,4066,5066,paid,fulfilled,2020-09-09 23:01:53.000 ,direct,2cc983716a820bc713b793a6e8e73f42,,,2020-09-10 15:38:26.000 ,0,2.8,0,web,109.249.185.68,FALSE,fb489b3ccc0ae36ce47744d7595e9746,b1ff04883dfeab658cd5211050476729,7bdb994e1196de3e4f34586e357613f9,FALSE,8584e97b29b0802fb393fa453a8b6a7a,2020-09-11 00:14:33.536,one -2669509541985,,dce90c7b4e52e045e5975836aff49cf1,TRUE,GBP,4.4,0,5.39,2020-09-09 22:57:51.000 ,2020-09-10 15:38:25.000 ,9e346f2e912c60e16679f4a4c8d29422,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,3584045351009,,,4065,5065,paid,fulfilled,2020-09-09 22:57:50.000 ,direct,2cc983716a820bc713b793a6e8e73f42,,,2020-09-10 15:38:25.000 ,0,4.4,0,web,109.249.185.68,FALSE,e44b7f04610a8f4032530cc7f12663de,9600543f4d4613db59ac58a1009ecbb9,cf0a9fe2c7c606b86559007dbb890a62,FALSE,8584e97b29b0802fb393fa453a8b6a7a,2020-09-11 00:14:33.037,one \ No newline at end of file +2674098602081,71509c29301d2cc14e37ecb53f735608,021cb20b5c78751fc7ddc091b6b69b3e,TRUE,GBP,2.8,0,2.8,2020-09-11 19:35:42.000,2020-09-11 19:35:46.000,d1743fc58a1e4d78769eaac49994a994,8b121314a4d97bc9dc15bfba8518ec88,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,d6f4a399883df85d9d4b3a02bf6e738a,bc9b8576178dcd886639ba718f1d45c8,ac08c606d455cde42980f980524a8038,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,d41d8cd98f00b204e9800998ecf8427e,,00079ce435afddc28205639142773870,d97319f64674c02595f2989019970fc8,c08dae474c5d4d3326fd6764d2a0ebe6,8b121314a4d97bc9dc15bfba8518ec88,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,d6f4a399883df85d9d4b3a02bf6e738a,bc9b8576178dcd886639ba718f1d45c8,ac08c606d455cde42980f980524a8038,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,d41d8cd98f00b204e9800998ecf8427e,,00079ce435afddc28205639142773870,d97319f64674c02595f2989019970fc8,c08dae474c5d4d3326fd6764d2a0ebe6,3589760876641,,,4135,5135,paid,,2020-09-11 19:35:42.000,,,,,,2.8,5.6,0,294517,,TRUE,0f9c2880de17f71511eee5542c29b999,,,FALSE,,2020-09-12 00:15:10.199,one +2669516488801,,dce90c7b4e52e045e5975836aff49cf1,TRUE,GBP,2.8,0,3.79,2020-09-09 23:01:54.000,2020-09-10 15:38:26.000,4fcb884b5b46413bae526a6e7e49d706,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,3584045351009,,,4066,5066,paid,fulfilled,2020-09-09 23:01:53.000,direct,2cc983716a820bc713b793a6e8e73f42,,,2020-09-10 15:38:26.000,0,2.8,0,web,109.249.185.68,FALSE,fb489b3ccc0ae36ce47744d7595e9746,b1ff04883dfeab658cd5211050476729,7bdb994e1196de3e4f34586e357613f9,FALSE,8584e97b29b0802fb393fa453a8b6a7a,2020-09-11 00:14:33.536,one +2669509541985,,dce90c7b4e52e045e5975836aff49cf1,TRUE,GBP,4.4,0,5.39,2020-09-09 22:57:51.000,2020-09-10 15:38:25.000,9e346f2e912c60e16679f4a4c8d29422,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,3584045351009,,,4065,5065,paid,fulfilled,2020-09-09 22:57:50.000,direct,2cc983716a820bc713b793a6e8e73f42,,,2020-09-10 15:38:25.000,0,4.4,0,web,109.249.185.68,FALSE,e44b7f04610a8f4032530cc7f12663de,9600543f4d4613db59ac58a1009ecbb9,cf0a9fe2c7c606b86559007dbb890a62,FALSE,8584e97b29b0802fb393fa453a8b6a7a,2020-09-11 00:14:33.037,one \ No newline at end of file diff --git a/integration_tests/seeds/shopify_product_variant_data.csv b/integration_tests/seeds/shopify_product_variant_data.csv index 18dcd0c..44dde0f 100644 --- a/integration_tests/seeds/shopify_product_variant_data.csv +++ b/integration_tests/seeds/shopify_product_variant_data.csv @@ -1,6 +1,6 @@ id,product_id,inventory_item_id,title,price,sku,position,inventory_policy,compare_at_price,fulfillment_service,inventory_management,created_at,updated_at,taxable,barcode,grams,image_id,inventory_quantity,weight,weight_unit,old_inventory_quantity,requires_shipping,_fivetran_synced,option_2,tax_code,option_3,option_1 -39262114414663,6540108431431,41356021661767,my title here,111,,1,deny,,manual,,2021-03-08 16:30:15.000 +0000,2021-04-12 19:49:43.000 +0000,FALSE,,0,,0,0,lb,0,FALSE,2021-04-16 07:50:32.995 +0000,,,,my title here -39273118957639,6544066379847,41367035936839,my title here,222,,1,deny,,manual,,2021-03-17 16:39:45.000 +0000,2021-04-12 19:46:59.000 +0000,FALSE,,0,,0,0,lb,0,FALSE,2021-04-16 07:50:29.241 +0000,,,,my title here -39290169262151,6548438188103,41384094924871,my title here,5,,1,deny,,manual,inventory manager,2021-03-30 19:48:15.000 +0000,2021-03-30 19:48:15.000 +0000,TRUE,,0,,0,0,lb,0,TRUE,2021-04-16 07:50:32.720 +0000,,,,my title here -39262115397703,6540109250631,41356022644807,my title here,333,,1,deny,,manual,,2021-03-08 16:31:31.000 +0000,2021-04-12 19:47:26.000 +0000,FALSE,,0,,-5,0,lb,-5,FALSE,2021-04-16 07:50:29.822 +0000,,,,my title here -29217058947142,3879735590982,30309980143686,my other title,444,,1,deny,,manual,inventory manager,2019-06-25 18:32:03.000 +0000,2019-10-01 23:40:09.000 +0000,TRUE,,222,,0,1,lb,0,TRUE,2021-04-16 07:50:25.006 +0000,,TR9999,,my other title \ No newline at end of file +39262114414663,6540108431431,41356021661767,my title here,111,,1,deny,,manual,,2021-03-08 16:30:15.000,2021-04-12 19:49:43.000,FALSE,,0,,0,0,lb,0,FALSE,2021-04-16 07:50:32.995,,,,my title here +39273118957639,6544066379847,41367035936839,my title here,222,,1,deny,,manual,,2021-03-17 16:39:45.000,2021-04-12 19:46:59.000,FALSE,,0,,0,0,lb,0,FALSE,2021-04-16 07:50:29.241,,,,my title here +39290169262151,6548438188103,41384094924871,my title here,5,,1,deny,,manual,inventory manager,2021-03-30 19:48:15.000,2021-03-30 19:48:15.000,TRUE,,0,,0,0,lb,0,TRUE,2021-04-16 07:50:32.720,,,,my title here +39262115397703,6540109250631,41356022644807,my title here,333,,1,deny,,manual,,2021-03-08 16:31:31.000,2021-04-12 19:47:26.000,FALSE,,0,,-5,0,lb,-5,FALSE,2021-04-16 07:50:29.822,,,,my title here +29217058947142,3879735590982,30309980143686,my other title,444,,1,deny,,manual,inventory manager,2019-06-25 18:32:03.000,2019-10-01 23:40:09.000,TRUE,,222,,0,1,lb,0,TRUE,2021-04-16 07:50:25.006,,TR9999,,my other title \ No newline at end of file diff --git a/integration_tests/seeds/shopify_refund_data.csv b/integration_tests/seeds/shopify_refund_data.csv index 3ed823d..7a7f50f 100644 --- a/integration_tests/seeds/shopify_refund_data.csv +++ b/integration_tests/seeds/shopify_refund_data.csv @@ -1,6 +1,6 @@ id,created_at,processed_at,note,restock,user_id,_fivetran_synced,total_duties_set,order_id -801704738887,2021-04-17 20:25:08.000 +0000,2021-04-17 20:25:08.000 +0000,,FALSE,40467791943,2021-04-18 08:05:22.056 +0000,,3726667481159 -801695039559,2021-04-17 15:45:21.000 +0000,2021-04-17 15:45:21.000 +0000,,FALSE,40467791943,2021-04-18 07:52:19.104 +0000,,3725521846343 -801704181831,2021-04-17 20:15:01.000 +0000,2021-04-17 20:15:01.000 +0000,,FALSE,40467791943,2021-04-18 08:05:22.522 +0000,,3726619476039 -801703428167,2021-04-17 19:56:51.000 +0000,2021-04-17 19:56:51.000 +0000,my refund note,FALSE,40467791943,2021-04-18 08:05:22.841 +0000,,3726370996295 -801707360327,2021-04-17 21:32:50.000 +0000,2021-04-17 21:32:50.000 +0000,,FALSE,40467791943,2021-04-18 08:02:24.256 +0000,,3726858289223 \ No newline at end of file +801704738887,2021-04-17 20:25:08.000,2021-04-17 20:25:08.000,,FALSE,40467791943,2021-04-18 08:05:22.056,,3726667481159 +801695039559,2021-04-17 15:45:21.000,2021-04-17 15:45:21.000,,FALSE,40467791943,2021-04-18 07:52:19.104,,3725521846343 +801704181831,2021-04-17 20:15:01.000,2021-04-17 20:15:01.000,,FALSE,40467791943,2021-04-18 08:05:22.522,,3726619476039 +801703428167,2021-04-17 19:56:51.000,2021-04-17 19:56:51.000,my refund note,FALSE,40467791943,2021-04-18 08:05:22.841,,3726370996295 +801707360327,2021-04-17 21:32:50.000,2021-04-17 21:32:50.000,,FALSE,40467791943,2021-04-18 08:02:24.256,,3726858289223 \ No newline at end of file diff --git a/integration_tests/seeds/shopify_transaction_data.csv b/integration_tests/seeds/shopify_transaction_data.csv index ed6dc8a..9bf0015 100644 --- a/integration_tests/seeds/shopify_transaction_data.csv +++ b/integration_tests/seeds/shopify_transaction_data.csv @@ -1,6 +1,6 @@ id,order_id,refund_id,amount,authorization,created_at,processed_at,device_id,gateway,source_name,message,currency,location_id,parent_id,payment_avs_result_code,kind,currency_exchange_id,currency_exchange_adjustment,currency_exchange_original_amount,currency_exchange_final_amount,currency_exchange_currency,error_code,status,test,user_id,_fivetran_synced,payment_credit_card_bin,payment_cvv_result_code,payment_credit_card_number,payment_credit_card_company,receipt -2667417567303,2181743870023,,415,abcd999999,2020-02-27 16:05:37.000 +0000,2020-02-27 16:05:37.000 +0000,,gateway_here,source_name,message_here,USD,,,Z,sale,,,,,,,success,FALSE,,2020-10-28 20:33:09.797 +0000,,,,, -2572210896967,2089104834631,,415,abcd888888,2020-01-12 20:06:37.000 +0000,2020-01-12 20:06:37.000 +0000,,gateway_here,source_name,message_here,USD,,,Y,sale,,,,,,,success,FALSE,,2020-10-28 17:05:27.756 +0000,,,,, -2664325611591,2179107356743,,415,abcd77777,2020-02-26 00:12:37.000 +0000,2020-02-26 00:12:37.000 +0000,,gateway_here,source_name,message_here,USD,,,,sale,,,,,,,success,FALSE,,2020-10-28 20:23:50.344 +0000,,,,, -2595729735751,2114590769223,,15.95,abcd66666,2020-01-26 11:04:41.000 +0000,2020-01-26 11:04:41.000 +0000,,gateway_here,source_name,message_here,USD,,,Y,sale,,,,,,,success,FALSE,,2020-10-28 18:10:27.604 +0000,,,,, -2705030512711,2214516916295,,212.12,abcd5555,2020-03-18 00:17:24.000 +0000,2020-03-18 00:17:24.000 +0000,,gateway_here,source_name,message_here,USD,,,,sale,,,,,,,success,FALSE,,2020-10-28 22:14:02.944 +0000,,,,, \ No newline at end of file +2667417567303,2181743870023,,415,abcd999999,2020-02-27 16:05:37.000,2020-02-27 16:05:37.000,,gateway_here,source_name,message_here,USD,,,Z,sale,,,,,,,success,FALSE,,2020-10-28 20:33:09.797,,,,, +2572210896967,2089104834631,,415,abcd888888,2020-01-12 20:06:37.000,2020-01-12 20:06:37.000,,gateway_here,source_name,message_here,USD,,,Y,sale,,,,,,,success,FALSE,,2020-10-28 17:05:27.756,,,,, +2664325611591,2179107356743,,415,abcd77777,2020-02-26 00:12:37.000,2020-02-26 00:12:37.000,,gateway_here,source_name,message_here,USD,,,,sale,,,,,,,success,FALSE,,2020-10-28 20:23:50.344,,,,, +2595729735751,2114590769223,,15.95,abcd66666,2020-01-26 11:04:41.000,2020-01-26 11:04:41.000,,gateway_here,source_name,message_here,USD,,,Y,sale,,,,,,,success,FALSE,,2020-10-28 18:10:27.604,,,,, +2705030512711,2214516916295,,212.12,abcd5555,2020-03-18 00:17:24.000,2020-03-18 00:17:24.000,,gateway_here,source_name,message_here,USD,,,,sale,,,,,,,success,FALSE,,2020-10-28 22:14:02.944,,,,, \ No newline at end of file diff --git a/models/src_shopify.yml b/models/src_shopify.yml index d00af69..aafd7c3 100644 --- a/models/src_shopify.yml +++ b/models/src_shopify.yml @@ -477,6 +477,7 @@ sources: - name: abandoned_checkout description: A checkout is considered abandoned after the customer has added contact information, but before the customer has completed their purchase. + identifier: "{{ var('shopify_abandoned_checkout_identifier', 'abandoned_checkout') }}" columns: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -607,6 +608,7 @@ sources: - name: collection_product description: Table relating products to the collections they belong to. Use this instead of the deprecated `COLLECT` table. + identifier: "{{ var('shopify_collection_product_identifier', 'collection_product') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -619,6 +621,7 @@ sources: description: > A collection is a grouping of products that merchants can create to make their stores easier to browse. Merchants can create collections by selecting products individually or by defining rules that automatically determine whether products are included (aka a "smart collection"). + identifier: "{{ var('shopify_collection_identifier', 'collection') }}" columns: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -661,6 +664,7 @@ sources: - name: customer_tag description: Tags that the shop owner has attached to the customer. A customer can have up to 250 tags. + identifier: "{{ var('shopify_customer_tag_identifier', 'customer_tag') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -675,6 +679,7 @@ sources: description: > Table storing discount codes that enable specific discounts to be redeemed. Merchants can distribute discount codes to their customers using a variety of means, such as an email or URL, and customers can apply these codes at checkout. Each discount code belongs to a price rule, which holds the logic for the discount. + identifier: "{{ var('shopify_discount_code_identifier', 'discount_code') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -695,6 +700,7 @@ sources: description: > A fulfillment order represents a group of one or more items in an order that will be fulfilled from the same location. A fulfillment represents work that is completed as part of a fulfillment order and can include one or more items. + identifier: "{{ var('shopify_fulfillment_identifier', 'fulfillment') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -750,6 +756,7 @@ sources: description: > An inventory item represents a physical good. It holds essential information about the physical good, including its SKU and whether its inventory is tracked. There is a 1:1 relationship between a product variant and an inventory item. + identifier: "{{ var('shopify_inventory_item_identifier', 'inventory_item') }}" columns: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -780,6 +787,7 @@ sources: Each inventory level belongs to one inventory item and has one location. For every location where an inventory item can be stocked, there's an inventory level that represents the inventory item's quantities relating to that location. + identifier: "{{ var('shopify_inventory_level_identifier', 'inventory_level') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -794,6 +802,7 @@ sources: - name: location description: A location represents a geographical location where your stores, pop-up stores, headquarters, and warehouses exist. + identifier: "{{ var('shopify_location_identifier', 'location') }}" columns: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -843,6 +852,7 @@ sources: description: > Metafields are a flexible way to attach additional information to a Shopify resource (e.g. Product, Collection, etc.). Some examples of data stored using metafields include specifications, size charts, downloadable documents, release dates, images, or part numbers. Metafields are identified by an owner resource, a namespace, and a key and they store a value along with type information for that context. + identifier: "{{ var('shopify_metafield_identifier', 'metafield') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -871,6 +881,7 @@ sources: - name: order_note_attribute description: Table storing custom attributes placed on orders. + identifier: "{{ var('shopify_order_note_attribute_identifier', 'order_note_attribute') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -883,6 +894,7 @@ sources: - name: order_shipping_line description: Represents the shipping details that the customer chose for their order. + identifier: "{{ var('shopify_order_shipping_line_identifier', 'order_shipping_line') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -915,6 +927,7 @@ sources: - name: order_shipping_tax_line description: Represents taxes associated with order shipping lines. + identifier: "{{ var('shopify_order_shipping_tax_line_identifier', 'order_shipping_tax_line') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -933,6 +946,7 @@ sources: - name: order_tag description: Tags that the shop owner has attached to the order. + identifier: "{{ var('shopify_order_tag_identifier', 'order_tag') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -945,6 +959,7 @@ sources: - name: order_url_tag description: Key-value tags that the shop owner has attached to the order url. + identifier: "{{ var('shopify_order_url_tag_identifier', 'order_url_tag') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -959,6 +974,7 @@ sources: description: > Table storing price rules. You can use price rules to create discounts dependent on conditions. Merchants can distribute the discount codes to their customers. Using price rules, you can create discounts that specify a discount as a percentage, a fixed amount, or free shipping. You use entitlements and prerequisites to dynamically build these discounts. + identifier: "{{ var('shopify_price_rule_identifier', 'price_rule') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1021,6 +1037,7 @@ sources: - name: product_image description: Table storing hosted images of products. + identifier: "{{ var('shopify_product_image_identifier', 'product_image') }}" columns: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -1047,6 +1064,7 @@ sources: - name: product_tag description: Tags that the shop owner has attached to the product. + identifier: "{{ var('shopify_product_tag_identifier', 'product_tag') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1059,6 +1077,7 @@ sources: - name: shop description: The Shop resource is a collection of general business and store management settings and information about the store. + identifier: "{{ var('shopify_shop_identifier', 'shop') }}" columns: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -1176,6 +1195,7 @@ sources: Each tender transaction represents money passing between the merchant and a customer. A tender transaction with a positive amount represents a transaction where the customer paid money to the merchant. A negative amount represents a transaction where the merchant refunded money back to the customer. Tender transactions represent transactions that modify the shop's balance. + identifier: "{{ var('shopify_tender_transaction_identifier', 'tender_transaction') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1213,6 +1233,7 @@ sources: - name: abandoned_checkout_discount_code description: Abandoned checkouts that applied a discount code. + identifier: "{{ var('shopify_abandoned_checkout_discount_code_identifier', 'abandoned_checkout_discount_code') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1235,6 +1256,7 @@ sources: - name: order_discount_code description: Discount codes placed on orders. + identifier: "{{ var('shopify_order_discount_code_identifier', 'order_discount_code') }}" columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" diff --git a/models/stg_shopify__order.sql b/models/stg_shopify__order.sql index ee065df..9f49115 100644 --- a/models/stg_shopify__order.sql +++ b/models/stg_shopify__order.sql @@ -60,7 +60,7 @@ final as ( cast(created_at as {{ dbt.type_timestamp() }}) as created_at, currency, customer_id, - email, + lower(email) as email, financial_status, fulfillment_status, processing_method, @@ -98,7 +98,7 @@ final as ( shipping_address_province, shipping_address_province_code, shipping_address_zip, - {# test as is_test_order, #} + test as is_test_order, token, _fivetran_deleted as is_deleted, app_id, @@ -114,8 +114,8 @@ final as ( {{ fivetran_utils.fill_pass_through_columns('order_pass_through_columns') }} from fields - where not coalesce(test, false) ) select * -from final \ No newline at end of file +from final +where not coalesce(is_test_order, false) \ No newline at end of file diff --git a/models/stg_shopify__order_line.sql b/models/stg_shopify__order_line.sql index 6cbb244..39f3c8a 100644 --- a/models/stg_shopify__order_line.sql +++ b/models/stg_shopify__order_line.sql @@ -28,13 +28,12 @@ fields as ( final as ( select - + id as order_line_id, fulfillable_quantity, fulfillment_service, fulfillment_status, gift_card as is_gift_card, grams, - id as order_line_id, index, name, order_id, From be87671f477cddc43e1b038696c6f8d0c5f81778 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:06:03 -0800 Subject: [PATCH 03/13] adding new table and work from other branches --- dbt_project.yml | 1 + integration_tests/dbt_project.yml | 1 + ..._abandoned_checkout_shipping_line_data.csv | 6 ++ ...andoned_checkout_shipping_line_columns.sql | 32 +++++++++++ models/src_shopify.yml | 5 +- ...pify__abandoned_checkout_shipping_line.sql | 56 +++++++++++++++++++ models/stg_shopify__customer.sql | 18 ++++-- models/stg_shopify__order.sql | 6 +- models/stg_shopify__order_line.sql | 9 ++- ...__abandoned_checkout_shipping_line_tmp.sql | 12 ++++ 10 files changed, 131 insertions(+), 15 deletions(-) create mode 100644 integration_tests/seeds/shopify_abandoned_checkout_shipping_line_data.csv create mode 100644 macros/get_abandoned_checkout_shipping_line_columns.sql create mode 100644 models/stg_shopify__abandoned_checkout_shipping_line.sql create mode 100644 models/tmp/stg_shopify__abandoned_checkout_shipping_line_tmp.sql diff --git a/dbt_project.yml b/dbt_project.yml index 4f27902..54a3cf4 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -42,6 +42,7 @@ vars: tender_transaction_source: "{{ source('shopify','tender_transaction') }}" abandoned_checkout_discount_code_source: "{{ source('shopify','abandoned_checkout_discount_code') }}" order_discount_code_source: "{{ source('shopify','order_discount_code') }}" + abandoned_checkout_shipping_line_source: "{{ source('shopify', 'abandoned_checkout_shipping_line') }}" customer_pass_through_columns: [] order_line_refund_pass_through_columns: [] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 4df4786..a61317a 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -38,6 +38,7 @@ vars: tender_transaction_source: "{{ ref('shopify_tender_transaction_data') }}" abandoned_checkout_discount_code_source: "{{ ref('shopify_abandoned_checkout_discount_code_data') }}" order_discount_code_source: "{{ ref('shopify_order_discount_code_data') }}" + abandoned_checkout_shipping_line_source: "{{ ref('shopify_abandoned_checkout_shipping_line_data') }}" dispatch: - macro_namespace: dbt_utils diff --git a/integration_tests/seeds/shopify_abandoned_checkout_shipping_line_data.csv b/integration_tests/seeds/shopify_abandoned_checkout_shipping_line_data.csv new file mode 100644 index 0000000..2d83f40 --- /dev/null +++ b/integration_tests/seeds/shopify_abandoned_checkout_shipping_line_data.csv @@ -0,0 +1,6 @@ +checkout_id,index,_fivetran_synced,api_client_id,carrier_identifier,carrier_service_id,code,delivery_category,discounted_price,id,markup,phone,price,requested_fulfillment_service_id,source,title,validation_context,delivery_expectation_range,delivery_expectation_type,original_shop_markup,original_shop_price,presentment_title,delivery_expectation_range_min,delivery_expectation_range_max +24587125653675,1,2023-01-09 06:48:18.093000 UTC,,,,Standard,,,c3ce0972c2ed226895b730eaf7001bea,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, +24589347422379,1,2023-01-09 06:48:23.540000 UTC,,,,Standard,,,bf7c909533bdce55017ed4df44902c13,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, +24589643710635,1,2023-01-09 06:48:24.243000 UTC,,,,Standard,,,519ff4275cd972eb26229aaaa4e282db,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, +24586978263211,1,2023-01-09 06:48:18.068000 UTC,,,,Standard,,,8d18671d4862fe463de583f05e1ad46a,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, +24586830381227,1,2023-01-09 06:48:16.985000 UTC,,,,Standard,,,8f2fa35a848caf61a85b1b455ec9e597,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, \ No newline at end of file diff --git a/macros/get_abandoned_checkout_shipping_line_columns.sql b/macros/get_abandoned_checkout_shipping_line_columns.sql new file mode 100644 index 0000000..5b572a2 --- /dev/null +++ b/macros/get_abandoned_checkout_shipping_line_columns.sql @@ -0,0 +1,32 @@ +{% macro get_abandoned_checkout_shipping_line_columns() %} + +{% set columns = [ + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "api_client_id", "datatype": dbt.type_int()}, + {"name": "carrier_identifier", "datatype": dbt.type_string()}, + {"name": "carrier_service_id", "datatype": dbt.type_int()}, + {"name": "checkout_id", "datatype": dbt.type_int()}, + {"name": "code", "datatype": dbt.type_string()}, + {"name": "delivery_category", "datatype": dbt.type_string()}, + {"name": "delivery_expectation_range", "datatype": dbt.type_string()}, + {"name": "delivery_expectation_range_max", "datatype": dbt.type_int()}, + {"name": "delivery_expectation_range_min", "datatype": dbt.type_int()}, + {"name": "delivery_expectation_type", "datatype": dbt.type_string()}, + {"name": "discounted_price", "datatype": dbt.type_float()}, + {"name": "id", "datatype": dbt.type_string()}, + {"name": "index", "datatype": dbt.type_int()}, + {"name": "markup", "datatype": dbt.type_float()}, + {"name": "original_shop_markup", "datatype": dbt.type_string()}, + {"name": "original_shop_price", "datatype": dbt.type_string()}, + {"name": "phone", "datatype": dbt.type_string()}, + {"name": "presentment_title", "datatype": dbt.type_string()}, + {"name": "price", "datatype": dbt.type_float()}, + {"name": "requested_fulfillment_service_id", "datatype": dbt.type_string()}, + {"name": "source", "datatype": dbt.type_string()}, + {"name": "title", "datatype": dbt.type_string()}, + {"name": "validation_context", "datatype": dbt.type_string()} +] %} + +{{ return(columns) }} + +{% endmacro %} diff --git a/models/src_shopify.yml b/models/src_shopify.yml index 7ab0171..a84c78b 100644 --- a/models/src_shopify.yml +++ b/models/src_shopify.yml @@ -1214,4 +1214,7 @@ sources: - name: order_id description: Associated order ID. - name: type - description: The type of discount - `fixed_amount`, `percentage`, or `shipping`. \ No newline at end of file + description: The type of discount - `fixed_amount`, `percentage`, or `shipping`. + + - name: abandoned_checkout_shipping_line + description: Shipping lines associated with abandoned checkouts. \ No newline at end of file diff --git a/models/stg_shopify__abandoned_checkout_shipping_line.sql b/models/stg_shopify__abandoned_checkout_shipping_line.sql new file mode 100644 index 0000000..e748cfe --- /dev/null +++ b/models/stg_shopify__abandoned_checkout_shipping_line.sql @@ -0,0 +1,56 @@ + +with base as ( + + select * + from {{ ref('stg_shopify__abandoned_checkout_shipping_line_tmp') }} +), + +fields as ( + + select + {{ + fivetran_utils.fill_staging_columns( + source_columns=adapter.get_columns_in_relation(ref('stg_shopify__abandoned_checkout_shipping_line_tmp')), + staging_columns=get_abandoned_checkout_shipping_line_columns() + ) + }} + from base +), + +final as ( + + select +20:31:45 [WARNING]: Configuration paths exist in your dbt_project.yml file which do not apply to any resources. +There are 2 unused configuration paths: +- models.netsuite +- models.ad_reporting + +20:31:48 _fivetran_synced, + api_client_id, + carrier_identifier, + carrier_service_id, + checkout_id, + code, + delivery_category, + delivery_expectation_range, + delivery_expectation_range_max, + delivery_expectation_range_min, + delivery_expectation_type, + discounted_price, + id, + index, + markup, + original_shop_markup, + original_shop_price, + phone, + presentment_title, + price, + requested_fulfillment_service_id, + source, + title, + validation_context + from fields +) + +select * +from final diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql index 5867f4b..4253723 100644 --- a/models/stg_shopify__customer.sql +++ b/models/stg_shopify__customer.sql @@ -28,12 +28,10 @@ fields as ( final as ( select - accepts_marketing as has_accepted_marketing, - email_marketing_consent_state, - default_address_id, + id as customer_id, lower(email) as email, + default_address_id, first_name, - id as customer_id, last_name, orders_count, phone, @@ -42,11 +40,19 @@ final as ( total_spent, verified_email as is_verified_email, note, - coalesce(marketing_opt_in_level, email_marketing_consent_opt_in_level) as marketing_opt_in_level, lifetime_duration, currency, + case + when email_marketing_consent_state is null then + case + when accepts_marketing is null then null + when accepts_marketing then 'subscribed (legacy)' + else 'not_subscribed (legacy)' end + else lower(email_marketing_consent_state) end as marketing_consent_state, + lower(coalesce(email_marketing_consent_opt_in_level, marketing_opt_in_level)) as marketing_opt_in_level, + + {{ dbt_date.convert_timezone(column='cast(coalesce(accepts_marketing_updated_at, email_marketing_consent_consent_updated_at) as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as marketing_consent_updated_at, {{ dbt_date.convert_timezone(column='cast(created_at as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as created_timestamp, - {{ dbt_date.convert_timezone(column='cast(coalesce(accepts_marketing_updated_at, email_marketing_consent_consent_updated_at) as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as accepts_marketing_updated_at, {{ dbt_date.convert_timezone(column='cast(updated_at as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as updated_timestamp, {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced, source_relation diff --git a/models/stg_shopify__order.sql b/models/stg_shopify__order.sql index 40f3e5d..c9f48ca 100644 --- a/models/stg_shopify__order.sql +++ b/models/stg_shopify__order.sql @@ -106,7 +106,7 @@ final as ( device_id, order_status_url, presentment_currency, - {# test as is_test_order, #} + test as is_test_order, _fivetran_deleted as is_deleted, buyer_accepts_marketing as has_buyer_accepted_marketing, confirmed as is_confirmed, @@ -116,8 +116,8 @@ final as ( {{ fivetran_utils.fill_pass_through_columns('order_pass_through_columns') }} from fields - where not coalesce(test, false) ) select * -from final \ No newline at end of file +from final +where not coalesce(is_test_order, false) \ No newline at end of file diff --git a/models/stg_shopify__order_line.sql b/models/stg_shopify__order_line.sql index 3c56784..18642e3 100644 --- a/models/stg_shopify__order_line.sql +++ b/models/stg_shopify__order_line.sql @@ -28,16 +28,15 @@ fields as ( final as ( select - + id as order_line_id, + index, + name, + order_id, fulfillable_quantity, fulfillment_service, fulfillment_status, gift_card as is_gift_card, grams, - id as order_line_id, - index, - name, - order_id, pre_tax_price, pre_tax_price_set, price, diff --git a/models/tmp/stg_shopify__abandoned_checkout_shipping_line_tmp.sql b/models/tmp/stg_shopify__abandoned_checkout_shipping_line_tmp.sql new file mode 100644 index 0000000..b7b7e7b --- /dev/null +++ b/models/tmp/stg_shopify__abandoned_checkout_shipping_line_tmp.sql @@ -0,0 +1,12 @@ +{{ + fivetran_utils.union_data( + table_identifier='abandoned_checkout_shipping_line', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='abandoned_checkout_shipping_line_source', + union_schema_variable='shopify_union_schemas', + union_database_variable='shopify_union_databases' + ) +}} \ No newline at end of file From 2e026972112f32faf00469e578e15eb76a92dfdc Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:14:02 -0800 Subject: [PATCH 04/13] buildkite --- .../shopify_abandoned_checkout_shipping_line_data.csv | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/integration_tests/seeds/shopify_abandoned_checkout_shipping_line_data.csv b/integration_tests/seeds/shopify_abandoned_checkout_shipping_line_data.csv index 2d83f40..62181c4 100644 --- a/integration_tests/seeds/shopify_abandoned_checkout_shipping_line_data.csv +++ b/integration_tests/seeds/shopify_abandoned_checkout_shipping_line_data.csv @@ -1,6 +1,6 @@ checkout_id,index,_fivetran_synced,api_client_id,carrier_identifier,carrier_service_id,code,delivery_category,discounted_price,id,markup,phone,price,requested_fulfillment_service_id,source,title,validation_context,delivery_expectation_range,delivery_expectation_type,original_shop_markup,original_shop_price,presentment_title,delivery_expectation_range_min,delivery_expectation_range_max -24587125653675,1,2023-01-09 06:48:18.093000 UTC,,,,Standard,,,c3ce0972c2ed226895b730eaf7001bea,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, -24589347422379,1,2023-01-09 06:48:23.540000 UTC,,,,Standard,,,bf7c909533bdce55017ed4df44902c13,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, -24589643710635,1,2023-01-09 06:48:24.243000 UTC,,,,Standard,,,519ff4275cd972eb26229aaaa4e282db,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, -24586978263211,1,2023-01-09 06:48:18.068000 UTC,,,,Standard,,,8d18671d4862fe463de583f05e1ad46a,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, -24586830381227,1,2023-01-09 06:48:16.985000 UTC,,,,Standard,,,8f2fa35a848caf61a85b1b455ec9e597,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, \ No newline at end of file +653675,1,2023-01-09 06:48:18.093000,,,,Standard,,,c3ce0972c2e30eaf7001bea,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, +379,1,2023-01-09 06:48:23.540000,,,,Standard,,,bf7c90953344902c13,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, +635,1,2023-01-09 06:48:24.243000,,,,Standard,,,519ff4275cd972e282db,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, +3211,1,2023-01-09 06:48:18.068000,,,,Standard,,,8d18671d481ad46a,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, +381227,1,2023-01-09 06:48:16.985000,,,,Standard,,,8f2fab1b455ec9e597,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,, \ No newline at end of file From 17ac3b64b3ed4bd3f15d646ec6942b13246ea73c Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:25:25 -0800 Subject: [PATCH 05/13] buildkite try 2 --- integration_tests/dbt_project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 06f5873..c920f44 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -4,6 +4,7 @@ profile: 'integration_tests' config-version: 2 vars: + shopify_schema: shopify_source_integration_tests_2 shopify_source: shopify_customer_identifier: "shopify_customer_data" shopify_order_line_refund_identifier: "shopify_order_line_refund_data" From 4b70ff08b6094787283a6d35b323a671d188eddf Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:35:05 -0800 Subject: [PATCH 06/13] buildkite' --- ...g_shopify__abandoned_checkout_shipping_line.sql | 14 ++++++++------ models/stg_shopify__customer.sql | 5 ----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/models/stg_shopify__abandoned_checkout_shipping_line.sql b/models/stg_shopify__abandoned_checkout_shipping_line.sql index 98804f6..6ce0d25 100644 --- a/models/stg_shopify__abandoned_checkout_shipping_line.sql +++ b/models/stg_shopify__abandoned_checkout_shipping_line.sql @@ -20,20 +20,19 @@ fields as ( final as ( select - _fivetran_synced, + id as abandoned_checkout_shipping_line_id, + checkout_id, + index, api_client_id, carrier_identifier, carrier_service_id, - checkout_id, - code, + code as shipping_code, delivery_category, delivery_expectation_range, delivery_expectation_range_max, delivery_expectation_range_min, delivery_expectation_type, discounted_price, - id, - index, markup, original_shop_markup, original_shop_price, @@ -43,7 +42,10 @@ final as ( requested_fulfillment_service_id, source, title, - validation_context + validation_context, + {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced, + source_relation + from fields ) diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql index 9b713bc..a5d169d 100644 --- a/models/stg_shopify__customer.sql +++ b/models/stg_shopify__customer.sql @@ -29,12 +29,7 @@ final as ( select id as customer_id, -<<<<<<< HEAD lower(email) as email, - default_address_id, -======= - email, ->>>>>>> d1c1f494e9fe77be75bdaf29bde38582d57302c3 first_name, last_name, orders_count, From 7a7f60ad48918db2722811d3bd8fe9e3f20b90fa Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 9 Jan 2023 15:52:05 -0800 Subject: [PATCH 07/13] working --- models/src_shopify.yml | 2 +- models/stg_shopify__abandoned_checkout_discount_code.sql | 2 +- models/stg_shopify__abandoned_checkout_shipping_line.sql | 6 ++++++ models/stg_shopify__discount_code.sql | 2 +- models/stg_shopify__order_discount_code.sql | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/models/src_shopify.yml b/models/src_shopify.yml index 50979c9..96531f4 100644 --- a/models/src_shopify.yml +++ b/models/src_shopify.yml @@ -1270,5 +1270,5 @@ sources: description: The type of discount - `fixed_amount`, `percentage`, or `shipping`. - name: abandoned_checkout_shipping_line - description: Shipping lines associated with abandoned checkouts. + description: Shipping lines associated with abandoned checkouts. TODO identifier: "{{ var('shopify_abandoned_checkout_shipping_line_identifier', 'abandoned_checkout_shipping_line') }}" \ No newline at end of file diff --git a/models/stg_shopify__abandoned_checkout_discount_code.sql b/models/stg_shopify__abandoned_checkout_discount_code.sql index 7eb4482..8b32827 100644 --- a/models/stg_shopify__abandoned_checkout_discount_code.sql +++ b/models/stg_shopify__abandoned_checkout_discount_code.sql @@ -28,7 +28,7 @@ final as ( select checkout_id, - code, + upper(code) as code, discount_id, amount, type, diff --git a/models/stg_shopify__abandoned_checkout_shipping_line.sql b/models/stg_shopify__abandoned_checkout_shipping_line.sql index 6ce0d25..dd559db 100644 --- a/models/stg_shopify__abandoned_checkout_shipping_line.sql +++ b/models/stg_shopify__abandoned_checkout_shipping_line.sql @@ -14,6 +14,12 @@ fields as ( staging_columns=get_abandoned_checkout_shipping_line_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='shopify_union_schemas', + union_database_variable='shopify_union_databases') + }} + from base ), diff --git a/models/stg_shopify__discount_code.sql b/models/stg_shopify__discount_code.sql index af2205b..fa33d0f 100644 --- a/models/stg_shopify__discount_code.sql +++ b/models/stg_shopify__discount_code.sql @@ -27,7 +27,7 @@ final as ( select id as discount_code_id, - code, + upper(code) as code, price_rule_id, usage_count, {{ dbt_date.convert_timezone(column='cast(created_at as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as created_at, diff --git a/models/stg_shopify__order_discount_code.sql b/models/stg_shopify__order_discount_code.sql index 325e79f..851f3ba 100644 --- a/models/stg_shopify__order_discount_code.sql +++ b/models/stg_shopify__order_discount_code.sql @@ -27,7 +27,7 @@ final as ( select order_id, - code, + upper(code) as code, type, amount, {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced, From 4f403a3f3a0a909e02b0083561f786468392e64b Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 10 Jan 2023 13:50:03 -0800 Subject: [PATCH 08/13] buildkite seeds --- integration_tests/dbt_project.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index c920f44..e42b44b 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -155,4 +155,10 @@ seeds: shopify_inventory_item_data: +column_types: updated_at: timestamp - created_at: timestamp \ No newline at end of file + created_at: timestamp + shopify_abandoned_checkout_shipping_line_data: + +column_types: + markup: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + price: float + original_shop_markup: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + original_shop_price: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" \ No newline at end of file From 1752768efe575f0bb49e4af4e9a8ecfd1ee502fa Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 11 Jan 2023 11:36:10 -0800 Subject: [PATCH 09/13] addeded discount code does_table_exist logic --- ...pify__abandoned_checkout_discount_code.sql | 1 - models/stg_shopify__discount_code.sql | 1 + models/tmp/stg_shopify__discount_code_tmp.sql | 27 ++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/models/stg_shopify__abandoned_checkout_discount_code.sql b/models/stg_shopify__abandoned_checkout_discount_code.sql index 8b32827..38aff52 100644 --- a/models/stg_shopify__abandoned_checkout_discount_code.sql +++ b/models/stg_shopify__abandoned_checkout_discount_code.sql @@ -1,4 +1,3 @@ - with base as ( select * diff --git a/models/stg_shopify__discount_code.sql b/models/stg_shopify__discount_code.sql index fa33d0f..f464046 100644 --- a/models/stg_shopify__discount_code.sql +++ b/models/stg_shopify__discount_code.sql @@ -1,3 +1,4 @@ +-- this model will be all NULL until you create a discount code in Shopify with base as ( diff --git a/models/tmp/stg_shopify__discount_code_tmp.sql b/models/tmp/stg_shopify__discount_code_tmp.sql index d59b6af..eba47d3 100644 --- a/models/tmp/stg_shopify__discount_code_tmp.sql +++ b/models/tmp/stg_shopify__discount_code_tmp.sql @@ -1,3 +1,14 @@ +-- this model will be all NULL until you create a discount code in Shopify + +{%- set source_relation = adapter.get_relation( + database=source('shopify', 'discount_code').database, + schema=source('shopify', 'discount_code').schema, + identifier=source('shopify', 'discount_code').name) -%} + +{% set table_exists=source_relation is not none %} + +{% if table_exists %} + {{ fivetran_utils.union_data( table_identifier='discount_code', @@ -9,4 +20,18 @@ union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) -}} \ No newline at end of file +}} + +{% else %} + +select + cast(null as {{ dbt.type_timestamp() }}) as _fivetran_synced, + cast(null as {{ dbt.type_string() }}) as code, + cast(null as {{ dbt.type_timestamp() }}) as created_at, + cast(null as {{ dbt.type_int() }}) as id, + cast(null as {{ dbt.type_int() }}) as price_rule_id, + cast(null as {{ dbt.type_timestamp() }}) as updated_at, + cast(null as {{ dbt.type_float() }}) as usage_count, + cast(null as {{ dbt.type_string() }}) as _dbt_source_relation + +{% endif %} \ No newline at end of file From 5c636903701af64aabbde06a435798f39da2e9e3 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:14:30 -0800 Subject: [PATCH 10/13] updated passthrough column docs --- README.md | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 0d3eb61..e2705a2 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,15 @@ To use this dbt package, you must have the following: - At least one Fivetran Shopify connector syncing data into your destination. - A **BigQuery**, **Snowflake**, **Redshift**, **Databricks**, or **PostgreSQL** destination. -## Step 2: Install the package -Include the following shopify_source package version in your `packages.yml` file. +## Step 2: Install the package (skip if also using the `dbt_shopify` package) +If you are **not** using the [Shopify transformation package](https://github.com/fivetran/dbt_shopify), include the following package version in your `packages.yml` file. If you are installing the transform package, the source package is automatically installed as a dependency. > TIP: Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. ```yml packages: - package: fivetran/shopify_source version: [">=0.7.0", "<0.8.0"] ``` + ## Step 3: Define database and schema variables ### Single connector By default, this package runs using your destination and the `shopify` schema. If this is not where your Shopify data is (for example, if your Shopify schema is named `shopify_fivetran` and your `issue` table is named `usa_issue`), add the following configuration to your root `dbt_project.yml` file: @@ -59,20 +60,31 @@ vars: ## (Optional) Step 5: Additional configurations
Expand to view configurations -### Add Passthrough Columns - todo, update with new syntax -This package includes all source columns defined in the [staging_columns.sql](https://github.com/fivetran/dbt_shopify_source/blob/master/macros/staging_columns.sql) macro. To add additional columns to this package, do so using our pass-through column variables in your root `dbt_project.yml`. This is extremely useful if you'd like to include custom fields to the package. +### Passing Through Additional Fields +This package includes all source columns defined in the macros folder. You can add more columns using our pass-through column variables. These variables allow for the pass-through fields to be aliased (`alias`) and casted (`transform_sql`) if desired, but not required. Datatype casting is configured via a sql snippet within the `transform_sql` key. You may add the desired sql while omitting the `as field_name` at the end and your custom pass-though fields will be casted accordingly. Use the below format for declaring the respective pass-through variables: ```yml # dbt_project.yml vars: shopify_source: - customer_pass_through_columns: [] - order_line_refund_pass_through_columns: [] - order_line_pass_through_columns: [] - order_pass_through_columns: [] - product_pass_through_columns: [] - product_variant_pass_through_columns: [] + customer_pass_through_columns: + - name: "customer_custom_field" + alias: "customer_field" + order_line_refund_pass_through_columns: + - name: "unique_string_field" + alias: "field_id" + transform_sql: "cast(field_id as string)" + order_line_pass_through_columns: + - name: "that_field" + order_pass_through_columns: + - name: "sub_field" + alias: "subsidiary_field" + product_pass_through_columns: + - name: "this_field" + product_variant_pass_through_columns: + - name: "new_custom_field" + alias: "custom_field" ``` ### Changing the Build Schema From bd3e29a7accbd02bb2225b6f82997198bbb5a7bd Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 11 Jan 2023 14:17:00 -0800 Subject: [PATCH 11/13] joe feedback --- models/stg_shopify.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index a7f537c..981fd36 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -818,6 +818,9 @@ models: - cancelled: The fulfillment was cancelled. - error: There was an error with the fulfillment request. - failure: The fulfillment request failed. + tests: + - accepted_values: + values: ['pending', 'open', 'success', 'cancelled', 'error', 'failure'] - name: tracking_company description: The name of the tracking company. - name: tracking_number @@ -1168,6 +1171,9 @@ models: `entitled` (the price rule applies the discount to selected entitlements only). - name: target_type description: The target type that the price rule applies to. Valid values include `line_item` (the price rule applies to the cart's line items) and `shipping_line` (the price rule applies to the cart's shipping lines). + tests: + - accepted_values: + values: ['line_item', 'shipping_line'] - name: title description: > The title of the price rule. This is used by the Shopify admin search to retrieve discounts. It is also displayed on the Discounts page of the Shopify admin for bulk discounts. @@ -1184,6 +1190,9 @@ models: and `percentage` (applies a percentage discount of value. For example, if value is -30, then 30% will be deducted when the discount is applied). If `target_type` is `shipping_line`, then only `percentage` is accepted. + tests: + - accepted_values: + values: ['percentage', 'fixed_amount'] - name: stg_shopify__product_image description: Table storing hosted images of products. From 6279e4991cc6a06d9615bad759777f36dfc4947e Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 11 Jan 2023 14:51:27 -0800 Subject: [PATCH 12/13] test severtiy --- models/stg_shopify.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index 981fd36..6ec39c1 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -821,6 +821,8 @@ models: tests: - accepted_values: values: ['pending', 'open', 'success', 'cancelled', 'error', 'failure'] + config: + severity: warn # we pivot these out in shopify__inventory_levels - name: tracking_company description: The name of the tracking company. - name: tracking_number @@ -1174,6 +1176,8 @@ models: tests: - accepted_values: values: ['line_item', 'shipping_line'] + config: + severity: warn # we use these in a JOIN condition in shopify__discounts - name: title description: > The title of the price rule. This is used by the Shopify admin search to retrieve discounts. It is also displayed on the Discounts page of the Shopify admin for bulk discounts. From 7d548c9c1d0ad132f84e4d00978d69f4bf2cef71 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 11 Jan 2023 15:02:11 -0800 Subject: [PATCH 13/13] test severity and note --- DECISIONLOG.md | 11 ++++++++++- models/stg_shopify.yml | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/DECISIONLOG.md b/DECISIONLOG.md index 4d83a48..dc9257c 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -12,4 +12,13 @@ Thus, the source package will create empty (1 row of all `NULL` fields) staging ## Keeping Deleted Entities -todo - not filtering out _fivetran_deleted in staging models. when joining these tables together in the transform package, bring in _fivetran_deleted as is__deleted \ No newline at end of file +todo - not filtering out _fivetran_deleted in staging models. when joining these tables together in the transform package, bring in _fivetran_deleted as is__deleted + +## Accepted Value Test Severity + +We test the following columns for accepted values because their values are hard-coded to be pivoted out into columns and/or used as `JOIN` conditions in downstream models. +- `stg_shopify__price_rule.target_type`: accepted values are `line_item`, `shipping_line` +- `stg_shopify__price_rule.value_type`: accepted values are `percentage`, `fixed_amount` +- `stg_shopify__fulfillment.status`: accepted values are `pending`, `open`, `success`, `cancelled`, `error`, `failure` + +We have chosen to make the severity of these tests `warn`, as non-accepted values will be filtered out in the transformation models. They will not introduce erroneous data. \ No newline at end of file diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index 6ec39c1..0a05364 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -1197,6 +1197,8 @@ models: tests: - accepted_values: values: ['percentage', 'fixed_amount'] + config: + severity: warn # we use these in a JOIN condition in shopify__discounts - name: stg_shopify__product_image description: Table storing hosted images of products.