From 110cbc9a7b172f9a1def8979432aedb7070a4536 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 9 Dec 2022 15:54:26 -0800 Subject: [PATCH 1/8] fields seelction --- macros/get_customer_columns.sql | 33 +++ macros/get_order_adjustment_columns.sql | 18 ++ macros/get_order_columns.sql | 93 +++++++ macros/get_order_line_columns.sql | 43 ++++ macros/get_order_line_refund_columns.sql | 21 ++ macros/get_product_columns.sql | 22 ++ macros/get_product_variant_columns.sql | 36 +++ macros/get_refund_columns.sql | 17 ++ macros/get_transaction_columns.sql | 45 ++++ macros/staging_columns.sql | 287 ---------------------- models/stg_shopify__customer.sql | 50 +++- models/stg_shopify__order.sql | 110 ++++++++- models/stg_shopify__order_adjustment.sql | 25 +- models/stg_shopify__order_line.sql | 62 ++++- models/stg_shopify__order_line_refund.sql | 38 ++- models/tmp/stg_shopify__refund_tmp.sql | 26 +- 16 files changed, 590 insertions(+), 336 deletions(-) create mode 100644 macros/get_customer_columns.sql create mode 100644 macros/get_order_adjustment_columns.sql create mode 100644 macros/get_order_columns.sql create mode 100644 macros/get_order_line_columns.sql create mode 100644 macros/get_order_line_refund_columns.sql create mode 100644 macros/get_product_columns.sql create mode 100644 macros/get_product_variant_columns.sql create mode 100644 macros/get_refund_columns.sql create mode 100644 macros/get_transaction_columns.sql delete mode 100644 macros/staging_columns.sql diff --git a/macros/get_customer_columns.sql b/macros/get_customer_columns.sql new file mode 100644 index 0000000..6167587 --- /dev/null +++ b/macros/get_customer_columns.sql @@ -0,0 +1,33 @@ +{% macro get_customer_columns() %} + +{% set columns = [ + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "accepts_marketing", "datatype": "boolean"}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "default_address_id", "datatype": dbt.type_numeric()}, + {"name": "email", "datatype": dbt.type_string()}, + {"name": "first_name", "datatype": dbt.type_string()}, + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "last_name", "datatype": dbt.type_string()}, + {"name": "orders_count", "datatype": dbt.type_numeric()}, + {"name": "phone", "datatype": dbt.type_string()}, + {"name": "state", "datatype": dbt.type_string()}, + {"name": "tax_exempt", "datatype": "boolean"}, + {"name": "total_spent", "datatype": dbt.type_float()}, + {"name": "updated_at", "datatype": dbt.type_timestamp()}, + {"name": "verified_email", "datatype": "boolean"}, + {"name": "email_marketing_consent_consent_updated_at", "datatype": dbt.type_timestamp()}, + {"name": "email_marketing_consent_opt_in_level", "datatype": dbt.type_string()}, + {"name": "email_marketing_consent_state", "datatype": dbt.type_string()}, + {"name": "note", "datatype": dbt.type_string()}, + {"name": "accepts_marketing_updated_at", "datatype": dbt.type_timestamp()}, + {"name": "marketing_opt_in_level", "datatype": dbt.type_string()}, + {"name": "lifetime_duration", "datatype": dbt.type_string()}, + {"name": "currency", "datatype": dbt.type_string()} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('customer_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_order_adjustment_columns.sql b/macros/get_order_adjustment_columns.sql new file mode 100644 index 0000000..77b9028 --- /dev/null +++ b/macros/get_order_adjustment_columns.sql @@ -0,0 +1,18 @@ +{% macro get_order_adjustment_columns() %} + +{% set columns = [ + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "order_id", "datatype": dbt.type_numeric()}, + {"name": "refund_id", "datatype": dbt.type_numeric()}, + {"name": "amount", "datatype": dbt.type_float()}, + {"name": "amount_set", "datatype": dbt.type_string()}, + {"name": "tax_amount", "datatype": dbt.type_float()}, + {"name": "tax_amount_set", "datatype": dbt.type_string()}, + {"name": "kind", "datatype": dbt.type_string()}, + {"name": "reason", "datatype": dbt.type_string()}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()} +] %} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_order_columns.sql b/macros/get_order_columns.sql new file mode 100644 index 0000000..119d131 --- /dev/null +++ b/macros/get_order_columns.sql @@ -0,0 +1,93 @@ +{% macro get_order_columns() %} + +{% set columns = [ + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "processed_at", "datatype": dbt.type_timestamp()}, + {"name": "updated_at", "datatype": dbt.type_timestamp()}, + {"name": "user_id", "datatype": dbt.type_numeric()}, + {"name": "total_discounts", "datatype": dbt.type_float()}, + {"name": "total_discounts_set", "datatype": dbt.type_string()}, + {"name": "total_line_items_price", "datatype": dbt.type_float()}, + {"name": "total_line_items_price_set", "datatype": dbt.type_string()}, + {"name": "total_price", "datatype": dbt.type_float()}, + {"name": "total_price_set", "datatype": dbt.type_string()}, + {"name": "total_price_usd", "datatype": dbt.type_float()}, + {"name": "total_tax_set", "datatype": dbt.type_string()}, + {"name": "total_tax", "datatype": dbt.type_float()}, + {"name": "source_name", "datatype": dbt.type_string()}, + {"name": "subtotal_price", "datatype": dbt.type_float()}, + {"name": "taxes_included", "datatype": "boolean"}, + {"name": "total_weight", "datatype": dbt.type_numeric()}, + {"name": "total_tip_received", "datatype": dbt.type_float()}, + {"name": "landing_site_base_url", "datatype": dbt.type_string()}, + {"name": "landing_site_ref", "datatype": dbt.type_string()}, + {"name": "location_id", "datatype": dbt.type_numeric()}, + {"name": "name", "datatype": dbt.type_string()}, + {"name": "note", "datatype": dbt.type_string()}, + {"name": "number", "datatype": dbt.type_numeric()}, + {"name": "order_number", "datatype": dbt.type_numeric()}, + {"name": "cancel_reason", "datatype": dbt.type_string()}, + {"name": "cancelled_at", "datatype": dbt.type_timestamp()}, + {"name": "cart_token", "datatype": dbt.type_string()}, + {"name": "checkout_token", "datatype": dbt.type_string()}, + {"name": "closed_at", "datatype": dbt.type_timestamp()}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "currency", "datatype": dbt.type_string()}, + {"name": "customer_id", "datatype": dbt.type_numeric()}, + {"name": "email", "datatype": dbt.type_string()}, + {"name": "financial_status", "datatype": dbt.type_string()}, + {"name": "fulfillment_status", "datatype": dbt.type_string()}, + {"name": "processing_method", "datatype": dbt.type_string()}, + {"name": "referring_site", "datatype": dbt.type_string()}, + {"name": "billing_address_address_1", "datatype": dbt.type_string()}, + {"name": "billing_address_address_2", "datatype": dbt.type_string()}, + {"name": "billing_address_city", "datatype": dbt.type_string()}, + {"name": "billing_address_company", "datatype": dbt.type_string()}, + {"name": "billing_address_country", "datatype": dbt.type_string()}, + {"name": "billing_address_country_code", "datatype": dbt.type_string()}, + {"name": "billing_address_first_name", "datatype": dbt.type_string()}, + {"name": "billing_address_last_name", "datatype": dbt.type_string()}, + {"name": "billing_address_latitude", "datatype": dbt.type_string()}, + {"name": "billing_address_longitude", "datatype": dbt.type_string()}, + {"name": "billing_address_name", "datatype": dbt.type_string()}, + {"name": "billing_address_phone", "datatype": dbt.type_string()}, + {"name": "billing_address_province", "datatype": dbt.type_string()}, + {"name": "billing_address_province_code", "datatype": dbt.type_string()}, + {"name": "billing_address_zip", "datatype": dbt.type_string()}, + {"name": "browser_ip", "datatype": dbt.type_string()}, + {"name": "buyer_accepts_marketing", "datatype": "boolean"}, + {"name": "total_shipping_price_set", "datatype": dbt.type_string()}, + {"name": "shipping_address_address_1", "datatype": dbt.type_string()}, + {"name": "shipping_address_address_2", "datatype": dbt.type_string()}, + {"name": "shipping_address_city", "datatype": dbt.type_string()}, + {"name": "shipping_address_company", "datatype": dbt.type_string()}, + {"name": "shipping_address_country", "datatype": dbt.type_string()}, + {"name": "shipping_address_country_code", "datatype": dbt.type_string()}, + {"name": "shipping_address_first_name", "datatype": dbt.type_string()}, + {"name": "shipping_address_last_name", "datatype": dbt.type_string()}, + {"name": "shipping_address_latitude", "datatype": dbt.type_string()}, + {"name": "shipping_address_longitude", "datatype": dbt.type_string()}, + {"name": "shipping_address_name", "datatype": dbt.type_string()}, + {"name": "shipping_address_phone", "datatype": dbt.type_string()}, + {"name": "shipping_address_province", "datatype": dbt.type_string()}, + {"name": "shipping_address_province_code", "datatype": dbt.type_string()}, + {"name": "shipping_address_zip", "datatype": dbt.type_string()}, + {"name": "test", "datatype": "boolean", "alias": "is_test_order"}, + {"name": "token", "datatype": dbt.type_string()}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "_fivetran_deleted", "datatype": "boolean"}, + {"name": "app_id", "datatype": dbt.type_int()}, + {"name": "checkout_id", "datatype": dbt.type_int()}, + {"name": "client_details_user_agent", "datatype": dbt.type_string()}, + {"name": "customer_locale", "datatype": dbt.type_string()}, + {"name": "device_id", "datatype": dbt.type_string()}, + {"name": "order_status_url", "datatype": dbt.type_string()}, + {"name": "presentment_currency", "datatype": dbt.type_string()}, + {"name": "confirmed", "datatype": "boolean"} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('order_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_order_line_columns.sql b/macros/get_order_line_columns.sql new file mode 100644 index 0000000..574cc72 --- /dev/null +++ b/macros/get_order_line_columns.sql @@ -0,0 +1,43 @@ +{% macro get_order_line_columns() %} + +{% set columns = [ + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "fulfillable_quantity", "datatype": dbt.type_numeric()}, + {"name": "fulfillment_service", "datatype": dbt.type_string()}, + {"name": "fulfillment_status", "datatype": dbt.type_string()}, + {"name": "gift_card", "datatype": "boolean"}, + {"name": "grams", "datatype": dbt.type_numeric()}, + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "index", "datatype": dbt.type_numeric()}, + {"name": "name", "datatype": dbt.type_string()}, + {"name": "order_id", "datatype": dbt.type_numeric()}, + {"name": "pre_tax_price", "datatype": dbt.type_float()}, + {"name": "pre_tax_price_set", "datatype": dbt.type_string()}, + {"name": "price", "datatype": dbt.type_float()}, + {"name": "price_set", "datatype": dbt.type_string()}, + {"name": "product_id", "datatype": dbt.type_numeric()}, + {"name": "property_charge_interval_frequency", "datatype": dbt.type_numeric()}, + {"name": "property_for_shipping_jan_3_rd_2020", "datatype": dbt.type_string()}, + {"name": "property_shipping_interval_frequency", "datatype": dbt.type_numeric()}, + {"name": "property_shipping_interval_unit_type", "datatype": dbt.type_string()}, + {"name": "property_subscription_id", "datatype": dbt.type_numeric()}, + {"name": "quantity", "datatype": dbt.type_numeric()}, + {"name": "requires_shipping", "datatype": "boolean"}, + {"name": "sku", "datatype": dbt.type_string()}, + {"name": "taxable", "datatype": "boolean"}, + {"name": "tax_code", "datatype": dbt.type_string()}, + {"name": "title", "datatype": dbt.type_string()}, + {"name": "total_discount", "datatype": dbt.type_float()}, + {"name": "total_discount_set", "datatype": dbt.type_string()}, + {"name": "variant_id", "datatype": dbt.type_numeric()}, + {"name": "variant_title", "datatype": dbt.type_string()}, + {"name": "variant_inventory_management", "datatype": dbt.type_string()}, + {"name": "vendor", "datatype": dbt.type_string()}, + {"name": "properties", "datatype": dbt.type_string()} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('order_line_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_order_line_refund_columns.sql b/macros/get_order_line_refund_columns.sql new file mode 100644 index 0000000..6cd8afa --- /dev/null +++ b/macros/get_order_line_refund_columns.sql @@ -0,0 +1,21 @@ +{% macro get_order_line_refund_columns() %} + +{% set columns = [ + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "id", "datatype": dbt.type_numeric(), "alias": "order_line_refund_id"}, + {"name": "location_id", "datatype": dbt.type_numeric()}, + {"name": "order_line_id", "datatype": dbt.type_numeric()}, + {"name": "subtotal", "datatype": dbt.type_numeric()}, + {"name": "subtotal_set", "datatype": dbt.type_string()}, + {"name": "total_tax", "datatype": dbt.type_numeric()}, + {"name": "total_tax_set", "datatype": dbt.type_string()}, + {"name": "quantity", "datatype": dbt.type_float()}, + {"name": "refund_id", "datatype": dbt.type_numeric()}, + {"name": "restock_type", "datatype": dbt.type_string()} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('order_line_refund_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_product_columns.sql b/macros/get_product_columns.sql new file mode 100644 index 0000000..9d49830 --- /dev/null +++ b/macros/get_product_columns.sql @@ -0,0 +1,22 @@ +{% macro get_product_columns() %} + +{% set columns = [ + {"name": "_fivetran_deleted", "datatype": "boolean"}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, + {"name": "handle", "datatype": dbt.type_string()}, + {"name": "id", "datatype": dbt.type_numeric(), "alias": "product_id"}, + {"name": "product_type", "datatype": dbt.type_string()}, + {"name": "published_at", "datatype": dbt.type_timestamp(), "alias": "published_timestamp"}, + {"name": "published_scope", "datatype": dbt.type_string()}, + {"name": "title", "datatype": dbt.type_string()}, + {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, + {"name": "vendor", "datatype": dbt.type_string()}, + {"name": "status", "datatype": dbt.type_string()} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('product_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_product_variant_columns.sql b/macros/get_product_variant_columns.sql new file mode 100644 index 0000000..80e5c00 --- /dev/null +++ b/macros/get_product_variant_columns.sql @@ -0,0 +1,36 @@ +{% macro get_product_variant_columns() %} + +{% set columns = [ + {"name": "id", "datatype": dbt.type_numeric(), "alias": "variant_id"}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, + {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, + {"name": "product_id", "datatype": dbt.type_numeric()}, + {"name": "inventory_item_id", "datatype": dbt.type_numeric()}, + {"name": "image_id", "datatype": dbt.type_numeric()}, + {"name": "title", "datatype": dbt.type_string()}, + {"name": "price", "datatype": dbt.type_float()}, + {"name": "sku", "datatype": dbt.type_string()}, + {"name": "position", "datatype": dbt.type_numeric()}, + {"name": "inventory_policy", "datatype": dbt.type_string()}, + {"name": "compare_at_price", "datatype": dbt.type_float()}, + {"name": "fulfillment_service", "datatype": dbt.type_string()}, + {"name": "inventory_management", "datatype": dbt.type_string()}, + {"name": "taxable", "datatype": "boolean", "alias": "is_taxable"}, + {"name": "barcode", "datatype": dbt.type_string()}, + {"name": "grams", "datatype": dbt.type_float()}, + {"name": "inventory_quantity", "datatype": dbt.type_numeric()}, + {"name": "weight", "datatype": dbt.type_float()}, + {"name": "weight_unit", "datatype": dbt.type_string()}, + {"name": "option_1", "datatype": dbt.type_string()}, + {"name": "option_2", "datatype": dbt.type_string()}, + {"name": "option_3", "datatype": dbt.type_string()}, + {"name": "tax_code", "datatype": dbt.type_string()}, + {"name": "requires_shipping", "datatype": "boolean", "alias": "is_requiring_shipping"} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('product_variant_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_refund_columns.sql b/macros/get_refund_columns.sql new file mode 100644 index 0000000..8460132 --- /dev/null +++ b/macros/get_refund_columns.sql @@ -0,0 +1,17 @@ +{% macro get_refund_columns() %} + +{% set columns = [ + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "id", "datatype": dbt.type_numeric(), "alias": "refund_id"}, + {"name": "note", "datatype": dbt.type_string()}, + {"name": "order_id", "datatype": dbt.type_numeric()}, + {"name": "processed_at", "datatype": dbt.type_timestamp()}, + {"name": "restock", "datatype": "boolean"}, + {"name": "total_duties_set", "datatype": dbt.type_string()}, + {"name": "user_id", "datatype": dbt.type_numeric()} +] %} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_transaction_columns.sql b/macros/get_transaction_columns.sql new file mode 100644 index 0000000..0e7d9ab --- /dev/null +++ b/macros/get_transaction_columns.sql @@ -0,0 +1,45 @@ +{% macro get_transaction_columns() %} + +{% set columns = [ + {"name": "id", "datatype": dbt.type_numeric(), "alias": "transaction_id"}, + {"name": "order_id", "datatype": dbt.type_numeric()}, + {"name": "refund_id", "datatype": dbt.type_numeric()}, + {"name": "amount", "datatype": dbt.type_numeric()}, + {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, + {"name": "processed_at", "datatype": dbt.type_timestamp(), "alias": "processed_timestamp"}, + {"name": "device_id", "datatype": dbt.type_numeric()}, + {"name": "gateway", "datatype": dbt.type_string()}, + {"name": "source_name", "datatype": dbt.type_string()}, + {"name": "message", "datatype": dbt.type_string()}, + {"name": "currency", "datatype": dbt.type_string()}, + {"name": "location_id", "datatype": dbt.type_numeric()}, + {"name": "parent_id", "datatype": dbt.type_numeric()}, + {"name": "payment_avs_result_code", "datatype": dbt.type_string()}, + {"name": "payment_credit_card_bin", "datatype": dbt.type_string()}, + {"name": "payment_cvv_result_code", "datatype": dbt.type_string()}, + {"name": "payment_credit_card_number", "datatype": dbt.type_string()}, + {"name": "payment_credit_card_company", "datatype": dbt.type_string()}, + {"name": "kind", "datatype": dbt.type_string()}, + {"name": "receipt", "datatype": dbt.type_string()}, + {"name": "currency_exchange_id", "datatype": dbt.type_numeric()}, + {"name": "currency_exchange_adjustment", "datatype": dbt.type_numeric()}, + {"name": "currency_exchange_original_amount", "datatype": dbt.type_numeric()}, + {"name": "currency_exchange_final_amount", "datatype": dbt.type_numeric()}, + {"name": "currency_exchange_currency", "datatype": dbt.type_string()}, + {"name": "error_code", "datatype": dbt.type_string()}, + {"name": "status", "datatype": dbt.type_string()}, + {"name": "test", "datatype": "boolean"}, + {"name": "user_id", "datatype": dbt.type_numeric()}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "authorization_expires_at", "datatype": dbt.type_timestamp()} +] %} + +{% if target.type in ('redshift','postgres') %} + {{ columns.append({"name": "authorization", "datatype": dbt.type_string(), "quote": True, "alias": "authorization"}) }} +{% else %} + {"name": "authorization", "datatype": dbt.type_string()} +{% endif %} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/staging_columns.sql b/macros/staging_columns.sql deleted file mode 100644 index a147538..0000000 --- a/macros/staging_columns.sql +++ /dev/null @@ -1,287 +0,0 @@ -{% macro get_order_columns() %} - -{% set columns = [ - {"name": "id", "datatype": dbt.type_numeric(), "alias": "order_id"}, - {"name": "processed_at", "datatype": dbt.type_timestamp(), "alias": "processed_timestamp"}, - {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, - {"name": "user_id", "datatype": dbt.type_numeric()}, - {"name": "total_discounts", "datatype": dbt.type_float()}, - {"name": "total_line_items_price", "datatype": dbt.type_float()}, - {"name": "total_price", "datatype": dbt.type_float()}, - {"name": "total_tax", "datatype": dbt.type_float()}, - {"name": "source_name", "datatype": dbt.type_string()}, - {"name": "subtotal_price", "datatype": dbt.type_float()}, - {"name": "taxes_included", "datatype": "boolean", "alias": "has_taxes_included"}, - {"name": "total_weight", "datatype": dbt.type_numeric()}, - {"name": "landing_site_base_url", "datatype": dbt.type_string()}, - {"name": "location_id", "datatype": dbt.type_numeric()}, - {"name": "name", "datatype": dbt.type_string()}, - {"name": "note", "datatype": dbt.type_string()}, - {"name": "number", "datatype": dbt.type_numeric()}, - {"name": "order_number", "datatype": dbt.type_numeric()}, - {"name": "cancel_reason", "datatype": dbt.type_string()}, - {"name": "cancelled_at", "datatype": dbt.type_timestamp(), "alias": "cancelled_timestamp"}, - {"name": "cart_token", "datatype": dbt.type_string()}, - {"name": "checkout_token", "datatype": dbt.type_string()}, - {"name": "closed_at", "datatype": dbt.type_timestamp(), "alias": "closed_timestamp"}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "currency", "datatype": dbt.type_string()}, - {"name": "customer_id", "datatype": dbt.type_numeric()}, - {"name": "email", "datatype": dbt.type_string()}, - {"name": "financial_status", "datatype": dbt.type_string()}, - {"name": "fulfillment_status", "datatype": dbt.type_string()}, - {"name": "processing_method", "datatype": dbt.type_string()}, - {"name": "referring_site", "datatype": dbt.type_string()}, - {"name": "billing_address_address_1", "datatype": dbt.type_string()}, - {"name": "billing_address_address_2", "datatype": dbt.type_string()}, - {"name": "billing_address_city", "datatype": dbt.type_string()}, - {"name": "billing_address_company", "datatype": dbt.type_string()}, - {"name": "billing_address_country", "datatype": dbt.type_string()}, - {"name": "billing_address_country_code", "datatype": dbt.type_string()}, - {"name": "billing_address_first_name", "datatype": dbt.type_string()}, - {"name": "billing_address_last_name", "datatype": dbt.type_string()}, - {"name": "billing_address_latitude", "datatype": dbt.type_string()}, - {"name": "billing_address_longitude", "datatype": dbt.type_string()}, - {"name": "billing_address_name", "datatype": dbt.type_string()}, - {"name": "billing_address_phone", "datatype": dbt.type_string()}, - {"name": "billing_address_province", "datatype": dbt.type_string()}, - {"name": "billing_address_province_code", "datatype": dbt.type_string()}, - {"name": "billing_address_zip", "datatype": dbt.type_string()}, - {"name": "browser_ip", "datatype": dbt.type_string()}, - {"name": "buyer_accepts_marketing", "datatype": "boolean", "alias": "has_buyer_accepted_marketing"}, - {"name": "total_shipping_price_set", "datatype": dbt.type_string()}, - {"name": "shipping_address_address_1", "datatype": dbt.type_string()}, - {"name": "shipping_address_address_2", "datatype": dbt.type_string()}, - {"name": "shipping_address_city", "datatype": dbt.type_string()}, - {"name": "shipping_address_company", "datatype": dbt.type_string()}, - {"name": "shipping_address_country", "datatype": dbt.type_string()}, - {"name": "shipping_address_country_code", "datatype": dbt.type_string()}, - {"name": "shipping_address_first_name", "datatype": dbt.type_string()}, - {"name": "shipping_address_last_name", "datatype": dbt.type_string()}, - {"name": "shipping_address_latitude", "datatype": dbt.type_string()}, - {"name": "shipping_address_longitude", "datatype": dbt.type_string()}, - {"name": "shipping_address_name", "datatype": dbt.type_string()}, - {"name": "shipping_address_phone", "datatype": dbt.type_string()}, - {"name": "shipping_address_province", "datatype": dbt.type_string()}, - {"name": "shipping_address_province_code", "datatype": dbt.type_string()}, - {"name": "shipping_address_zip", "datatype": dbt.type_string()}, - {"name": "test", "datatype": "boolean", "alias": "is_test_order"}, - {"name": "token", "datatype": dbt.type_string()}, - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_customer_columns() %} - -{% set columns = [ - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "accepts_marketing", "datatype": "boolean", "alias": "has_accepted_marketing"}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "default_address_id", "datatype": dbt.type_numeric()}, - {"name": "email", "datatype": dbt.type_string()}, - {"name": "first_name", "datatype": dbt.type_string()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "customer_id"}, - {"name": "last_name", "datatype": dbt.type_string()}, - {"name": "orders_count", "datatype": dbt.type_numeric()}, - {"name": "phone", "datatype": dbt.type_string()}, - {"name": "state", "datatype": dbt.type_string(), "alias": "account_state"}, - {"name": "tax_exempt", "datatype": "boolean", "alias": "is_tax_exempt"}, - {"name": "total_spent", "datatype": dbt.type_float()}, - {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, - {"name": "verified_email", "datatype": "boolean", "alias": "is_verified_email"} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_order_line_refund_columns() %} - -{% set columns = [ - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "order_line_refund_id"}, - {"name": "location_id", "datatype": dbt.type_numeric()}, - {"name": "order_line_id", "datatype": dbt.type_numeric()}, - {"name": "subtotal", "datatype": dbt.type_numeric()}, - {"name": "total_tax", "datatype": dbt.type_numeric()}, - {"name": "quantity", "datatype": dbt.type_float()}, - {"name": "refund_id", "datatype": dbt.type_numeric()}, - {"name": "restock_type", "datatype": dbt.type_string()} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_order_line_columns() %} - -{% set columns = [ - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "fulfillable_quantity", "datatype": dbt.type_numeric()}, - {"name": "fulfillment_service", "datatype": dbt.type_string()}, - {"name": "fulfillment_status", "datatype": dbt.type_string()}, - {"name": "gift_card", "datatype": "boolean", "alias": "is_gift_card"}, - {"name": "grams", "datatype": dbt.type_numeric()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "order_line_id"}, - {"name": "index", "datatype": dbt.type_numeric()}, - {"name": "name", "datatype": dbt.type_string()}, - {"name": "order_id", "datatype": dbt.type_numeric()}, - {"name": "pre_tax_price", "datatype": dbt.type_float()}, - {"name": "price", "datatype": dbt.type_float()}, - {"name": "product_id", "datatype": dbt.type_numeric()}, - {"name": "property_charge_interval_frequency", "datatype": dbt.type_numeric()}, - {"name": "property_for_shipping_jan_3_rd_2020", "datatype": dbt.type_string()}, - {"name": "property_shipping_interval_frequency", "datatype": dbt.type_numeric()}, - {"name": "property_shipping_interval_unit_type", "datatype": dbt.type_string()}, - {"name": "property_subscription_id", "datatype": dbt.type_numeric()}, - {"name": "quantity", "datatype": dbt.type_numeric()}, - {"name": "requires_shipping", "datatype": "boolean", "alias": "is_requiring_shipping"}, - {"name": "sku", "datatype": dbt.type_string()}, - {"name": "taxable", "datatype": "boolean", "alias": "is_taxable"}, - {"name": "title", "datatype": dbt.type_string()}, - {"name": "total_discount", "datatype": dbt.type_float()}, - {"name": "variant_id", "datatype": dbt.type_numeric()}, - {"name": "vendor", "datatype": dbt.type_string()} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_product_columns() %} - -{% set columns = [ - {"name": "_fivetran_deleted", "datatype": "boolean"}, - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "handle", "datatype": dbt.type_string()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "product_id"}, - {"name": "product_type", "datatype": dbt.type_string()}, - {"name": "published_at", "datatype": dbt.type_timestamp(), "alias": "published_timestamp"}, - {"name": "published_scope", "datatype": dbt.type_string()}, - {"name": "title", "datatype": dbt.type_string()}, - {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, - {"name": "vendor", "datatype": dbt.type_string()} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_product_variant_columns() %} - -{% set columns = [ - {"name": "id", "datatype": dbt.type_numeric(), "alias": "variant_id"}, - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, - {"name": "product_id", "datatype": dbt.type_numeric()}, - {"name": "inventory_item_id", "datatype": dbt.type_numeric()}, - {"name": "image_id", "datatype": dbt.type_numeric()}, - {"name": "title", "datatype": dbt.type_string()}, - {"name": "price", "datatype": dbt.type_float()}, - {"name": "sku", "datatype": dbt.type_string()}, - {"name": "position", "datatype": dbt.type_numeric()}, - {"name": "inventory_policy", "datatype": dbt.type_string()}, - {"name": "compare_at_price", "datatype": dbt.type_float()}, - {"name": "fulfillment_service", "datatype": dbt.type_string()}, - {"name": "inventory_management", "datatype": dbt.type_string()}, - {"name": "taxable", "datatype": "boolean", "alias": "is_taxable"}, - {"name": "barcode", "datatype": dbt.type_string()}, - {"name": "grams", "datatype": dbt.type_float()}, - {"name": "inventory_quantity", "datatype": dbt.type_numeric()}, - {"name": "weight", "datatype": dbt.type_float()}, - {"name": "weight_unit", "datatype": dbt.type_string()}, - {"name": "option_1", "datatype": dbt.type_string()}, - {"name": "option_2", "datatype": dbt.type_string()}, - {"name": "option_3", "datatype": dbt.type_string()}, - {"name": "tax_code", "datatype": dbt.type_string()}, - {"name": "old_inventory_quantity", "datatype": dbt.type_numeric()}, - {"name": "requires_shipping", "datatype": "boolean", "alias": "is_requiring_shipping"} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_transaction_columns() %} - -{% set columns = [ - {"name": "id", "datatype": dbt.type_numeric(), "alias": "transaction_id"}, - {"name": "order_id", "datatype": dbt.type_numeric()}, - {"name": "refund_id", "datatype": dbt.type_numeric()}, - {"name": "amount", "datatype": dbt.type_numeric()}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "processed_at", "datatype": dbt.type_timestamp(), "alias": "processed_timestamp"}, - {"name": "device_id", "datatype": dbt.type_numeric()}, - {"name": "gateway", "datatype": dbt.type_string()}, - {"name": "source_name", "datatype": dbt.type_string()}, - {"name": "message", "datatype": dbt.type_string()}, - {"name": "currency", "datatype": dbt.type_string()}, - {"name": "location_id", "datatype": dbt.type_numeric()}, - {"name": "parent_id", "datatype": dbt.type_numeric()}, - {"name": "payment_avs_result_code", "datatype": dbt.type_string()}, - {"name": "payment_credit_card_bin", "datatype": dbt.type_string()}, - {"name": "payment_cvv_result_code", "datatype": dbt.type_string()}, - {"name": "payment_credit_card_number", "datatype": dbt.type_string()}, - {"name": "payment_credit_card_company", "datatype": dbt.type_string()}, - {"name": "kind", "datatype": dbt.type_string()}, - {"name": "receipt", "datatype": dbt.type_string()}, - {"name": "currency_exchange_id", "datatype": dbt.type_numeric()}, - {"name": "currency_exchange_adjustment", "datatype": dbt.type_numeric()}, - {"name": "currency_exchange_original_amount", "datatype": dbt.type_numeric()}, - {"name": "currency_exchange_final_amount", "datatype": dbt.type_numeric()}, - {"name": "currency_exchange_currency", "datatype": dbt.type_string()}, - {"name": "error_code", "datatype": dbt.type_string()}, - {"name": "status", "datatype": dbt.type_string()}, - {"name": "test", "datatype": "boolean"}, - {"name": "user_id", "datatype": dbt.type_numeric()}, - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()} -] %} - -{% if target.type in ('redshift','postgres') %} - {{ columns.append({"name": "authorization", "datatype": dbt.type_string(), "quote": True, "alias": "authorization"}) }} -{% else %} - {"name": "authorization", "datatype": dbt.type_string()} -{% endif %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_refund_columns() %} - -{% set columns = [ - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "created_at", "datatype": dbt.type_timestamp()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "refund_id"}, - {"name": "note", "datatype": dbt.type_string()}, - {"name": "order_id", "datatype": dbt.type_numeric()}, - {"name": "processed_at", "datatype": dbt.type_timestamp()}, - {"name": "restock", "datatype": "boolean"}, - {"name": "user_id", "datatype": dbt.type_numeric()} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_order_adjustment_columns() %} - -{% set columns = [ - {"name": "id", "datatype": dbt.type_numeric(), "alias": "order_adjustment_id"}, - {"name": "order_id", "datatype": dbt.type_numeric()}, - {"name": "refund_id", "datatype": dbt.type_numeric()}, - {"name": "amount", "datatype": dbt.type_float()}, - {"name": "tax_amount", "datatype": dbt.type_float()}, - {"name": "kind", "datatype": dbt.type_string()}, - {"name": "reason", "datatype": dbt.type_string()}, - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()} -] %} - -{{ return(columns) }} - -{% endmacro %} diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql index 32e943c..3e2262b 100644 --- a/models/stg_shopify__customer.sql +++ b/models/stg_shopify__customer.sql @@ -1,10 +1,11 @@ -with source as ( +with base as ( - select * from {{ ref('stg_shopify__customer_tmp') }} + select * + from {{ ref('stg_shopify__customer_tmp') }} ), -renamed as ( +fields as ( select @@ -15,21 +16,46 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('customer_pass_through_columns') %} - , - {{ var('customer_pass_through_columns') | join (", ")}} - - {% endif %} {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source + from base -) +), -select * from renamed +final as ( + + select + coalesce(accepts_marketing, email_marketing_consent_state) as has_accepted_marketing, + created_at as created_timestamp, + default_address_id, + email, + first_name, + id as customer_id, + last_name, + orders_count, + phone, + state as account_state, + tax_exempt as is_tax_exempt, + total_spent, + updated_at as updated_timestamp, + verified_email as is_verified_email, + note, + coalesce(accepts_marketing_updated_at, email_marketing_consent_consent_updated_at) as accepts_marketing_updated_at, + coalesce(marketing_opt_in_level, email_marketing_consent_opt_in_level) as marketing_opt_in_level, + lifetime_duration, + currency, + source_relation, + _fivetran_synced + + {{ fivetran_utils.fill_pass_through_columns('customer_pass_through_columns') }} + + from fields + +) +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__order.sql b/models/stg_shopify__order.sql index 17798dd..49197f5 100644 --- a/models/stg_shopify__order.sql +++ b/models/stg_shopify__order.sql @@ -1,10 +1,11 @@ -with source as ( +with base as ( - select * from {{ ref('stg_shopify__order_tmp') }} + select * + from {{ ref('stg_shopify__order_tmp') }} ), -renamed as ( +fields as ( select @@ -15,13 +16,6 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('order_pass_through_columns') %} - , - {{ var('order_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') @@ -29,6 +23,100 @@ renamed as ( from source +), + +final as ( + + select + id as order_id, + processed_at as processed_timestamp, + updated_at as updated_timestamp, + user_id, + total_discounts, + total_discounts_set, + total_line_items_price, + total_line_items_price_set, + total_price, + total_price_set, + total_price_usd, + total_tax_set, + total_tax, + source_name, + subtotal_price, + taxes_included as has_taxes_included, + total_weight, + total_tip_received, + landing_site_base_url, + landing_site_ref, + location_id, + name, + note, + number, + order_number, + cancel_reason, + cancelled_at as cancelled_timestamp, + cart_token, + checkout_token, + closed_at as closed_timestamp, + created_at as created_timestamp, + currency, + customer_id, + email, + financial_status, + fulfillment_status, + processing_method, + referring_site, + billing_address_address_1, + billing_address_address_2, + billing_address_city, + billing_address_company, + billing_address_country, + billing_address_country_code, + billing_address_first_name, + billing_address_last_name, + billing_address_latitude, + billing_address_longitude, + billing_address_name, + billing_address_phone, + billing_address_province, + billing_address_province_code, + billing_address_zip, + browser_ip, + buyer_accepts_marketing as has_buyer_accepted_marketing, + total_shipping_price_set, + shipping_address_address_1, + shipping_address_address_2, + shipping_address_city, + shipping_address_company, + shipping_address_country, + shipping_address_country_code, + shipping_address_first_name, + shipping_address_last_name, + shipping_address_latitude, + shipping_address_longitude, + shipping_address_name, + shipping_address_phone, + shipping_address_province, + shipping_address_province_code, + shipping_address_zip, + test, + token, + _fivetran_synced, + _fivetran_deleted, + app_id, + checkout_id, + client_details_user_agent, + customer_locale, + device_id, + order_status_url, + presentment_currency, + confirmed, + source_relation + + {{ fivetran_utils.fill_pass_through_columns('order_pass_through_columns') }} + + from fields ) -select * from renamed \ No newline at end of file +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__order_adjustment.sql b/models/stg_shopify__order_adjustment.sql index 9570f35..fbd59a6 100644 --- a/models/stg_shopify__order_adjustment.sql +++ b/models/stg_shopify__order_adjustment.sql @@ -1,14 +1,14 @@ --To disable this model, set the shopify__using_order_adjustment variable within your dbt_project.yml file to False. {{ config(enabled=var('shopify__using_order_adjustment', True)) }} -with source as ( +with base as ( select * from {{ ref('stg_shopify__order_adjustment_tmp') }} ), -renamed as ( +fields as ( select {{ @@ -24,6 +24,25 @@ renamed as ( }} from source +), + +final as ( + + select + id as order_adjustment_id, + order_id, + refund_id, + amount, + amount_set, + tax_amount, + tax_amount_set, + kind, + reason, + source_relation, + _fivetran_synced + + from fields ) -select * from renamed +select * +from final diff --git a/models/stg_shopify__order_line.sql b/models/stg_shopify__order_line.sql index ab0297f..4811e1e 100644 --- a/models/stg_shopify__order_line.sql +++ b/models/stg_shopify__order_line.sql @@ -1,10 +1,11 @@ -with source as ( +with base as ( - select * from {{ ref('stg_shopify__order_line_tmp') }} + select * + from {{ ref('stg_shopify__order_line_tmp') }} ), -renamed as ( +fields as ( select @@ -15,13 +16,6 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('order_line_pass_through_columns') %} - , - {{ var('order_line_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') @@ -29,7 +23,53 @@ renamed as ( from source +), + +final as ( + + select + + fulfillable_quantity, + fulfillment_service, + fulfillment_status, + gift_card as is_gift_card, + grams, + id as order_line_id, + index, + name, + order_id, + pre_tax_price, + pre_tax_price_set, + price, + price_set, + product_id, + property_charge_interval_frequency, + property_for_shipping_jan_3_rd_2020, + property_shipping_interval_frequency, + property_shipping_interval_unit_type, + property_subscription_id, + quantity, + requires_shipping as is_shipping_required, + sku, + taxable as is_taxable, + tax_code, + title, + total_discount, + total_discount_set, + variant_id, + variant_title, + variant_inventory_management, + vendor, + properties, + source_relation, + _fivetran_synced + + {{ fivetran_utils.fill_pass_through_columns('order_line_pass_through_columns') }} + + from fields + ) -select * from renamed +select * +from final diff --git a/models/stg_shopify__order_line_refund.sql b/models/stg_shopify__order_line_refund.sql index f1ad8bb..7d20b1a 100644 --- a/models/stg_shopify__order_line_refund.sql +++ b/models/stg_shopify__order_line_refund.sql @@ -1,13 +1,14 @@ --To disable this model, set the shopify__using_order_line_refund variable within your dbt_project.yml file to False. {{ config(enabled=var('shopify__using_order_line_refund', True)) }} -with source as ( +with base as ( - select * from {{ ref('stg_shopify__order_line_refund_tmp') }} + select * + from {{ ref('stg_shopify__order_line_refund_tmp') }} ), -renamed as ( +fields as ( select @@ -18,13 +19,6 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('order_line_refund_pass_through_columns') %} - , - {{ var('order_line_refund_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') @@ -32,7 +26,29 @@ renamed as ( from source +), + +final as ( + + select + id as order_line_refund_id, + location_id, + order_line_id, + subtotal, + subtotal_set, + total_tax, + total_tax_set, + quantity, + refund_id, + restock_type, + _fivetran_synced, + source_relation, + + {{ fivetran_utils.fill_pass_through_columns('order_line_refund_pass_through_columns') }} + + from fields ) -select * from renamed +select * +from final diff --git a/models/tmp/stg_shopify__refund_tmp.sql b/models/tmp/stg_shopify__refund_tmp.sql index 80de149..c3eff2d 100644 --- a/models/tmp/stg_shopify__refund_tmp.sql +++ b/models/tmp/stg_shopify__refund_tmp.sql @@ -1,6 +1,15 @@ --To disable this model, set the shopify__using_refund variable within your dbt_project.yml file to False. {{ config(enabled=var('shopify__using_refund', True)) }} +{%- set source_relation = adapter.get_relation( + database=source('shopify', 'refund').database, + schema=source('shopify', 'refund').schema, + identifier=source('shopify', 'refund').name) -%} + +{% set table_exists=source_relation is not none %} + +{% if table_exists %} + {{ fivetran_utils.union_data( table_identifier='refund', @@ -12,4 +21,19 @@ union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) -}} \ No newline at end of file +}} + +{% else %} + +select + cast(null as {{ dbt_utils.type_timestamp() }}) as _fivetran_synced, + cast(null as {{ dbt_utils.type_timestamp() }}) as created_at, + cast(null as {{ dbt_utils.type_numeric() }}) as id, + cast(null as {{ dbt_utils.type_string() }}) as note, + cast(null as {{ dbt_utils.type_numeric() }}) as order_id, + cast(null as {{ dbt_utils.type_timestamp() }}) as processed_at, + cast(null as boolean) as restock, + cast(null as {{ dbt_utils.type_string() }}) as total_duties_set, + cast(null as {{ dbt_utils.type_numeric() }}) as user_id + +{% endif %} \ No newline at end of file From 785c2ee302c57cb0a5e5ef42dcbe47652ba48d46 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 12 Dec 2022 17:07:26 -0800 Subject: [PATCH 2/8] lotsa work --- CHANGELOG.md | 6 +++ macros/get_order_columns.sql | 2 +- macros/get_order_line_refund_columns.sql | 2 +- macros/get_product_columns.sql | 8 +-- macros/get_product_variant_columns.sql | 11 ++--- macros/get_refund_columns.sql | 2 +- macros/get_transaction_columns.sql | 8 +-- models/stg_shopify.yml | 4 +- models/stg_shopify__customer.sql | 8 +-- models/stg_shopify__order.sql | 14 +++--- models/stg_shopify__order_adjustment.sql | 4 +- models/stg_shopify__order_line.sql | 5 +- models/stg_shopify__order_line_refund.sql | 5 +- models/stg_shopify__product.sql | 43 ++++++++++++----- models/stg_shopify__product_variant.sql | 54 ++++++++++++++++----- models/stg_shopify__refund.sql | 26 ++++++++-- models/stg_shopify__transaction.sql | 59 ++++++++++++++++++----- models/tmp/stg_shopify__refund_tmp.sql | 17 ++++--- packages.yml | 7 ++- 19 files changed, 195 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ee4f1b..96928a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,12 @@ - `dbt.current_timestamp_backcompat` - `dbt.current_timestamp_in_utc_backcompat` - Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`. +- In the following tables, `*_timestamp` columns have been renamed to `*_at` for consistency: + - `stg_shopify__customer` + - `stg_shopify__order` + - `stg_shopify__product_variant` + - `stg_shopify__product` + - `stg_shopify__transaction` # dbt_shopify_source v0.6.0 🎉 dbt v1.0.0 Compatibility 🎉 diff --git a/macros/get_order_columns.sql b/macros/get_order_columns.sql index 119d131..c3b9a16 100644 --- a/macros/get_order_columns.sql +++ b/macros/get_order_columns.sql @@ -72,7 +72,7 @@ {"name": "shipping_address_province", "datatype": dbt.type_string()}, {"name": "shipping_address_province_code", "datatype": dbt.type_string()}, {"name": "shipping_address_zip", "datatype": dbt.type_string()}, - {"name": "test", "datatype": "boolean", "alias": "is_test_order"}, + {"name": "test", "datatype": "boolean"}, {"name": "token", "datatype": dbt.type_string()}, {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, {"name": "_fivetran_deleted", "datatype": "boolean"}, diff --git a/macros/get_order_line_refund_columns.sql b/macros/get_order_line_refund_columns.sql index 6cd8afa..818757b 100644 --- a/macros/get_order_line_refund_columns.sql +++ b/macros/get_order_line_refund_columns.sql @@ -2,7 +2,7 @@ {% set columns = [ {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "order_line_refund_id"}, + {"name": "id", "datatype": dbt.type_numeric()}, {"name": "location_id", "datatype": dbt.type_numeric()}, {"name": "order_line_id", "datatype": dbt.type_numeric()}, {"name": "subtotal", "datatype": dbt.type_numeric()}, diff --git a/macros/get_product_columns.sql b/macros/get_product_columns.sql index 9d49830..28c2254 100644 --- a/macros/get_product_columns.sql +++ b/macros/get_product_columns.sql @@ -3,14 +3,14 @@ {% set columns = [ {"name": "_fivetran_deleted", "datatype": "boolean"}, {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, {"name": "handle", "datatype": dbt.type_string()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "product_id"}, + {"name": "id", "datatype": dbt.type_numeric()}, {"name": "product_type", "datatype": dbt.type_string()}, - {"name": "published_at", "datatype": dbt.type_timestamp(), "alias": "published_timestamp"}, + {"name": "published_at", "datatype": dbt.type_timestamp()}, {"name": "published_scope", "datatype": dbt.type_string()}, {"name": "title", "datatype": dbt.type_string()}, - {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, + {"name": "updated_at", "datatype": dbt.type_timestamp()}, {"name": "vendor", "datatype": dbt.type_string()}, {"name": "status", "datatype": dbt.type_string()} ] %} diff --git a/macros/get_product_variant_columns.sql b/macros/get_product_variant_columns.sql index 80e5c00..5738745 100644 --- a/macros/get_product_variant_columns.sql +++ b/macros/get_product_variant_columns.sql @@ -1,10 +1,10 @@ {% macro get_product_variant_columns() %} {% set columns = [ - {"name": "id", "datatype": dbt.type_numeric(), "alias": "variant_id"}, + {"name": "id", "datatype": dbt.type_numeric()}, {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "updated_at", "datatype": dbt.type_timestamp()}, {"name": "product_id", "datatype": dbt.type_numeric()}, {"name": "inventory_item_id", "datatype": dbt.type_numeric()}, {"name": "image_id", "datatype": dbt.type_numeric()}, @@ -16,7 +16,7 @@ {"name": "compare_at_price", "datatype": dbt.type_float()}, {"name": "fulfillment_service", "datatype": dbt.type_string()}, {"name": "inventory_management", "datatype": dbt.type_string()}, - {"name": "taxable", "datatype": "boolean", "alias": "is_taxable"}, + {"name": "taxable", "datatype": "boolean"}, {"name": "barcode", "datatype": dbt.type_string()}, {"name": "grams", "datatype": dbt.type_float()}, {"name": "inventory_quantity", "datatype": dbt.type_numeric()}, @@ -25,8 +25,7 @@ {"name": "option_1", "datatype": dbt.type_string()}, {"name": "option_2", "datatype": dbt.type_string()}, {"name": "option_3", "datatype": dbt.type_string()}, - {"name": "tax_code", "datatype": dbt.type_string()}, - {"name": "requires_shipping", "datatype": "boolean", "alias": "is_requiring_shipping"} + {"name": "tax_code", "datatype": dbt.type_string()} ] %} {{ fivetran_utils.add_pass_through_columns(columns, var('product_variant_pass_through_columns')) }} diff --git a/macros/get_refund_columns.sql b/macros/get_refund_columns.sql index 8460132..affce51 100644 --- a/macros/get_refund_columns.sql +++ b/macros/get_refund_columns.sql @@ -3,7 +3,7 @@ {% set columns = [ {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, {"name": "created_at", "datatype": dbt.type_timestamp()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "refund_id"}, + {"name": "id", "datatype": dbt.type_numeric()}, {"name": "note", "datatype": dbt.type_string()}, {"name": "order_id", "datatype": dbt.type_numeric()}, {"name": "processed_at", "datatype": dbt.type_timestamp()}, diff --git a/macros/get_transaction_columns.sql b/macros/get_transaction_columns.sql index 0e7d9ab..dfd2f6e 100644 --- a/macros/get_transaction_columns.sql +++ b/macros/get_transaction_columns.sql @@ -1,12 +1,12 @@ {% macro get_transaction_columns() %} {% set columns = [ - {"name": "id", "datatype": dbt.type_numeric(), "alias": "transaction_id"}, + {"name": "id", "datatype": dbt.type_numeric()}, {"name": "order_id", "datatype": dbt.type_numeric()}, {"name": "refund_id", "datatype": dbt.type_numeric()}, {"name": "amount", "datatype": dbt.type_numeric()}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "processed_at", "datatype": dbt.type_timestamp(), "alias": "processed_timestamp"}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "processed_at", "datatype": dbt.type_timestamp()}, {"name": "device_id", "datatype": dbt.type_numeric()}, {"name": "gateway", "datatype": dbt.type_string()}, {"name": "source_name", "datatype": dbt.type_string()}, @@ -37,7 +37,7 @@ {% if target.type in ('redshift','postgres') %} {{ columns.append({"name": "authorization", "datatype": dbt.type_string(), "quote": True, "alias": "authorization"}) }} {% else %} - {"name": "authorization", "datatype": dbt.type_string()} + {{ columns.append({"name": "authorization", "datatype": dbt.type_string()}) }} {% endif %} {{ return(columns) }} diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index e8c066e..8122bb6 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -99,7 +99,7 @@ models: description: The ID of the product that the line item belongs to. Can be null if the original product associated with the order is deleted at a later date. - name: quantity description: The number of items that were purchased. - - name: is_requiring_shipping + - name: is_shipping_required description: Whether the item requires shipping. - name: sku description: The item's SKU (stock keeping unit). @@ -336,8 +336,6 @@ models: description: The price of the product variant. - name: product_id description: The unique numeric identifier for the product. - - name: is_requiring_shipping - description: This property is deprecated. Use the `requires_shipping` property on the InventoryItem resource instead. - name: sku description: A unique identifier for the product variant in the shop. Required in order to connect to a FulfillmentService. - name: is_taxable diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql index 3e2262b..5c84f2c 100644 --- a/models/stg_shopify__customer.sql +++ b/models/stg_shopify__customer.sql @@ -16,7 +16,6 @@ fields as ( ) }} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') @@ -29,8 +28,9 @@ fields as ( final as ( select - coalesce(accepts_marketing, email_marketing_consent_state) as has_accepted_marketing, - created_at as created_timestamp, + accepts_marketing as has_accepted_marketing, + email_marketing_consent_state, + created_at, default_address_id, email, first_name, @@ -41,7 +41,7 @@ final as ( state as account_state, tax_exempt as is_tax_exempt, total_spent, - updated_at as updated_timestamp, + updated_at, verified_email as is_verified_email, note, coalesce(accepts_marketing_updated_at, email_marketing_consent_consent_updated_at) as accepts_marketing_updated_at, diff --git a/models/stg_shopify__order.sql b/models/stg_shopify__order.sql index 49197f5..ab9194f 100644 --- a/models/stg_shopify__order.sql +++ b/models/stg_shopify__order.sql @@ -21,7 +21,7 @@ fields as ( union_database_variable='shopify_union_databases') }} - from source + from base ), @@ -29,8 +29,8 @@ final as ( select id as order_id, - processed_at as processed_timestamp, - updated_at as updated_timestamp, + processed_at, + updated_at, user_id, total_discounts, total_discounts_set, @@ -54,11 +54,11 @@ final as ( number, order_number, cancel_reason, - cancelled_at as cancelled_timestamp, + cancelled_at, cart_token, checkout_token, - closed_at as closed_timestamp, - created_at as created_timestamp, + closed_at, + created_at, currency, customer_id, email, @@ -99,7 +99,7 @@ final as ( shipping_address_province, shipping_address_province_code, shipping_address_zip, - test, + test as is_test_order, token, _fivetran_synced, _fivetran_deleted, diff --git a/models/stg_shopify__order_adjustment.sql b/models/stg_shopify__order_adjustment.sql index fbd59a6..a24d915 100644 --- a/models/stg_shopify__order_adjustment.sql +++ b/models/stg_shopify__order_adjustment.sql @@ -23,7 +23,7 @@ fields as ( union_database_variable='shopify_union_databases') }} - from source + from base ), final as ( @@ -45,4 +45,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/stg_shopify__order_line.sql b/models/stg_shopify__order_line.sql index 4811e1e..a52b6ef 100644 --- a/models/stg_shopify__order_line.sql +++ b/models/stg_shopify__order_line.sql @@ -21,7 +21,7 @@ fields as ( union_database_variable='shopify_union_databases') }} - from source + from base ), @@ -71,5 +71,4 @@ final as ( ) select * -from final - +from final \ No newline at end of file diff --git a/models/stg_shopify__order_line_refund.sql b/models/stg_shopify__order_line_refund.sql index 7d20b1a..80fd3f0 100644 --- a/models/stg_shopify__order_line_refund.sql +++ b/models/stg_shopify__order_line_refund.sql @@ -24,7 +24,7 @@ fields as ( union_database_variable='shopify_union_databases') }} - from source + from base ), @@ -50,5 +50,4 @@ final as ( ) select * -from final - +from final \ No newline at end of file diff --git a/models/stg_shopify__product.sql b/models/stg_shopify__product.sql index 6be960f..405acca 100644 --- a/models/stg_shopify__product.sql +++ b/models/stg_shopify__product.sql @@ -1,10 +1,11 @@ -with source as ( +with base as ( - select * from {{ ref('stg_shopify__product_tmp') }} + select * + from {{ ref('stg_shopify__product_tmp') }} ), -renamed as ( +fields as ( select @@ -15,21 +16,37 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('product_pass_through_columns') %} - , - {{ var('product_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source + from base -) +), -select * from renamed +final as ( + + select + created_at, + handle, + id as product_id, + product_type, + published_at, + published_scope, + title, + updated_at, + vendor, + status, + _fivetran_deleted as is_deleted, + _fivetran_synced, + source_relation + + {{ fivetran_utils.fill_pass_through_columns('product_pass_through_columns') }} + +from fields + +) +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__product_variant.sql b/models/stg_shopify__product_variant.sql index 13c0fb5..07f9c80 100644 --- a/models/stg_shopify__product_variant.sql +++ b/models/stg_shopify__product_variant.sql @@ -1,10 +1,11 @@ -with source as ( +with base as ( - select * from {{ ref('stg_shopify__product_variant_tmp') }} + select * + from {{ ref('stg_shopify__product_variant_tmp') }} ), -renamed as ( +fields as ( select @@ -15,20 +16,49 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('product_variant_pass_through_columns') %} - , - {{ var('product_variant_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source + from base + +), + +final as ( + + select + id as variant_id, + _fivetran_synced, + created_at, + updated_at, + product_id, + inventory_item_id, + image_id, + title, + price, + sku, + position, + inventory_policy, + compare_at_price, + fulfillment_service, + inventory_management, + taxable as is_taxable, + barcode, + grams, + inventory_quantity, + weight, + weight_unit, + option_1, + option_2, + option_3, + tax_code, + source_relation + + {{ fivetran_utils.fill_pass_through_columns('product_variant_pass_through_columns') }} + from fields ) -select * from renamed \ No newline at end of file +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__refund.sql b/models/stg_shopify__refund.sql index 4e7c6e1..ef550d6 100644 --- a/models/stg_shopify__refund.sql +++ b/models/stg_shopify__refund.sql @@ -1,14 +1,14 @@ --To disable this model, set the shopify__using_refund variable within your dbt_project.yml file to False. {{ config(enabled=var('shopify__using_refund', True)) }} -with source as ( +with base as ( select * from {{ ref('stg_shopify__refund_tmp') }} ), -renamed as ( +fields as ( select {{ @@ -23,7 +23,25 @@ renamed as ( union_database_variable='shopify_union_databases') }} - from source + from base +), + +final as ( + + select + created_at, + id as refund_id, + note, + order_id, + processed_at, + restock, + total_duties_set, + user_id, + _fivetran_synced, + source_relation + + from fields ) -select * from renamed +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__transaction.sql b/models/stg_shopify__transaction.sql index 6dc51d2..c0c5417 100644 --- a/models/stg_shopify__transaction.sql +++ b/models/stg_shopify__transaction.sql @@ -1,10 +1,10 @@ -with source as ( +with base as ( select * from {{ ref('stg_shopify__transaction_tmp') }} ), -renamed as ( +fields as ( select @@ -15,21 +15,56 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('transaction_pass_through_columns') %} - , - {{ var('transaction_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source - where not test + from base + +), + +final as ( + + select + id as transaction_id, + order_id, + refund_id, + amount, + created_at, + processed_at, + device_id, + gateway, + source_name, + message, + currency, + location_id, + parent_id, + payment_avs_result_code, + payment_credit_card_bin, + payment_cvv_result_code, + payment_credit_card_number, + payment_credit_card_company, + kind, + receipt, + currency_exchange_id, + currency_exchange_adjustment, + currency_exchange_original_amount, + currency_exchange_final_amount, + currency_exchange_currency, + error_code, + status, + user_id, + _fivetran_synced, + authorization_expires_at, + authorization, + source_relation + + {{ fivetran_utils.fill_pass_through_columns('transaction_pass_through_columns') }} + from fields + where not coalesce(test, false) ) -select * from renamed +select * +from final \ No newline at end of file diff --git a/models/tmp/stg_shopify__refund_tmp.sql b/models/tmp/stg_shopify__refund_tmp.sql index c3eff2d..d57c0ce 100644 --- a/models/tmp/stg_shopify__refund_tmp.sql +++ b/models/tmp/stg_shopify__refund_tmp.sql @@ -26,14 +26,15 @@ {% else %} select - cast(null as {{ dbt_utils.type_timestamp() }}) as _fivetran_synced, - cast(null as {{ dbt_utils.type_timestamp() }}) as created_at, - cast(null as {{ dbt_utils.type_numeric() }}) as id, - cast(null as {{ dbt_utils.type_string() }}) as note, - cast(null as {{ dbt_utils.type_numeric() }}) as order_id, - cast(null as {{ dbt_utils.type_timestamp() }}) as processed_at, + cast(null as {{ dbt.type_timestamp() }}) as _fivetran_synced, + cast(null as {{ dbt.type_timestamp() }}) as created_at, + cast(null as {{ dbt.type_numeric() }}) as id, + cast(null as {{ dbt.type_string() }}) as note, + cast(null as {{ dbt.type_numeric() }}) as order_id, + cast(null as {{ dbt.type_timestamp() }}) as processed_at, cast(null as boolean) as restock, - cast(null as {{ dbt_utils.type_string() }}) as total_duties_set, - cast(null as {{ dbt_utils.type_numeric() }}) as user_id + cast(null as {{ dbt.type_string() }}) as total_duties_set, + cast(null as {{ dbt.type_numeric() }}) as user_id, + cast(null as {{ dbt.type_string() }}) as _dbt_source_relation {% endif %} \ No newline at end of file diff --git a/packages.yml b/packages.yml index 908f471..2f19c7a 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,8 @@ packages: -- package: fivetran/fivetran_utils - version: [">=0.4.0", "<0.5.0"] +# - package: fivetran/fivetran_utils +# version: [">=0.4.0", "<0.5.0"] +- git: https://github.com/fivetran/dbt_fivetran_utils.git + revision: backwards-compat-passthru + warn-unpinned: false - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] From 2fbc84e7df200404cd810b8e7e165558e99f45e1 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 12 Dec 2022 17:22:41 -0800 Subject: [PATCH 3/8] one more changelog thing --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96928a9..91311c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - `stg_shopify__product_variant` - `stg_shopify__product` - `stg_shopify__transaction` +- The `is_requiring_shipping` field in `stg_shopify__order_line` has been renamed to `is_shipping_required`, just because it sounds better :) # dbt_shopify_source v0.6.0 🎉 dbt v1.0.0 Compatibility 🎉 From 0e1df179e729610ac8756df7fba2aa60c97d4cd2 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 13 Dec 2022 15:55:57 -0800 Subject: [PATCH 4/8] merged figvetran utils --- packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.yml b/packages.yml index 2f19c7a..fd66aa3 100644 --- a/packages.yml +++ b/packages.yml @@ -2,7 +2,7 @@ packages: # - package: fivetran/fivetran_utils # version: [">=0.4.0", "<0.5.0"] - git: https://github.com/fivetran/dbt_fivetran_utils.git - revision: backwards-compat-passthru + revision: releases/v0.4.latest warn-unpinned: false - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] From 714f26b5b90a73b4cc2f387731910e4aa9be10ad Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 14 Dec 2022 09:51:49 -0800 Subject: [PATCH 5/8] buildkite --- macros/get_transaction_columns.sql | 4 ++-- models/stg_shopify__order_line_refund.sql | 2 +- models/stg_shopify__transaction.sql | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/macros/get_transaction_columns.sql b/macros/get_transaction_columns.sql index dfd2f6e..2835938 100644 --- a/macros/get_transaction_columns.sql +++ b/macros/get_transaction_columns.sql @@ -35,9 +35,9 @@ ] %} {% if target.type in ('redshift','postgres') %} - {{ columns.append({"name": "authorization", "datatype": dbt.type_string(), "quote": True, "alias": "authorization"}) }} + {{ columns.append({"name": "authorization", "datatype": dbt.type_string(), "quote": True, "alias": "authorization_code"}) }} {% else %} - {{ columns.append({"name": "authorization", "datatype": dbt.type_string()}) }} + {{ columns.append({"name": "authorization", "datatype": dbt.type_string(), "alias": "authorization_code"}) }} {% endif %} {{ return(columns) }} diff --git a/models/stg_shopify__order_line_refund.sql b/models/stg_shopify__order_line_refund.sql index 80fd3f0..844f97e 100644 --- a/models/stg_shopify__order_line_refund.sql +++ b/models/stg_shopify__order_line_refund.sql @@ -42,7 +42,7 @@ final as ( refund_id, restock_type, _fivetran_synced, - source_relation, + source_relation {{ fivetran_utils.fill_pass_through_columns('order_line_refund_pass_through_columns') }} diff --git a/models/stg_shopify__transaction.sql b/models/stg_shopify__transaction.sql index c0c5417..91a94e0 100644 --- a/models/stg_shopify__transaction.sql +++ b/models/stg_shopify__transaction.sql @@ -57,7 +57,7 @@ final as ( user_id, _fivetran_synced, authorization_expires_at, - authorization, + authorization_code, source_relation {{ fivetran_utils.fill_pass_through_columns('transaction_pass_through_columns') }} From 08c561893ebeefe979f3989f68167b5bdff6d32d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 14 Dec 2022 10:01:57 -0800 Subject: [PATCH 6/8] databricks --- integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 8ba31eb..01582ec 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: shopify_source_integration_test + schema: shopify_source_integration_tests threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: shopify_source_integration_test + schema: shopify_source_integration_tests threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: shopify_source_integration_test + schema: shopify_source_integration_tests threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: shopify_source_integration_test + schema: shopify_source_integration_tests threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: shopify_source_integration_test + schema: shopify_source_integration_tests threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index ccf73d1..1e8358e 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -4,6 +4,7 @@ profile: 'integration_tests' config-version: 2 vars: + shopify_schema: shopify_source_integration_tests shopify_source: customer_source: "{{ ref('shopify_customer_data') }}" order_line_refund_source: "{{ ref('shopify_order_line_refund_data') }}" From 0def2e302be2b3a4bb45a5a865e290a48f01de26 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 14 Dec 2022 10:02:15 -0800 Subject: [PATCH 7/8] docs --- models/stg_shopify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index 8122bb6..570c7e1 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -369,7 +369,7 @@ models: description: The ID associated with a refund in the refund table. - name: amount description: The amount of money included in the transaction. - - name: authorization + - name: authorization_code description: The authorization code associated with the transaction. - name: created_timestamp description: The date and time when the transaction was created. From 17aedd2718fb67fb8dbdec2da330ac39e90b383d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 14 Dec 2022 11:33:03 -0800 Subject: [PATCH 8/8] Refunds explanation --- CHANGELOG.md | 5 +++++ README.md | 2 ++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91311c3..f5afab2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ - `dbt.current_timestamp_backcompat` - `dbt.current_timestamp_in_utc_backcompat` - Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`. + + +[PR #40](https://github.com/fivetran/dbt_shopify_source/pull/40) includes the following breaking changes: - In the following tables, `*_timestamp` columns have been renamed to `*_at` for consistency: - `stg_shopify__customer` - `stg_shopify__order` @@ -43,6 +46,8 @@ - `stg_shopify__product` - `stg_shopify__transaction` - The `is_requiring_shipping` field in `stg_shopify__order_line` has been renamed to `is_shipping_required`, just because it sounds better :) +- TODO more stuff +- By default, if the refunds table does not exist, the package will create an empty `stg_shopify__refunds` model. This can still be overwritten by setting `shopify__using_refund` to `false`. We did this because the refunds table is only created once your Shopify account has processed its first refund, and it is unnecessary overhead to have to monitor this and change your dbt_project accordingly. Thus, we took the approach of having this refund table be one that syncs as empty if it is not populated, then seamlessly switches to the source table once it exists. # dbt_shopify_source v0.6.0 🎉 dbt v1.0.0 Compatibility 🎉 diff --git a/README.md b/README.md index d97d7ce..f8941e9 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,8 @@ vars: shopify__using_refund: false # true by default ``` +> By default, if the refunds table does not exist, the package will create an empty `stg_shopify__refunds` model. This can still be overwritten by setting `shopify__using_refund` to `false`. We did this because the refunds table is only created once your Shopify account has processed its first refund, and it is unnecessary overhead to have to monitor this and change your dbt_project accordingly. Thus, we took the approach of having this refund table be one that syncs as empty if it is not populated, then seamlessly switches to the source table once it exists. + ### Changing the Build Schema By default this package will build the Shopify staging models within a schema titled ( + `_stg_shopify`) in your target database. If this is not where you would like your staging Shopify data to be written to, add the following configuration to your `dbt_project.yml` file: