Update field selection, etc

CHANGELOG.md

@@ -36,6 +36,13 @@
     - `dbt.current_timestamp_backcompat`
     - `dbt.current_timestamp_in_utc_backcompat`
 - Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
+- In the following tables, `*_timestamp` columns have been renamed to `*_at` for consistency:
+  - `stg_shopify__customer`
+  - `stg_shopify__order`
+  - `stg_shopify__product_variant`
+  - `stg_shopify__product`
+  - `stg_shopify__transaction`
+- The `is_requiring_shipping` field in `stg_shopify__order_line` has been renamed to `is_shipping_required`, just because it sounds better :)
 
 # dbt_shopify_source v0.6.0
 🎉 dbt v1.0.0 Compatibility 🎉

macros/get_customer_columns.sql

@@ -0,0 +1,33 @@
+{% macro get_customer_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "accepts_marketing", "datatype": "boolean"},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "default_address_id", "datatype": dbt.type_numeric()},
+    {"name": "email", "datatype": dbt.type_string()},
+    {"name": "first_name", "datatype": dbt.type_string()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "last_name", "datatype": dbt.type_string()},
+    {"name": "orders_count", "datatype": dbt.type_numeric()},
+    {"name": "phone", "datatype": dbt.type_string()},
+    {"name": "state", "datatype": dbt.type_string()},
+    {"name": "tax_exempt", "datatype": "boolean"},
+    {"name": "total_spent", "datatype": dbt.type_float()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "verified_email", "datatype": "boolean"},
+    {"name": "email_marketing_consent_consent_updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "email_marketing_consent_opt_in_level", "datatype": dbt.type_string()},
+    {"name": "email_marketing_consent_state", "datatype": dbt.type_string()},
+    {"name": "note", "datatype": dbt.type_string()},
+    {"name": "accepts_marketing_updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "marketing_opt_in_level", "datatype": dbt.type_string()},
+    {"name": "lifetime_duration", "datatype": dbt.type_string()},
+    {"name": "currency", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('customer_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_adjustment_columns.sql

@@ -0,0 +1,18 @@
+{% macro get_order_adjustment_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype":  dbt.type_numeric()},
+    {"name": "order_id", "datatype":  dbt.type_numeric()},
+    {"name": "refund_id", "datatype":  dbt.type_numeric()},
+    {"name": "amount", "datatype": dbt.type_float()},
+    {"name": "amount_set", "datatype": dbt.type_string()},
+    {"name": "tax_amount", "datatype": dbt.type_float()},
+    {"name": "tax_amount_set", "datatype": dbt.type_string()},
+    {"name": "kind", "datatype": dbt.type_string()},
+    {"name": "reason", "datatype": dbt.type_string()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_columns.sql

@@ -0,0 +1,93 @@
+{% macro get_order_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "processed_at", "datatype": dbt.type_timestamp()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "user_id", "datatype": dbt.type_numeric()},
+    {"name": "total_discounts", "datatype": dbt.type_float()},
+    {"name": "total_discounts_set", "datatype": dbt.type_string()},
+    {"name": "total_line_items_price", "datatype": dbt.type_float()},
+    {"name": "total_line_items_price_set", "datatype": dbt.type_string()},
+    {"name": "total_price", "datatype": dbt.type_float()},
+    {"name": "total_price_set", "datatype": dbt.type_string()},
+    {"name": "total_price_usd", "datatype": dbt.type_float()},
+    {"name": "total_tax_set", "datatype": dbt.type_string()},
+    {"name": "total_tax", "datatype": dbt.type_float()},
+    {"name": "source_name", "datatype": dbt.type_string()},
+    {"name": "subtotal_price", "datatype": dbt.type_float()},
+    {"name": "taxes_included", "datatype": "boolean"},
+    {"name": "total_weight", "datatype": dbt.type_numeric()},
+    {"name": "total_tip_received", "datatype": dbt.type_float()},
+    {"name": "landing_site_base_url", "datatype": dbt.type_string()},
+    {"name": "landing_site_ref", "datatype": dbt.type_string()},
+    {"name": "location_id", "datatype": dbt.type_numeric()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "note", "datatype": dbt.type_string()},
+    {"name": "number", "datatype": dbt.type_numeric()},
+    {"name": "order_number", "datatype": dbt.type_numeric()},
+    {"name": "cancel_reason", "datatype": dbt.type_string()},
+    {"name": "cancelled_at", "datatype": dbt.type_timestamp()},
+    {"name": "cart_token", "datatype": dbt.type_string()},
+    {"name": "checkout_token", "datatype": dbt.type_string()},
+    {"name": "closed_at", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "currency", "datatype": dbt.type_string()},
+    {"name": "customer_id", "datatype": dbt.type_numeric()},
+    {"name": "email", "datatype": dbt.type_string()},
+    {"name": "financial_status", "datatype": dbt.type_string()},
+    {"name": "fulfillment_status", "datatype": dbt.type_string()},
+    {"name": "processing_method", "datatype": dbt.type_string()},
+    {"name": "referring_site", "datatype": dbt.type_string()},
+    {"name": "billing_address_address_1", "datatype": dbt.type_string()},
+    {"name": "billing_address_address_2", "datatype": dbt.type_string()},
+    {"name": "billing_address_city", "datatype": dbt.type_string()},
+    {"name": "billing_address_company", "datatype": dbt.type_string()},
+    {"name": "billing_address_country", "datatype": dbt.type_string()},
+    {"name": "billing_address_country_code", "datatype": dbt.type_string()},
+    {"name": "billing_address_first_name", "datatype": dbt.type_string()},
+    {"name": "billing_address_last_name", "datatype": dbt.type_string()},
+    {"name": "billing_address_latitude", "datatype": dbt.type_string()},
+    {"name": "billing_address_longitude", "datatype": dbt.type_string()},
+    {"name": "billing_address_name", "datatype": dbt.type_string()},
+    {"name": "billing_address_phone", "datatype": dbt.type_string()},
+    {"name": "billing_address_province", "datatype": dbt.type_string()},
+    {"name": "billing_address_province_code", "datatype": dbt.type_string()},
+    {"name": "billing_address_zip", "datatype": dbt.type_string()},
+    {"name": "browser_ip", "datatype": dbt.type_string()},
+    {"name": "buyer_accepts_marketing", "datatype": "boolean"},
+    {"name": "total_shipping_price_set", "datatype": dbt.type_string()},
+    {"name": "shipping_address_address_1", "datatype": dbt.type_string()},
+    {"name": "shipping_address_address_2", "datatype": dbt.type_string()},
+    {"name": "shipping_address_city", "datatype": dbt.type_string()},
+    {"name": "shipping_address_company", "datatype": dbt.type_string()},
+    {"name": "shipping_address_country", "datatype": dbt.type_string()},
+    {"name": "shipping_address_country_code", "datatype": dbt.type_string()},
+    {"name": "shipping_address_first_name", "datatype": dbt.type_string()},
+    {"name": "shipping_address_last_name", "datatype": dbt.type_string()},
+    {"name": "shipping_address_latitude", "datatype": dbt.type_string()},
+    {"name": "shipping_address_longitude", "datatype": dbt.type_string()},
+    {"name": "shipping_address_name", "datatype": dbt.type_string()},
+    {"name": "shipping_address_phone", "datatype": dbt.type_string()},
+    {"name": "shipping_address_province", "datatype": dbt.type_string()},
+    {"name": "shipping_address_province_code", "datatype": dbt.type_string()},
+    {"name": "shipping_address_zip", "datatype": dbt.type_string()},
+    {"name": "test", "datatype": "boolean"},
+    {"name": "token", "datatype": dbt.type_string()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "app_id", "datatype": dbt.type_int()},
+    {"name": "checkout_id", "datatype": dbt.type_int()},
+    {"name": "client_details_user_agent", "datatype": dbt.type_string()},
+    {"name": "customer_locale", "datatype": dbt.type_string()},
+    {"name": "device_id", "datatype": dbt.type_string()},
+    {"name": "order_status_url", "datatype": dbt.type_string()},
+    {"name": "presentment_currency", "datatype": dbt.type_string()},
+    {"name": "confirmed", "datatype": "boolean"}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('order_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_line_columns.sql

@@ -0,0 +1,43 @@
+{% macro get_order_line_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "fulfillable_quantity", "datatype": dbt.type_numeric()},
+    {"name": "fulfillment_service", "datatype": dbt.type_string()},
+    {"name": "fulfillment_status", "datatype": dbt.type_string()},
+    {"name": "gift_card", "datatype": "boolean"},
+    {"name": "grams", "datatype": dbt.type_numeric()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "index", "datatype": dbt.type_numeric()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "order_id", "datatype": dbt.type_numeric()},
+    {"name": "pre_tax_price", "datatype": dbt.type_float()},
+    {"name": "pre_tax_price_set", "datatype": dbt.type_string()},
+    {"name": "price", "datatype": dbt.type_float()},
+    {"name": "price_set", "datatype": dbt.type_string()},
+    {"name": "product_id", "datatype": dbt.type_numeric()},
+    {"name": "property_charge_interval_frequency", "datatype": dbt.type_numeric()},
+    {"name": "property_for_shipping_jan_3_rd_2020", "datatype": dbt.type_string()},
+    {"name": "property_shipping_interval_frequency", "datatype": dbt.type_numeric()},
+    {"name": "property_shipping_interval_unit_type", "datatype": dbt.type_string()},
+    {"name": "property_subscription_id", "datatype": dbt.type_numeric()},
+    {"name": "quantity", "datatype": dbt.type_numeric()},
+    {"name": "requires_shipping", "datatype": "boolean"},
+    {"name": "sku", "datatype": dbt.type_string()},
+    {"name": "taxable", "datatype": "boolean"},
+    {"name": "tax_code", "datatype": dbt.type_string()},
+    {"name": "title", "datatype": dbt.type_string()},
+    {"name": "total_discount", "datatype": dbt.type_float()},
+    {"name": "total_discount_set", "datatype": dbt.type_string()},
+    {"name": "variant_id", "datatype": dbt.type_numeric()},
+    {"name": "variant_title", "datatype": dbt.type_string()},
+    {"name": "variant_inventory_management", "datatype": dbt.type_string()},
+    {"name": "vendor", "datatype": dbt.type_string()},
+    {"name": "properties", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('order_line_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_line_refund_columns.sql

@@ -0,0 +1,21 @@
+{% macro get_order_line_refund_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "location_id", "datatype": dbt.type_numeric()},
+    {"name": "order_line_id", "datatype": dbt.type_numeric()},
+    {"name": "subtotal", "datatype": dbt.type_numeric()},
+    {"name": "subtotal_set", "datatype": dbt.type_string()},
+    {"name": "total_tax", "datatype": dbt.type_numeric()},
+    {"name": "total_tax_set", "datatype": dbt.type_string()},
+    {"name": "quantity", "datatype": dbt.type_float()},
+    {"name": "refund_id", "datatype": dbt.type_numeric()},
+    {"name": "restock_type", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('order_line_refund_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_product_columns.sql

@@ -0,0 +1,22 @@
+{% macro get_product_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "handle", "datatype": dbt.type_string()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "product_type", "datatype": dbt.type_string()},
+    {"name": "published_at", "datatype": dbt.type_timestamp()},
+    {"name": "published_scope", "datatype": dbt.type_string()},
+    {"name": "title", "datatype": dbt.type_string()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "vendor", "datatype": dbt.type_string()},
+    {"name": "status", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('product_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_product_variant_columns.sql

@@ -0,0 +1,35 @@
+{% macro get_product_variant_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "product_id", "datatype": dbt.type_numeric()},
+    {"name": "inventory_item_id", "datatype": dbt.type_numeric()},
+    {"name": "image_id", "datatype": dbt.type_numeric()},
+    {"name": "title", "datatype": dbt.type_string()},
+    {"name": "price", "datatype": dbt.type_float()},
+    {"name": "sku", "datatype": dbt.type_string()},
+    {"name": "position", "datatype": dbt.type_numeric()},
+    {"name": "inventory_policy", "datatype": dbt.type_string()},
+    {"name": "compare_at_price", "datatype": dbt.type_float()},
+    {"name": "fulfillment_service", "datatype": dbt.type_string()},
+    {"name": "inventory_management", "datatype": dbt.type_string()},
+    {"name": "taxable", "datatype": "boolean"},
+    {"name": "barcode", "datatype": dbt.type_string()},
+    {"name": "grams", "datatype": dbt.type_float()},
+    {"name": "inventory_quantity", "datatype": dbt.type_numeric()},
+    {"name": "weight", "datatype": dbt.type_float()},
+    {"name": "weight_unit", "datatype": dbt.type_string()},
+    {"name": "option_1", "datatype": dbt.type_string()},
+    {"name": "option_2", "datatype": dbt.type_string()},
+    {"name": "option_3", "datatype": dbt.type_string()},
+    {"name": "tax_code", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('product_variant_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_refund_columns.sql

@@ -0,0 +1,17 @@
+{% macro get_refund_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "note", "datatype": dbt.type_string()},
+    {"name": "order_id", "datatype": dbt.type_numeric()},
+    {"name": "processed_at", "datatype": dbt.type_timestamp()},
+    {"name": "restock", "datatype": "boolean"},
+    {"name": "total_duties_set", "datatype": dbt.type_string()},
+    {"name": "user_id", "datatype": dbt.type_numeric()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_transaction_columns.sql

@@ -0,0 +1,45 @@
+{% macro get_transaction_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "order_id", "datatype": dbt.type_numeric()},
+    {"name": "refund_id", "datatype": dbt.type_numeric()},
+    {"name": "amount", "datatype": dbt.type_numeric()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "processed_at", "datatype": dbt.type_timestamp()},
+    {"name": "device_id", "datatype": dbt.type_numeric()},
+    {"name": "gateway", "datatype": dbt.type_string()},
+    {"name": "source_name", "datatype": dbt.type_string()},
+    {"name": "message", "datatype": dbt.type_string()},
+    {"name": "currency", "datatype": dbt.type_string()},
+    {"name": "location_id", "datatype": dbt.type_numeric()},
+    {"name": "parent_id", "datatype": dbt.type_numeric()},
+    {"name": "payment_avs_result_code", "datatype": dbt.type_string()},
+    {"name": "payment_credit_card_bin", "datatype": dbt.type_string()},
+    {"name": "payment_cvv_result_code", "datatype": dbt.type_string()},
+    {"name": "payment_credit_card_number", "datatype": dbt.type_string()},
+    {"name": "payment_credit_card_company", "datatype": dbt.type_string()},
+    {"name": "kind", "datatype": dbt.type_string()},
+    {"name": "receipt", "datatype": dbt.type_string()},
+    {"name": "currency_exchange_id", "datatype": dbt.type_numeric()},
+    {"name": "currency_exchange_adjustment", "datatype": dbt.type_numeric()},
+    {"name": "currency_exchange_original_amount", "datatype": dbt.type_numeric()},
+    {"name": "currency_exchange_final_amount", "datatype": dbt.type_numeric()},
+    {"name": "currency_exchange_currency", "datatype": dbt.type_string()},
+    {"name": "error_code", "datatype": dbt.type_string()},
+    {"name": "status", "datatype": dbt.type_string()},
+    {"name": "test", "datatype": "boolean"},
+    {"name": "user_id", "datatype": dbt.type_numeric()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "authorization_expires_at", "datatype": dbt.type_timestamp()}
+] %}
+
+{% if target.type in ('redshift','postgres') %}
+ {{ columns.append({"name": "authorization", "datatype": dbt.type_string(), "quote": True, "alias": "authorization"}) }}
+{% else %}
+ {{ columns.append({"name": "authorization", "datatype": dbt.type_string()}) }}
+{% endif %}
+
+{{ return(columns) }}
+
+{% endmacro %}