Update field selection, etc

CHANGELOG.md

@@ -36,6 +36,13 @@
     - `dbt.current_timestamp_backcompat`
     - `dbt.current_timestamp_in_utc_backcompat`
 - Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
+- In the following tables, `*_timestamp` columns have been renamed to `*_at` for consistency:
+  - `stg_shopify__customer`
+  - `stg_shopify__order`
+  - `stg_shopify__product_variant`
+  - `stg_shopify__product`
+  - `stg_shopify__transaction`
+- The `is_requiring_shipping` field in `stg_shopify__order_line` has been renamed to `is_shipping_required`, just because it sounds better :)
 
 # dbt_shopify_source v0.6.0
 🎉 dbt v1.0.0 Compatibility 🎉

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_source_integration_test
+      schema: shopify_source_integration_tests
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_source_integration_test
+      schema: shopify_source_integration_tests
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_source_integration_test
+      schema: shopify_source_integration_tests
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_source_integration_test
+      schema: shopify_source_integration_tests
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_source_integration_test
+      schema: shopify_source_integration_tests
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -4,6 +4,7 @@ profile: 'integration_tests'
 config-version: 2
 
 vars:
+  shopify_schema: shopify_source_integration_tests
   shopify_source:
     customer_source:          "{{ ref('shopify_customer_data') }}"
     order_line_refund_source: "{{ ref('shopify_order_line_refund_data') }}"

macros/get_customer_columns.sql

@@ -0,0 +1,33 @@
+{% macro get_customer_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "accepts_marketing", "datatype": "boolean"},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "default_address_id", "datatype": dbt.type_numeric()},
+    {"name": "email", "datatype": dbt.type_string()},
+    {"name": "first_name", "datatype": dbt.type_string()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "last_name", "datatype": dbt.type_string()},
+    {"name": "orders_count", "datatype": dbt.type_numeric()},
+    {"name": "phone", "datatype": dbt.type_string()},
+    {"name": "state", "datatype": dbt.type_string()},
+    {"name": "tax_exempt", "datatype": "boolean"},
+    {"name": "total_spent", "datatype": dbt.type_float()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "verified_email", "datatype": "boolean"},
+    {"name": "email_marketing_consent_consent_updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "email_marketing_consent_opt_in_level", "datatype": dbt.type_string()},
+    {"name": "email_marketing_consent_state", "datatype": dbt.type_string()},
+    {"name": "note", "datatype": dbt.type_string()},
+    {"name": "accepts_marketing_updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "marketing_opt_in_level", "datatype": dbt.type_string()},
+    {"name": "lifetime_duration", "datatype": dbt.type_string()},
+    {"name": "currency", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('customer_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_adjustment_columns.sql

@@ -0,0 +1,18 @@
+{% macro get_order_adjustment_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype":  dbt.type_numeric()},
+    {"name": "order_id", "datatype":  dbt.type_numeric()},
+    {"name": "refund_id", "datatype":  dbt.type_numeric()},
+    {"name": "amount", "datatype": dbt.type_float()},
+    {"name": "amount_set", "datatype": dbt.type_string()},
+    {"name": "tax_amount", "datatype": dbt.type_float()},
+    {"name": "tax_amount_set", "datatype": dbt.type_string()},
+    {"name": "kind", "datatype": dbt.type_string()},
+    {"name": "reason", "datatype": dbt.type_string()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_columns.sql

@@ -0,0 +1,93 @@
+{% macro get_order_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "processed_at", "datatype": dbt.type_timestamp()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "user_id", "datatype": dbt.type_numeric()},
+    {"name": "total_discounts", "datatype": dbt.type_float()},
+    {"name": "total_discounts_set", "datatype": dbt.type_string()},
+    {"name": "total_line_items_price", "datatype": dbt.type_float()},
+    {"name": "total_line_items_price_set", "datatype": dbt.type_string()},
+    {"name": "total_price", "datatype": dbt.type_float()},
+    {"name": "total_price_set", "datatype": dbt.type_string()},
+    {"name": "total_price_usd", "datatype": dbt.type_float()},
+    {"name": "total_tax_set", "datatype": dbt.type_string()},
+    {"name": "total_tax", "datatype": dbt.type_float()},
+    {"name": "source_name", "datatype": dbt.type_string()},
+    {"name": "subtotal_price", "datatype": dbt.type_float()},
+    {"name": "taxes_included", "datatype": "boolean"},
+    {"name": "total_weight", "datatype": dbt.type_numeric()},
+    {"name": "total_tip_received", "datatype": dbt.type_float()},
+    {"name": "landing_site_base_url", "datatype": dbt.type_string()},
+    {"name": "landing_site_ref", "datatype": dbt.type_string()},
+    {"name": "location_id", "datatype": dbt.type_numeric()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "note", "datatype": dbt.type_string()},
+    {"name": "number", "datatype": dbt.type_numeric()},
+    {"name": "order_number", "datatype": dbt.type_numeric()},
+    {"name": "cancel_reason", "datatype": dbt.type_string()},
+    {"name": "cancelled_at", "datatype": dbt.type_timestamp()},
+    {"name": "cart_token", "datatype": dbt.type_string()},
+    {"name": "checkout_token", "datatype": dbt.type_string()},
+    {"name": "closed_at", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "currency", "datatype": dbt.type_string()},
+    {"name": "customer_id", "datatype": dbt.type_numeric()},
+    {"name": "email", "datatype": dbt.type_string()},
+    {"name": "financial_status", "datatype": dbt.type_string()},
+    {"name": "fulfillment_status", "datatype": dbt.type_string()},
+    {"name": "processing_method", "datatype": dbt.type_string()},
+    {"name": "referring_site", "datatype": dbt.type_string()},
+    {"name": "billing_address_address_1", "datatype": dbt.type_string()},
+    {"name": "billing_address_address_2", "datatype": dbt.type_string()},
+    {"name": "billing_address_city", "datatype": dbt.type_string()},
+    {"name": "billing_address_company", "datatype": dbt.type_string()},
+    {"name": "billing_address_country", "datatype": dbt.type_string()},
+    {"name": "billing_address_country_code", "datatype": dbt.type_string()},
+    {"name": "billing_address_first_name", "datatype": dbt.type_string()},
+    {"name": "billing_address_last_name", "datatype": dbt.type_string()},
+    {"name": "billing_address_latitude", "datatype": dbt.type_string()},
+    {"name": "billing_address_longitude", "datatype": dbt.type_string()},
+    {"name": "billing_address_name", "datatype": dbt.type_string()},
+    {"name": "billing_address_phone", "datatype": dbt.type_string()},
+    {"name": "billing_address_province", "datatype": dbt.type_string()},
+    {"name": "billing_address_province_code", "datatype": dbt.type_string()},
+    {"name": "billing_address_zip", "datatype": dbt.type_string()},
+    {"name": "browser_ip", "datatype": dbt.type_string()},
+    {"name": "buyer_accepts_marketing", "datatype": "boolean"},
+    {"name": "total_shipping_price_set", "datatype": dbt.type_string()},
+    {"name": "shipping_address_address_1", "datatype": dbt.type_string()},
+    {"name": "shipping_address_address_2", "datatype": dbt.type_string()},
+    {"name": "shipping_address_city", "datatype": dbt.type_string()},
+    {"name": "shipping_address_company", "datatype": dbt.type_string()},
+    {"name": "shipping_address_country", "datatype": dbt.type_string()},
+    {"name": "shipping_address_country_code", "datatype": dbt.type_string()},
+    {"name": "shipping_address_first_name", "datatype": dbt.type_string()},
+    {"name": "shipping_address_last_name", "datatype": dbt.type_string()},
+    {"name": "shipping_address_latitude", "datatype": dbt.type_string()},
+    {"name": "shipping_address_longitude", "datatype": dbt.type_string()},
+    {"name": "shipping_address_name", "datatype": dbt.type_string()},
+    {"name": "shipping_address_phone", "datatype": dbt.type_string()},
+    {"name": "shipping_address_province", "datatype": dbt.type_string()},
+    {"name": "shipping_address_province_code", "datatype": dbt.type_string()},
+    {"name": "shipping_address_zip", "datatype": dbt.type_string()},
+    {"name": "test", "datatype": "boolean"},
+    {"name": "token", "datatype": dbt.type_string()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "app_id", "datatype": dbt.type_int()},
+    {"name": "checkout_id", "datatype": dbt.type_int()},
+    {"name": "client_details_user_agent", "datatype": dbt.type_string()},
+    {"name": "customer_locale", "datatype": dbt.type_string()},
+    {"name": "device_id", "datatype": dbt.type_string()},
+    {"name": "order_status_url", "datatype": dbt.type_string()},
+    {"name": "presentment_currency", "datatype": dbt.type_string()},
+    {"name": "confirmed", "datatype": "boolean"}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('order_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_line_columns.sql

@@ -0,0 +1,43 @@
+{% macro get_order_line_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "fulfillable_quantity", "datatype": dbt.type_numeric()},
+    {"name": "fulfillment_service", "datatype": dbt.type_string()},
+    {"name": "fulfillment_status", "datatype": dbt.type_string()},
+    {"name": "gift_card", "datatype": "boolean"},
+    {"name": "grams", "datatype": dbt.type_numeric()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "index", "datatype": dbt.type_numeric()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "order_id", "datatype": dbt.type_numeric()},
+    {"name": "pre_tax_price", "datatype": dbt.type_float()},
+    {"name": "pre_tax_price_set", "datatype": dbt.type_string()},
+    {"name": "price", "datatype": dbt.type_float()},
+    {"name": "price_set", "datatype": dbt.type_string()},
+    {"name": "product_id", "datatype": dbt.type_numeric()},
+    {"name": "property_charge_interval_frequency", "datatype": dbt.type_numeric()},
+    {"name": "property_for_shipping_jan_3_rd_2020", "datatype": dbt.type_string()},
+    {"name": "property_shipping_interval_frequency", "datatype": dbt.type_numeric()},
+    {"name": "property_shipping_interval_unit_type", "datatype": dbt.type_string()},
+    {"name": "property_subscription_id", "datatype": dbt.type_numeric()},
+    {"name": "quantity", "datatype": dbt.type_numeric()},
+    {"name": "requires_shipping", "datatype": "boolean"},
+    {"name": "sku", "datatype": dbt.type_string()},
+    {"name": "taxable", "datatype": "boolean"},
+    {"name": "tax_code", "datatype": dbt.type_string()},
+    {"name": "title", "datatype": dbt.type_string()},
+    {"name": "total_discount", "datatype": dbt.type_float()},
+    {"name": "total_discount_set", "datatype": dbt.type_string()},
+    {"name": "variant_id", "datatype": dbt.type_numeric()},
+    {"name": "variant_title", "datatype": dbt.type_string()},
+    {"name": "variant_inventory_management", "datatype": dbt.type_string()},
+    {"name": "vendor", "datatype": dbt.type_string()},
+    {"name": "properties", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('order_line_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_line_refund_columns.sql

@@ -0,0 +1,21 @@
+{% macro get_order_line_refund_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "location_id", "datatype": dbt.type_numeric()},
+    {"name": "order_line_id", "datatype": dbt.type_numeric()},
+    {"name": "subtotal", "datatype": dbt.type_numeric()},
+    {"name": "subtotal_set", "datatype": dbt.type_string()},
+    {"name": "total_tax", "datatype": dbt.type_numeric()},
+    {"name": "total_tax_set", "datatype": dbt.type_string()},
+    {"name": "quantity", "datatype": dbt.type_float()},
+    {"name": "refund_id", "datatype": dbt.type_numeric()},
+    {"name": "restock_type", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('order_line_refund_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_product_columns.sql

@@ -0,0 +1,22 @@
+{% macro get_product_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "handle", "datatype": dbt.type_string()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "product_type", "datatype": dbt.type_string()},
+    {"name": "published_at", "datatype": dbt.type_timestamp()},
+    {"name": "published_scope", "datatype": dbt.type_string()},
+    {"name": "title", "datatype": dbt.type_string()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "vendor", "datatype": dbt.type_string()},
+    {"name": "status", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('product_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_product_variant_columns.sql

@@ -0,0 +1,35 @@
+{% macro get_product_variant_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "product_id", "datatype": dbt.type_numeric()},
+    {"name": "inventory_item_id", "datatype": dbt.type_numeric()},
+    {"name": "image_id", "datatype": dbt.type_numeric()},
+    {"name": "title", "datatype": dbt.type_string()},
+    {"name": "price", "datatype": dbt.type_float()},
+    {"name": "sku", "datatype": dbt.type_string()},
+    {"name": "position", "datatype": dbt.type_numeric()},
+    {"name": "inventory_policy", "datatype": dbt.type_string()},
+    {"name": "compare_at_price", "datatype": dbt.type_float()},
+    {"name": "fulfillment_service", "datatype": dbt.type_string()},
+    {"name": "inventory_management", "datatype": dbt.type_string()},
+    {"name": "taxable", "datatype": "boolean"},
+    {"name": "barcode", "datatype": dbt.type_string()},
+    {"name": "grams", "datatype": dbt.type_float()},
+    {"name": "inventory_quantity", "datatype": dbt.type_numeric()},
+    {"name": "weight", "datatype": dbt.type_float()},
+    {"name": "weight_unit", "datatype": dbt.type_string()},
+    {"name": "option_1", "datatype": dbt.type_string()},
+    {"name": "option_2", "datatype": dbt.type_string()},
+    {"name": "option_3", "datatype": dbt.type_string()},
+    {"name": "tax_code", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('product_variant_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_refund_columns.sql

@@ -0,0 +1,17 @@
+{% macro get_refund_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "note", "datatype": dbt.type_string()},
+    {"name": "order_id", "datatype": dbt.type_numeric()},
+    {"name": "processed_at", "datatype": dbt.type_timestamp()},
+    {"name": "restock", "datatype": "boolean"},
+    {"name": "total_duties_set", "datatype": dbt.type_string()},
+    {"name": "user_id", "datatype": dbt.type_numeric()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}