Merge pull request #40 from fivetran/feature/revamp/update-field-sele…

…ction

Update field selection, etc

CHANGELOG.md

@@ -41,7 +41,18 @@
 - Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
 
 ## Updates:
-[PR #38](https://github.com/fivetran/dbt_shopify_source/pull/38) includes the following breaking changes:
+[PR #40](https://github.com/fivetran/dbt_shopify_source/pull/40) includes the following changes:
+- In the following tables, `*_timestamp` columns have been renamed to `*_at` for consistency:
+  - `stg_shopify__customer`
+  - `stg_shopify__order`
+  - `stg_shopify__product_variant`
+  - `stg_shopify__product`
+  - `stg_shopify__transaction`
+- The `is_requiring_shipping` field in `stg_shopify__order_line` has been renamed to `is_shipping_required`, just because it sounds better :)
+- TODO more stuff
+- By default, if the refunds table does not exist, the package will create an empty `stg_shopify__refunds` model. This can still be overwritten by setting `shopify__using_refund` to `false`. We did this because the refunds table is only created once your Shopify account has processed its first refund, and it is unnecessary overhead to have to monitor this and change your dbt_project accordingly. Thus, we took the approach of having this refund table be one that syncs as empty if it is not populated, then seamlessly switches to the source table once it exists.
+
+[PR #38](https://github.com/fivetran/dbt_shopify_source/pull/38) includes the following changes:
 - Esnured Postgres compatibility.
 - README updated for easier package use and navigation.
 - `shopify_<default_source_table_name>_identifier` variables added if an individual source table has a different name than the package expects.

README.md

@@ -85,6 +85,21 @@ vars:
     product_variant_pass_through_columns: []
 ```
 
+### Disable Models
+This package was designed with the intention that users have all relevant Shopify tables being synced by Fivetran. However, if you are a Shopify user that does not operate on returns or adjustments then you will not have the related source tables. As such, you may use the below variable configurations to disable the respective downstream models. All variables are `true` by default. Only add the below configuration if you are wishing to disable the models:
+
+```yml
+# dbt_project.yml
+
+...
+vars:
+  shopify__using_order_adjustment:  false  # true by default
+  shopify__using_order_line_refund: false  # true by default
+  shopify__using_refund:      false  # true by default
+```
+
+> By default, if the refunds table does not exist, the package will create an empty `stg_shopify__refunds` model. This can still be overwritten by setting `shopify__using_refund` to `false`. We did this because the refunds table is only created once your Shopify account has processed its first refund, and it is unnecessary overhead to have to monitor this and change your dbt_project accordingly. Thus, we took the approach of having this refund table be one that syncs as empty if it is not populated, then seamlessly switches to the source table once it exists.
+
 ### Changing the Build Schema
 By default this package will build the Shopify staging models within a schema titled (<target_schema> + `_stg_shopify`) in your target database. If this is not where you would like your staging Shopify data to be written to, add the following configuration to your `dbt_project.yml` file:
 

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_source_integration_tests_1
+      schema: shopify_source_integration_tests
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_source_integration_tests_1
+      schema: shopify_source_integration_tests
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_source_integration_tests_1
+      schema: shopify_source_integration_tests
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_source_integration_tests_1
+      schema: shopify_source_integration_tests
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_source_integration_tests_1
+      schema: shopify_source_integration_tests
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -4,6 +4,7 @@ profile: 'integration_tests'
 config-version: 2
 
 vars:
+  shopify_schema: shopify_source_integration_tests
   shopify_source:
     shopify_customer_identifier:          "shopify_customer_data"
     shopify_order_line_refund_identifier: "shopify_order_line_refund_data"

macros/get_customer_columns.sql

@@ -0,0 +1,33 @@
+{% macro get_customer_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "accepts_marketing", "datatype": "boolean"},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "default_address_id", "datatype": dbt.type_numeric()},
+    {"name": "email", "datatype": dbt.type_string()},
+    {"name": "first_name", "datatype": dbt.type_string()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "last_name", "datatype": dbt.type_string()},
+    {"name": "orders_count", "datatype": dbt.type_numeric()},
+    {"name": "phone", "datatype": dbt.type_string()},
+    {"name": "state", "datatype": dbt.type_string()},
+    {"name": "tax_exempt", "datatype": "boolean"},
+    {"name": "total_spent", "datatype": dbt.type_float()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "verified_email", "datatype": "boolean"},
+    {"name": "email_marketing_consent_consent_updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "email_marketing_consent_opt_in_level", "datatype": dbt.type_string()},
+    {"name": "email_marketing_consent_state", "datatype": dbt.type_string()},
+    {"name": "note", "datatype": dbt.type_string()},
+    {"name": "accepts_marketing_updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "marketing_opt_in_level", "datatype": dbt.type_string()},
+    {"name": "lifetime_duration", "datatype": dbt.type_string()},
+    {"name": "currency", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('customer_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_adjustment_columns.sql

@@ -0,0 +1,18 @@
+{% macro get_order_adjustment_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype":  dbt.type_numeric()},
+    {"name": "order_id", "datatype":  dbt.type_numeric()},
+    {"name": "refund_id", "datatype":  dbt.type_numeric()},
+    {"name": "amount", "datatype": dbt.type_float()},
+    {"name": "amount_set", "datatype": dbt.type_string()},
+    {"name": "tax_amount", "datatype": dbt.type_float()},
+    {"name": "tax_amount_set", "datatype": dbt.type_string()},
+    {"name": "kind", "datatype": dbt.type_string()},
+    {"name": "reason", "datatype": dbt.type_string()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_columns.sql

@@ -0,0 +1,93 @@
+{% macro get_order_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "processed_at", "datatype": dbt.type_timestamp()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "user_id", "datatype": dbt.type_numeric()},
+    {"name": "total_discounts", "datatype": dbt.type_float()},
+    {"name": "total_discounts_set", "datatype": dbt.type_string()},
+    {"name": "total_line_items_price", "datatype": dbt.type_float()},
+    {"name": "total_line_items_price_set", "datatype": dbt.type_string()},
+    {"name": "total_price", "datatype": dbt.type_float()},
+    {"name": "total_price_set", "datatype": dbt.type_string()},
+    {"name": "total_price_usd", "datatype": dbt.type_float()},
+    {"name": "total_tax_set", "datatype": dbt.type_string()},
+    {"name": "total_tax", "datatype": dbt.type_float()},
+    {"name": "source_name", "datatype": dbt.type_string()},
+    {"name": "subtotal_price", "datatype": dbt.type_float()},
+    {"name": "taxes_included", "datatype": "boolean"},
+    {"name": "total_weight", "datatype": dbt.type_numeric()},
+    {"name": "total_tip_received", "datatype": dbt.type_float()},
+    {"name": "landing_site_base_url", "datatype": dbt.type_string()},
+    {"name": "landing_site_ref", "datatype": dbt.type_string()},
+    {"name": "location_id", "datatype": dbt.type_numeric()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "note", "datatype": dbt.type_string()},
+    {"name": "number", "datatype": dbt.type_numeric()},
+    {"name": "order_number", "datatype": dbt.type_numeric()},
+    {"name": "cancel_reason", "datatype": dbt.type_string()},
+    {"name": "cancelled_at", "datatype": dbt.type_timestamp()},
+    {"name": "cart_token", "datatype": dbt.type_string()},
+    {"name": "checkout_token", "datatype": dbt.type_string()},
+    {"name": "closed_at", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "currency", "datatype": dbt.type_string()},
+    {"name": "customer_id", "datatype": dbt.type_numeric()},
+    {"name": "email", "datatype": dbt.type_string()},
+    {"name": "financial_status", "datatype": dbt.type_string()},
+    {"name": "fulfillment_status", "datatype": dbt.type_string()},
+    {"name": "processing_method", "datatype": dbt.type_string()},
+    {"name": "referring_site", "datatype": dbt.type_string()},
+    {"name": "billing_address_address_1", "datatype": dbt.type_string()},
+    {"name": "billing_address_address_2", "datatype": dbt.type_string()},
+    {"name": "billing_address_city", "datatype": dbt.type_string()},
+    {"name": "billing_address_company", "datatype": dbt.type_string()},
+    {"name": "billing_address_country", "datatype": dbt.type_string()},
+    {"name": "billing_address_country_code", "datatype": dbt.type_string()},
+    {"name": "billing_address_first_name", "datatype": dbt.type_string()},
+    {"name": "billing_address_last_name", "datatype": dbt.type_string()},
+    {"name": "billing_address_latitude", "datatype": dbt.type_string()},
+    {"name": "billing_address_longitude", "datatype": dbt.type_string()},
+    {"name": "billing_address_name", "datatype": dbt.type_string()},
+    {"name": "billing_address_phone", "datatype": dbt.type_string()},
+    {"name": "billing_address_province", "datatype": dbt.type_string()},
+    {"name": "billing_address_province_code", "datatype": dbt.type_string()},
+    {"name": "billing_address_zip", "datatype": dbt.type_string()},
+    {"name": "browser_ip", "datatype": dbt.type_string()},
+    {"name": "buyer_accepts_marketing", "datatype": "boolean"},
+    {"name": "total_shipping_price_set", "datatype": dbt.type_string()},
+    {"name": "shipping_address_address_1", "datatype": dbt.type_string()},
+    {"name": "shipping_address_address_2", "datatype": dbt.type_string()},
+    {"name": "shipping_address_city", "datatype": dbt.type_string()},
+    {"name": "shipping_address_company", "datatype": dbt.type_string()},
+    {"name": "shipping_address_country", "datatype": dbt.type_string()},
+    {"name": "shipping_address_country_code", "datatype": dbt.type_string()},
+    {"name": "shipping_address_first_name", "datatype": dbt.type_string()},
+    {"name": "shipping_address_last_name", "datatype": dbt.type_string()},
+    {"name": "shipping_address_latitude", "datatype": dbt.type_string()},
+    {"name": "shipping_address_longitude", "datatype": dbt.type_string()},
+    {"name": "shipping_address_name", "datatype": dbt.type_string()},
+    {"name": "shipping_address_phone", "datatype": dbt.type_string()},
+    {"name": "shipping_address_province", "datatype": dbt.type_string()},
+    {"name": "shipping_address_province_code", "datatype": dbt.type_string()},
+    {"name": "shipping_address_zip", "datatype": dbt.type_string()},
+    {"name": "test", "datatype": "boolean"},
+    {"name": "token", "datatype": dbt.type_string()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "app_id", "datatype": dbt.type_int()},
+    {"name": "checkout_id", "datatype": dbt.type_int()},
+    {"name": "client_details_user_agent", "datatype": dbt.type_string()},
+    {"name": "customer_locale", "datatype": dbt.type_string()},
+    {"name": "device_id", "datatype": dbt.type_string()},
+    {"name": "order_status_url", "datatype": dbt.type_string()},
+    {"name": "presentment_currency", "datatype": dbt.type_string()},
+    {"name": "confirmed", "datatype": "boolean"}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('order_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_line_columns.sql

@@ -0,0 +1,43 @@
+{% macro get_order_line_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "fulfillable_quantity", "datatype": dbt.type_numeric()},
+    {"name": "fulfillment_service", "datatype": dbt.type_string()},
+    {"name": "fulfillment_status", "datatype": dbt.type_string()},
+    {"name": "gift_card", "datatype": "boolean"},
+    {"name": "grams", "datatype": dbt.type_numeric()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "index", "datatype": dbt.type_numeric()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "order_id", "datatype": dbt.type_numeric()},
+    {"name": "pre_tax_price", "datatype": dbt.type_float()},
+    {"name": "pre_tax_price_set", "datatype": dbt.type_string()},
+    {"name": "price", "datatype": dbt.type_float()},
+    {"name": "price_set", "datatype": dbt.type_string()},
+    {"name": "product_id", "datatype": dbt.type_numeric()},
+    {"name": "property_charge_interval_frequency", "datatype": dbt.type_numeric()},
+    {"name": "property_for_shipping_jan_3_rd_2020", "datatype": dbt.type_string()},
+    {"name": "property_shipping_interval_frequency", "datatype": dbt.type_numeric()},
+    {"name": "property_shipping_interval_unit_type", "datatype": dbt.type_string()},
+    {"name": "property_subscription_id", "datatype": dbt.type_numeric()},
+    {"name": "quantity", "datatype": dbt.type_numeric()},
+    {"name": "requires_shipping", "datatype": "boolean"},
+    {"name": "sku", "datatype": dbt.type_string()},
+    {"name": "taxable", "datatype": "boolean"},
+    {"name": "tax_code", "datatype": dbt.type_string()},
+    {"name": "title", "datatype": dbt.type_string()},
+    {"name": "total_discount", "datatype": dbt.type_float()},
+    {"name": "total_discount_set", "datatype": dbt.type_string()},
+    {"name": "variant_id", "datatype": dbt.type_numeric()},
+    {"name": "variant_title", "datatype": dbt.type_string()},
+    {"name": "variant_inventory_management", "datatype": dbt.type_string()},
+    {"name": "vendor", "datatype": dbt.type_string()},
+    {"name": "properties", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('order_line_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_order_line_refund_columns.sql

@@ -0,0 +1,21 @@
+{% macro get_order_line_refund_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "location_id", "datatype": dbt.type_numeric()},
+    {"name": "order_line_id", "datatype": dbt.type_numeric()},
+    {"name": "subtotal", "datatype": dbt.type_numeric()},
+    {"name": "subtotal_set", "datatype": dbt.type_string()},
+    {"name": "total_tax", "datatype": dbt.type_numeric()},
+    {"name": "total_tax_set", "datatype": dbt.type_string()},
+    {"name": "quantity", "datatype": dbt.type_float()},
+    {"name": "refund_id", "datatype": dbt.type_numeric()},
+    {"name": "restock_type", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('order_line_refund_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_product_columns.sql

@@ -0,0 +1,22 @@
+{% macro get_product_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "handle", "datatype": dbt.type_string()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "product_type", "datatype": dbt.type_string()},
+    {"name": "published_at", "datatype": dbt.type_timestamp()},
+    {"name": "published_scope", "datatype": dbt.type_string()},
+    {"name": "title", "datatype": dbt.type_string()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "vendor", "datatype": dbt.type_string()},
+    {"name": "status", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('product_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_product_variant_columns.sql

@@ -0,0 +1,35 @@
+{% macro get_product_variant_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "product_id", "datatype": dbt.type_numeric()},
+    {"name": "inventory_item_id", "datatype": dbt.type_numeric()},
+    {"name": "image_id", "datatype": dbt.type_numeric()},
+    {"name": "title", "datatype": dbt.type_string()},
+    {"name": "price", "datatype": dbt.type_float()},
+    {"name": "sku", "datatype": dbt.type_string()},
+    {"name": "position", "datatype": dbt.type_numeric()},
+    {"name": "inventory_policy", "datatype": dbt.type_string()},
+    {"name": "compare_at_price", "datatype": dbt.type_float()},
+    {"name": "fulfillment_service", "datatype": dbt.type_string()},
+    {"name": "inventory_management", "datatype": dbt.type_string()},
+    {"name": "taxable", "datatype": "boolean"},
+    {"name": "barcode", "datatype": dbt.type_string()},
+    {"name": "grams", "datatype": dbt.type_float()},
+    {"name": "inventory_quantity", "datatype": dbt.type_numeric()},
+    {"name": "weight", "datatype": dbt.type_float()},
+    {"name": "weight_unit", "datatype": dbt.type_string()},
+    {"name": "option_1", "datatype": dbt.type_string()},
+    {"name": "option_2", "datatype": dbt.type_string()},
+    {"name": "option_3", "datatype": dbt.type_string()},
+    {"name": "tax_code", "datatype": dbt.type_string()}
+] %}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('product_variant_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_refund_columns.sql

@@ -0,0 +1,17 @@
+{% macro get_refund_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_numeric()},
+    {"name": "note", "datatype": dbt.type_string()},
+    {"name": "order_id", "datatype": dbt.type_numeric()},
+    {"name": "processed_at", "datatype": dbt.type_timestamp()},
+    {"name": "restock", "datatype": "boolean"},
+    {"name": "total_duties_set", "datatype": dbt.type_string()},
+    {"name": "user_id", "datatype": dbt.type_numeric()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}