diff --git a/CHANGELOG.md b/CHANGELOG.md index 370ebe7..01fb853 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,18 @@ - Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`. ## Updates: -[PR #38](https://github.com/fivetran/dbt_shopify_source/pull/38) includes the following breaking changes: +[PR #40](https://github.com/fivetran/dbt_shopify_source/pull/40) includes the following changes: +- In the following tables, `*_timestamp` columns have been renamed to `*_at` for consistency: + - `stg_shopify__customer` + - `stg_shopify__order` + - `stg_shopify__product_variant` + - `stg_shopify__product` + - `stg_shopify__transaction` +- The `is_requiring_shipping` field in `stg_shopify__order_line` has been renamed to `is_shipping_required`, just because it sounds better :) +- TODO more stuff +- By default, if the refunds table does not exist, the package will create an empty `stg_shopify__refunds` model. This can still be overwritten by setting `shopify__using_refund` to `false`. We did this because the refunds table is only created once your Shopify account has processed its first refund, and it is unnecessary overhead to have to monitor this and change your dbt_project accordingly. Thus, we took the approach of having this refund table be one that syncs as empty if it is not populated, then seamlessly switches to the source table once it exists. + +[PR #38](https://github.com/fivetran/dbt_shopify_source/pull/38) includes the following changes: - Esnured Postgres compatibility. - README updated for easier package use and navigation. - `shopify__identifier` variables added if an individual source table has a different name than the package expects. diff --git a/README.md b/README.md index b19adc4..b086147 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,21 @@ vars: product_variant_pass_through_columns: [] ``` +### Disable Models +This package was designed with the intention that users have all relevant Shopify tables being synced by Fivetran. However, if you are a Shopify user that does not operate on returns or adjustments then you will not have the related source tables. As such, you may use the below variable configurations to disable the respective downstream models. All variables are `true` by default. Only add the below configuration if you are wishing to disable the models: + +```yml +# dbt_project.yml + +... +vars: + shopify__using_order_adjustment: false # true by default + shopify__using_order_line_refund: false # true by default + shopify__using_refund: false # true by default +``` + +> By default, if the refunds table does not exist, the package will create an empty `stg_shopify__refunds` model. This can still be overwritten by setting `shopify__using_refund` to `false`. We did this because the refunds table is only created once your Shopify account has processed its first refund, and it is unnecessary overhead to have to monitor this and change your dbt_project accordingly. Thus, we took the approach of having this refund table be one that syncs as empty if it is not populated, then seamlessly switches to the source table once it exists. + ### Changing the Build Schema By default this package will build the Shopify staging models within a schema titled ( + `_stg_shopify`) in your target database. If this is not where you would like your staging Shopify data to be written to, add the following configuration to your `dbt_project.yml` file: diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index fbf1d5a..594f55e 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: shopify_source_integration_tests_1 + schema: shopify_source_integration_tests threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: shopify_source_integration_tests_1 + schema: shopify_source_integration_tests threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: shopify_source_integration_tests_1 + schema: shopify_source_integration_tests threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: shopify_source_integration_tests_1 + schema: shopify_source_integration_tests threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: shopify_source_integration_tests_1 + schema: shopify_source_integration_tests threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 48ec16f..a471edd 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -4,6 +4,7 @@ profile: 'integration_tests' config-version: 2 vars: + shopify_schema: shopify_source_integration_tests shopify_source: shopify_customer_identifier: "shopify_customer_data" shopify_order_line_refund_identifier: "shopify_order_line_refund_data" diff --git a/macros/get_customer_columns.sql b/macros/get_customer_columns.sql new file mode 100644 index 0000000..6167587 --- /dev/null +++ b/macros/get_customer_columns.sql @@ -0,0 +1,33 @@ +{% macro get_customer_columns() %} + +{% set columns = [ + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "accepts_marketing", "datatype": "boolean"}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "default_address_id", "datatype": dbt.type_numeric()}, + {"name": "email", "datatype": dbt.type_string()}, + {"name": "first_name", "datatype": dbt.type_string()}, + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "last_name", "datatype": dbt.type_string()}, + {"name": "orders_count", "datatype": dbt.type_numeric()}, + {"name": "phone", "datatype": dbt.type_string()}, + {"name": "state", "datatype": dbt.type_string()}, + {"name": "tax_exempt", "datatype": "boolean"}, + {"name": "total_spent", "datatype": dbt.type_float()}, + {"name": "updated_at", "datatype": dbt.type_timestamp()}, + {"name": "verified_email", "datatype": "boolean"}, + {"name": "email_marketing_consent_consent_updated_at", "datatype": dbt.type_timestamp()}, + {"name": "email_marketing_consent_opt_in_level", "datatype": dbt.type_string()}, + {"name": "email_marketing_consent_state", "datatype": dbt.type_string()}, + {"name": "note", "datatype": dbt.type_string()}, + {"name": "accepts_marketing_updated_at", "datatype": dbt.type_timestamp()}, + {"name": "marketing_opt_in_level", "datatype": dbt.type_string()}, + {"name": "lifetime_duration", "datatype": dbt.type_string()}, + {"name": "currency", "datatype": dbt.type_string()} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('customer_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_order_adjustment_columns.sql b/macros/get_order_adjustment_columns.sql new file mode 100644 index 0000000..77b9028 --- /dev/null +++ b/macros/get_order_adjustment_columns.sql @@ -0,0 +1,18 @@ +{% macro get_order_adjustment_columns() %} + +{% set columns = [ + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "order_id", "datatype": dbt.type_numeric()}, + {"name": "refund_id", "datatype": dbt.type_numeric()}, + {"name": "amount", "datatype": dbt.type_float()}, + {"name": "amount_set", "datatype": dbt.type_string()}, + {"name": "tax_amount", "datatype": dbt.type_float()}, + {"name": "tax_amount_set", "datatype": dbt.type_string()}, + {"name": "kind", "datatype": dbt.type_string()}, + {"name": "reason", "datatype": dbt.type_string()}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()} +] %} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_order_columns.sql b/macros/get_order_columns.sql new file mode 100644 index 0000000..c3b9a16 --- /dev/null +++ b/macros/get_order_columns.sql @@ -0,0 +1,93 @@ +{% macro get_order_columns() %} + +{% set columns = [ + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "processed_at", "datatype": dbt.type_timestamp()}, + {"name": "updated_at", "datatype": dbt.type_timestamp()}, + {"name": "user_id", "datatype": dbt.type_numeric()}, + {"name": "total_discounts", "datatype": dbt.type_float()}, + {"name": "total_discounts_set", "datatype": dbt.type_string()}, + {"name": "total_line_items_price", "datatype": dbt.type_float()}, + {"name": "total_line_items_price_set", "datatype": dbt.type_string()}, + {"name": "total_price", "datatype": dbt.type_float()}, + {"name": "total_price_set", "datatype": dbt.type_string()}, + {"name": "total_price_usd", "datatype": dbt.type_float()}, + {"name": "total_tax_set", "datatype": dbt.type_string()}, + {"name": "total_tax", "datatype": dbt.type_float()}, + {"name": "source_name", "datatype": dbt.type_string()}, + {"name": "subtotal_price", "datatype": dbt.type_float()}, + {"name": "taxes_included", "datatype": "boolean"}, + {"name": "total_weight", "datatype": dbt.type_numeric()}, + {"name": "total_tip_received", "datatype": dbt.type_float()}, + {"name": "landing_site_base_url", "datatype": dbt.type_string()}, + {"name": "landing_site_ref", "datatype": dbt.type_string()}, + {"name": "location_id", "datatype": dbt.type_numeric()}, + {"name": "name", "datatype": dbt.type_string()}, + {"name": "note", "datatype": dbt.type_string()}, + {"name": "number", "datatype": dbt.type_numeric()}, + {"name": "order_number", "datatype": dbt.type_numeric()}, + {"name": "cancel_reason", "datatype": dbt.type_string()}, + {"name": "cancelled_at", "datatype": dbt.type_timestamp()}, + {"name": "cart_token", "datatype": dbt.type_string()}, + {"name": "checkout_token", "datatype": dbt.type_string()}, + {"name": "closed_at", "datatype": dbt.type_timestamp()}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "currency", "datatype": dbt.type_string()}, + {"name": "customer_id", "datatype": dbt.type_numeric()}, + {"name": "email", "datatype": dbt.type_string()}, + {"name": "financial_status", "datatype": dbt.type_string()}, + {"name": "fulfillment_status", "datatype": dbt.type_string()}, + {"name": "processing_method", "datatype": dbt.type_string()}, + {"name": "referring_site", "datatype": dbt.type_string()}, + {"name": "billing_address_address_1", "datatype": dbt.type_string()}, + {"name": "billing_address_address_2", "datatype": dbt.type_string()}, + {"name": "billing_address_city", "datatype": dbt.type_string()}, + {"name": "billing_address_company", "datatype": dbt.type_string()}, + {"name": "billing_address_country", "datatype": dbt.type_string()}, + {"name": "billing_address_country_code", "datatype": dbt.type_string()}, + {"name": "billing_address_first_name", "datatype": dbt.type_string()}, + {"name": "billing_address_last_name", "datatype": dbt.type_string()}, + {"name": "billing_address_latitude", "datatype": dbt.type_string()}, + {"name": "billing_address_longitude", "datatype": dbt.type_string()}, + {"name": "billing_address_name", "datatype": dbt.type_string()}, + {"name": "billing_address_phone", "datatype": dbt.type_string()}, + {"name": "billing_address_province", "datatype": dbt.type_string()}, + {"name": "billing_address_province_code", "datatype": dbt.type_string()}, + {"name": "billing_address_zip", "datatype": dbt.type_string()}, + {"name": "browser_ip", "datatype": dbt.type_string()}, + {"name": "buyer_accepts_marketing", "datatype": "boolean"}, + {"name": "total_shipping_price_set", "datatype": dbt.type_string()}, + {"name": "shipping_address_address_1", "datatype": dbt.type_string()}, + {"name": "shipping_address_address_2", "datatype": dbt.type_string()}, + {"name": "shipping_address_city", "datatype": dbt.type_string()}, + {"name": "shipping_address_company", "datatype": dbt.type_string()}, + {"name": "shipping_address_country", "datatype": dbt.type_string()}, + {"name": "shipping_address_country_code", "datatype": dbt.type_string()}, + {"name": "shipping_address_first_name", "datatype": dbt.type_string()}, + {"name": "shipping_address_last_name", "datatype": dbt.type_string()}, + {"name": "shipping_address_latitude", "datatype": dbt.type_string()}, + {"name": "shipping_address_longitude", "datatype": dbt.type_string()}, + {"name": "shipping_address_name", "datatype": dbt.type_string()}, + {"name": "shipping_address_phone", "datatype": dbt.type_string()}, + {"name": "shipping_address_province", "datatype": dbt.type_string()}, + {"name": "shipping_address_province_code", "datatype": dbt.type_string()}, + {"name": "shipping_address_zip", "datatype": dbt.type_string()}, + {"name": "test", "datatype": "boolean"}, + {"name": "token", "datatype": dbt.type_string()}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "_fivetran_deleted", "datatype": "boolean"}, + {"name": "app_id", "datatype": dbt.type_int()}, + {"name": "checkout_id", "datatype": dbt.type_int()}, + {"name": "client_details_user_agent", "datatype": dbt.type_string()}, + {"name": "customer_locale", "datatype": dbt.type_string()}, + {"name": "device_id", "datatype": dbt.type_string()}, + {"name": "order_status_url", "datatype": dbt.type_string()}, + {"name": "presentment_currency", "datatype": dbt.type_string()}, + {"name": "confirmed", "datatype": "boolean"} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('order_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_order_line_columns.sql b/macros/get_order_line_columns.sql new file mode 100644 index 0000000..574cc72 --- /dev/null +++ b/macros/get_order_line_columns.sql @@ -0,0 +1,43 @@ +{% macro get_order_line_columns() %} + +{% set columns = [ + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "fulfillable_quantity", "datatype": dbt.type_numeric()}, + {"name": "fulfillment_service", "datatype": dbt.type_string()}, + {"name": "fulfillment_status", "datatype": dbt.type_string()}, + {"name": "gift_card", "datatype": "boolean"}, + {"name": "grams", "datatype": dbt.type_numeric()}, + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "index", "datatype": dbt.type_numeric()}, + {"name": "name", "datatype": dbt.type_string()}, + {"name": "order_id", "datatype": dbt.type_numeric()}, + {"name": "pre_tax_price", "datatype": dbt.type_float()}, + {"name": "pre_tax_price_set", "datatype": dbt.type_string()}, + {"name": "price", "datatype": dbt.type_float()}, + {"name": "price_set", "datatype": dbt.type_string()}, + {"name": "product_id", "datatype": dbt.type_numeric()}, + {"name": "property_charge_interval_frequency", "datatype": dbt.type_numeric()}, + {"name": "property_for_shipping_jan_3_rd_2020", "datatype": dbt.type_string()}, + {"name": "property_shipping_interval_frequency", "datatype": dbt.type_numeric()}, + {"name": "property_shipping_interval_unit_type", "datatype": dbt.type_string()}, + {"name": "property_subscription_id", "datatype": dbt.type_numeric()}, + {"name": "quantity", "datatype": dbt.type_numeric()}, + {"name": "requires_shipping", "datatype": "boolean"}, + {"name": "sku", "datatype": dbt.type_string()}, + {"name": "taxable", "datatype": "boolean"}, + {"name": "tax_code", "datatype": dbt.type_string()}, + {"name": "title", "datatype": dbt.type_string()}, + {"name": "total_discount", "datatype": dbt.type_float()}, + {"name": "total_discount_set", "datatype": dbt.type_string()}, + {"name": "variant_id", "datatype": dbt.type_numeric()}, + {"name": "variant_title", "datatype": dbt.type_string()}, + {"name": "variant_inventory_management", "datatype": dbt.type_string()}, + {"name": "vendor", "datatype": dbt.type_string()}, + {"name": "properties", "datatype": dbt.type_string()} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('order_line_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_order_line_refund_columns.sql b/macros/get_order_line_refund_columns.sql new file mode 100644 index 0000000..818757b --- /dev/null +++ b/macros/get_order_line_refund_columns.sql @@ -0,0 +1,21 @@ +{% macro get_order_line_refund_columns() %} + +{% set columns = [ + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "location_id", "datatype": dbt.type_numeric()}, + {"name": "order_line_id", "datatype": dbt.type_numeric()}, + {"name": "subtotal", "datatype": dbt.type_numeric()}, + {"name": "subtotal_set", "datatype": dbt.type_string()}, + {"name": "total_tax", "datatype": dbt.type_numeric()}, + {"name": "total_tax_set", "datatype": dbt.type_string()}, + {"name": "quantity", "datatype": dbt.type_float()}, + {"name": "refund_id", "datatype": dbt.type_numeric()}, + {"name": "restock_type", "datatype": dbt.type_string()} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('order_line_refund_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_product_columns.sql b/macros/get_product_columns.sql new file mode 100644 index 0000000..28c2254 --- /dev/null +++ b/macros/get_product_columns.sql @@ -0,0 +1,22 @@ +{% macro get_product_columns() %} + +{% set columns = [ + {"name": "_fivetran_deleted", "datatype": "boolean"}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "handle", "datatype": dbt.type_string()}, + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "product_type", "datatype": dbt.type_string()}, + {"name": "published_at", "datatype": dbt.type_timestamp()}, + {"name": "published_scope", "datatype": dbt.type_string()}, + {"name": "title", "datatype": dbt.type_string()}, + {"name": "updated_at", "datatype": dbt.type_timestamp()}, + {"name": "vendor", "datatype": dbt.type_string()}, + {"name": "status", "datatype": dbt.type_string()} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('product_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_product_variant_columns.sql b/macros/get_product_variant_columns.sql new file mode 100644 index 0000000..5738745 --- /dev/null +++ b/macros/get_product_variant_columns.sql @@ -0,0 +1,35 @@ +{% macro get_product_variant_columns() %} + +{% set columns = [ + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "updated_at", "datatype": dbt.type_timestamp()}, + {"name": "product_id", "datatype": dbt.type_numeric()}, + {"name": "inventory_item_id", "datatype": dbt.type_numeric()}, + {"name": "image_id", "datatype": dbt.type_numeric()}, + {"name": "title", "datatype": dbt.type_string()}, + {"name": "price", "datatype": dbt.type_float()}, + {"name": "sku", "datatype": dbt.type_string()}, + {"name": "position", "datatype": dbt.type_numeric()}, + {"name": "inventory_policy", "datatype": dbt.type_string()}, + {"name": "compare_at_price", "datatype": dbt.type_float()}, + {"name": "fulfillment_service", "datatype": dbt.type_string()}, + {"name": "inventory_management", "datatype": dbt.type_string()}, + {"name": "taxable", "datatype": "boolean"}, + {"name": "barcode", "datatype": dbt.type_string()}, + {"name": "grams", "datatype": dbt.type_float()}, + {"name": "inventory_quantity", "datatype": dbt.type_numeric()}, + {"name": "weight", "datatype": dbt.type_float()}, + {"name": "weight_unit", "datatype": dbt.type_string()}, + {"name": "option_1", "datatype": dbt.type_string()}, + {"name": "option_2", "datatype": dbt.type_string()}, + {"name": "option_3", "datatype": dbt.type_string()}, + {"name": "tax_code", "datatype": dbt.type_string()} +] %} + +{{ fivetran_utils.add_pass_through_columns(columns, var('product_variant_pass_through_columns')) }} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_refund_columns.sql b/macros/get_refund_columns.sql new file mode 100644 index 0000000..affce51 --- /dev/null +++ b/macros/get_refund_columns.sql @@ -0,0 +1,17 @@ +{% macro get_refund_columns() %} + +{% set columns = [ + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "note", "datatype": dbt.type_string()}, + {"name": "order_id", "datatype": dbt.type_numeric()}, + {"name": "processed_at", "datatype": dbt.type_timestamp()}, + {"name": "restock", "datatype": "boolean"}, + {"name": "total_duties_set", "datatype": dbt.type_string()}, + {"name": "user_id", "datatype": dbt.type_numeric()} +] %} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/get_transaction_columns.sql b/macros/get_transaction_columns.sql new file mode 100644 index 0000000..2835938 --- /dev/null +++ b/macros/get_transaction_columns.sql @@ -0,0 +1,45 @@ +{% macro get_transaction_columns() %} + +{% set columns = [ + {"name": "id", "datatype": dbt.type_numeric()}, + {"name": "order_id", "datatype": dbt.type_numeric()}, + {"name": "refund_id", "datatype": dbt.type_numeric()}, + {"name": "amount", "datatype": dbt.type_numeric()}, + {"name": "created_at", "datatype": dbt.type_timestamp()}, + {"name": "processed_at", "datatype": dbt.type_timestamp()}, + {"name": "device_id", "datatype": dbt.type_numeric()}, + {"name": "gateway", "datatype": dbt.type_string()}, + {"name": "source_name", "datatype": dbt.type_string()}, + {"name": "message", "datatype": dbt.type_string()}, + {"name": "currency", "datatype": dbt.type_string()}, + {"name": "location_id", "datatype": dbt.type_numeric()}, + {"name": "parent_id", "datatype": dbt.type_numeric()}, + {"name": "payment_avs_result_code", "datatype": dbt.type_string()}, + {"name": "payment_credit_card_bin", "datatype": dbt.type_string()}, + {"name": "payment_cvv_result_code", "datatype": dbt.type_string()}, + {"name": "payment_credit_card_number", "datatype": dbt.type_string()}, + {"name": "payment_credit_card_company", "datatype": dbt.type_string()}, + {"name": "kind", "datatype": dbt.type_string()}, + {"name": "receipt", "datatype": dbt.type_string()}, + {"name": "currency_exchange_id", "datatype": dbt.type_numeric()}, + {"name": "currency_exchange_adjustment", "datatype": dbt.type_numeric()}, + {"name": "currency_exchange_original_amount", "datatype": dbt.type_numeric()}, + {"name": "currency_exchange_final_amount", "datatype": dbt.type_numeric()}, + {"name": "currency_exchange_currency", "datatype": dbt.type_string()}, + {"name": "error_code", "datatype": dbt.type_string()}, + {"name": "status", "datatype": dbt.type_string()}, + {"name": "test", "datatype": "boolean"}, + {"name": "user_id", "datatype": dbt.type_numeric()}, + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "authorization_expires_at", "datatype": dbt.type_timestamp()} +] %} + +{% if target.type in ('redshift','postgres') %} + {{ columns.append({"name": "authorization", "datatype": dbt.type_string(), "quote": True, "alias": "authorization_code"}) }} +{% else %} + {{ columns.append({"name": "authorization", "datatype": dbt.type_string(), "alias": "authorization_code"}) }} +{% endif %} + +{{ return(columns) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/staging_columns.sql b/macros/staging_columns.sql deleted file mode 100644 index a147538..0000000 --- a/macros/staging_columns.sql +++ /dev/null @@ -1,287 +0,0 @@ -{% macro get_order_columns() %} - -{% set columns = [ - {"name": "id", "datatype": dbt.type_numeric(), "alias": "order_id"}, - {"name": "processed_at", "datatype": dbt.type_timestamp(), "alias": "processed_timestamp"}, - {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, - {"name": "user_id", "datatype": dbt.type_numeric()}, - {"name": "total_discounts", "datatype": dbt.type_float()}, - {"name": "total_line_items_price", "datatype": dbt.type_float()}, - {"name": "total_price", "datatype": dbt.type_float()}, - {"name": "total_tax", "datatype": dbt.type_float()}, - {"name": "source_name", "datatype": dbt.type_string()}, - {"name": "subtotal_price", "datatype": dbt.type_float()}, - {"name": "taxes_included", "datatype": "boolean", "alias": "has_taxes_included"}, - {"name": "total_weight", "datatype": dbt.type_numeric()}, - {"name": "landing_site_base_url", "datatype": dbt.type_string()}, - {"name": "location_id", "datatype": dbt.type_numeric()}, - {"name": "name", "datatype": dbt.type_string()}, - {"name": "note", "datatype": dbt.type_string()}, - {"name": "number", "datatype": dbt.type_numeric()}, - {"name": "order_number", "datatype": dbt.type_numeric()}, - {"name": "cancel_reason", "datatype": dbt.type_string()}, - {"name": "cancelled_at", "datatype": dbt.type_timestamp(), "alias": "cancelled_timestamp"}, - {"name": "cart_token", "datatype": dbt.type_string()}, - {"name": "checkout_token", "datatype": dbt.type_string()}, - {"name": "closed_at", "datatype": dbt.type_timestamp(), "alias": "closed_timestamp"}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "currency", "datatype": dbt.type_string()}, - {"name": "customer_id", "datatype": dbt.type_numeric()}, - {"name": "email", "datatype": dbt.type_string()}, - {"name": "financial_status", "datatype": dbt.type_string()}, - {"name": "fulfillment_status", "datatype": dbt.type_string()}, - {"name": "processing_method", "datatype": dbt.type_string()}, - {"name": "referring_site", "datatype": dbt.type_string()}, - {"name": "billing_address_address_1", "datatype": dbt.type_string()}, - {"name": "billing_address_address_2", "datatype": dbt.type_string()}, - {"name": "billing_address_city", "datatype": dbt.type_string()}, - {"name": "billing_address_company", "datatype": dbt.type_string()}, - {"name": "billing_address_country", "datatype": dbt.type_string()}, - {"name": "billing_address_country_code", "datatype": dbt.type_string()}, - {"name": "billing_address_first_name", "datatype": dbt.type_string()}, - {"name": "billing_address_last_name", "datatype": dbt.type_string()}, - {"name": "billing_address_latitude", "datatype": dbt.type_string()}, - {"name": "billing_address_longitude", "datatype": dbt.type_string()}, - {"name": "billing_address_name", "datatype": dbt.type_string()}, - {"name": "billing_address_phone", "datatype": dbt.type_string()}, - {"name": "billing_address_province", "datatype": dbt.type_string()}, - {"name": "billing_address_province_code", "datatype": dbt.type_string()}, - {"name": "billing_address_zip", "datatype": dbt.type_string()}, - {"name": "browser_ip", "datatype": dbt.type_string()}, - {"name": "buyer_accepts_marketing", "datatype": "boolean", "alias": "has_buyer_accepted_marketing"}, - {"name": "total_shipping_price_set", "datatype": dbt.type_string()}, - {"name": "shipping_address_address_1", "datatype": dbt.type_string()}, - {"name": "shipping_address_address_2", "datatype": dbt.type_string()}, - {"name": "shipping_address_city", "datatype": dbt.type_string()}, - {"name": "shipping_address_company", "datatype": dbt.type_string()}, - {"name": "shipping_address_country", "datatype": dbt.type_string()}, - {"name": "shipping_address_country_code", "datatype": dbt.type_string()}, - {"name": "shipping_address_first_name", "datatype": dbt.type_string()}, - {"name": "shipping_address_last_name", "datatype": dbt.type_string()}, - {"name": "shipping_address_latitude", "datatype": dbt.type_string()}, - {"name": "shipping_address_longitude", "datatype": dbt.type_string()}, - {"name": "shipping_address_name", "datatype": dbt.type_string()}, - {"name": "shipping_address_phone", "datatype": dbt.type_string()}, - {"name": "shipping_address_province", "datatype": dbt.type_string()}, - {"name": "shipping_address_province_code", "datatype": dbt.type_string()}, - {"name": "shipping_address_zip", "datatype": dbt.type_string()}, - {"name": "test", "datatype": "boolean", "alias": "is_test_order"}, - {"name": "token", "datatype": dbt.type_string()}, - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_customer_columns() %} - -{% set columns = [ - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "accepts_marketing", "datatype": "boolean", "alias": "has_accepted_marketing"}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "default_address_id", "datatype": dbt.type_numeric()}, - {"name": "email", "datatype": dbt.type_string()}, - {"name": "first_name", "datatype": dbt.type_string()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "customer_id"}, - {"name": "last_name", "datatype": dbt.type_string()}, - {"name": "orders_count", "datatype": dbt.type_numeric()}, - {"name": "phone", "datatype": dbt.type_string()}, - {"name": "state", "datatype": dbt.type_string(), "alias": "account_state"}, - {"name": "tax_exempt", "datatype": "boolean", "alias": "is_tax_exempt"}, - {"name": "total_spent", "datatype": dbt.type_float()}, - {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, - {"name": "verified_email", "datatype": "boolean", "alias": "is_verified_email"} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_order_line_refund_columns() %} - -{% set columns = [ - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "order_line_refund_id"}, - {"name": "location_id", "datatype": dbt.type_numeric()}, - {"name": "order_line_id", "datatype": dbt.type_numeric()}, - {"name": "subtotal", "datatype": dbt.type_numeric()}, - {"name": "total_tax", "datatype": dbt.type_numeric()}, - {"name": "quantity", "datatype": dbt.type_float()}, - {"name": "refund_id", "datatype": dbt.type_numeric()}, - {"name": "restock_type", "datatype": dbt.type_string()} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_order_line_columns() %} - -{% set columns = [ - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "fulfillable_quantity", "datatype": dbt.type_numeric()}, - {"name": "fulfillment_service", "datatype": dbt.type_string()}, - {"name": "fulfillment_status", "datatype": dbt.type_string()}, - {"name": "gift_card", "datatype": "boolean", "alias": "is_gift_card"}, - {"name": "grams", "datatype": dbt.type_numeric()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "order_line_id"}, - {"name": "index", "datatype": dbt.type_numeric()}, - {"name": "name", "datatype": dbt.type_string()}, - {"name": "order_id", "datatype": dbt.type_numeric()}, - {"name": "pre_tax_price", "datatype": dbt.type_float()}, - {"name": "price", "datatype": dbt.type_float()}, - {"name": "product_id", "datatype": dbt.type_numeric()}, - {"name": "property_charge_interval_frequency", "datatype": dbt.type_numeric()}, - {"name": "property_for_shipping_jan_3_rd_2020", "datatype": dbt.type_string()}, - {"name": "property_shipping_interval_frequency", "datatype": dbt.type_numeric()}, - {"name": "property_shipping_interval_unit_type", "datatype": dbt.type_string()}, - {"name": "property_subscription_id", "datatype": dbt.type_numeric()}, - {"name": "quantity", "datatype": dbt.type_numeric()}, - {"name": "requires_shipping", "datatype": "boolean", "alias": "is_requiring_shipping"}, - {"name": "sku", "datatype": dbt.type_string()}, - {"name": "taxable", "datatype": "boolean", "alias": "is_taxable"}, - {"name": "title", "datatype": dbt.type_string()}, - {"name": "total_discount", "datatype": dbt.type_float()}, - {"name": "variant_id", "datatype": dbt.type_numeric()}, - {"name": "vendor", "datatype": dbt.type_string()} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_product_columns() %} - -{% set columns = [ - {"name": "_fivetran_deleted", "datatype": "boolean"}, - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "handle", "datatype": dbt.type_string()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "product_id"}, - {"name": "product_type", "datatype": dbt.type_string()}, - {"name": "published_at", "datatype": dbt.type_timestamp(), "alias": "published_timestamp"}, - {"name": "published_scope", "datatype": dbt.type_string()}, - {"name": "title", "datatype": dbt.type_string()}, - {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, - {"name": "vendor", "datatype": dbt.type_string()} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_product_variant_columns() %} - -{% set columns = [ - {"name": "id", "datatype": dbt.type_numeric(), "alias": "variant_id"}, - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}, - {"name": "product_id", "datatype": dbt.type_numeric()}, - {"name": "inventory_item_id", "datatype": dbt.type_numeric()}, - {"name": "image_id", "datatype": dbt.type_numeric()}, - {"name": "title", "datatype": dbt.type_string()}, - {"name": "price", "datatype": dbt.type_float()}, - {"name": "sku", "datatype": dbt.type_string()}, - {"name": "position", "datatype": dbt.type_numeric()}, - {"name": "inventory_policy", "datatype": dbt.type_string()}, - {"name": "compare_at_price", "datatype": dbt.type_float()}, - {"name": "fulfillment_service", "datatype": dbt.type_string()}, - {"name": "inventory_management", "datatype": dbt.type_string()}, - {"name": "taxable", "datatype": "boolean", "alias": "is_taxable"}, - {"name": "barcode", "datatype": dbt.type_string()}, - {"name": "grams", "datatype": dbt.type_float()}, - {"name": "inventory_quantity", "datatype": dbt.type_numeric()}, - {"name": "weight", "datatype": dbt.type_float()}, - {"name": "weight_unit", "datatype": dbt.type_string()}, - {"name": "option_1", "datatype": dbt.type_string()}, - {"name": "option_2", "datatype": dbt.type_string()}, - {"name": "option_3", "datatype": dbt.type_string()}, - {"name": "tax_code", "datatype": dbt.type_string()}, - {"name": "old_inventory_quantity", "datatype": dbt.type_numeric()}, - {"name": "requires_shipping", "datatype": "boolean", "alias": "is_requiring_shipping"} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_transaction_columns() %} - -{% set columns = [ - {"name": "id", "datatype": dbt.type_numeric(), "alias": "transaction_id"}, - {"name": "order_id", "datatype": dbt.type_numeric()}, - {"name": "refund_id", "datatype": dbt.type_numeric()}, - {"name": "amount", "datatype": dbt.type_numeric()}, - {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"}, - {"name": "processed_at", "datatype": dbt.type_timestamp(), "alias": "processed_timestamp"}, - {"name": "device_id", "datatype": dbt.type_numeric()}, - {"name": "gateway", "datatype": dbt.type_string()}, - {"name": "source_name", "datatype": dbt.type_string()}, - {"name": "message", "datatype": dbt.type_string()}, - {"name": "currency", "datatype": dbt.type_string()}, - {"name": "location_id", "datatype": dbt.type_numeric()}, - {"name": "parent_id", "datatype": dbt.type_numeric()}, - {"name": "payment_avs_result_code", "datatype": dbt.type_string()}, - {"name": "payment_credit_card_bin", "datatype": dbt.type_string()}, - {"name": "payment_cvv_result_code", "datatype": dbt.type_string()}, - {"name": "payment_credit_card_number", "datatype": dbt.type_string()}, - {"name": "payment_credit_card_company", "datatype": dbt.type_string()}, - {"name": "kind", "datatype": dbt.type_string()}, - {"name": "receipt", "datatype": dbt.type_string()}, - {"name": "currency_exchange_id", "datatype": dbt.type_numeric()}, - {"name": "currency_exchange_adjustment", "datatype": dbt.type_numeric()}, - {"name": "currency_exchange_original_amount", "datatype": dbt.type_numeric()}, - {"name": "currency_exchange_final_amount", "datatype": dbt.type_numeric()}, - {"name": "currency_exchange_currency", "datatype": dbt.type_string()}, - {"name": "error_code", "datatype": dbt.type_string()}, - {"name": "status", "datatype": dbt.type_string()}, - {"name": "test", "datatype": "boolean"}, - {"name": "user_id", "datatype": dbt.type_numeric()}, - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()} -] %} - -{% if target.type in ('redshift','postgres') %} - {{ columns.append({"name": "authorization", "datatype": dbt.type_string(), "quote": True, "alias": "authorization"}) }} -{% else %} - {"name": "authorization", "datatype": dbt.type_string()} -{% endif %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_refund_columns() %} - -{% set columns = [ - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, - {"name": "created_at", "datatype": dbt.type_timestamp()}, - {"name": "id", "datatype": dbt.type_numeric(), "alias": "refund_id"}, - {"name": "note", "datatype": dbt.type_string()}, - {"name": "order_id", "datatype": dbt.type_numeric()}, - {"name": "processed_at", "datatype": dbt.type_timestamp()}, - {"name": "restock", "datatype": "boolean"}, - {"name": "user_id", "datatype": dbt.type_numeric()} -] %} - -{{ return(columns) }} - -{% endmacro %} - -{% macro get_order_adjustment_columns() %} - -{% set columns = [ - {"name": "id", "datatype": dbt.type_numeric(), "alias": "order_adjustment_id"}, - {"name": "order_id", "datatype": dbt.type_numeric()}, - {"name": "refund_id", "datatype": dbt.type_numeric()}, - {"name": "amount", "datatype": dbt.type_float()}, - {"name": "tax_amount", "datatype": dbt.type_float()}, - {"name": "kind", "datatype": dbt.type_string()}, - {"name": "reason", "datatype": dbt.type_string()}, - {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()} -] %} - -{{ return(columns) }} - -{% endmacro %} diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index e8c066e..570c7e1 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -99,7 +99,7 @@ models: description: The ID of the product that the line item belongs to. Can be null if the original product associated with the order is deleted at a later date. - name: quantity description: The number of items that were purchased. - - name: is_requiring_shipping + - name: is_shipping_required description: Whether the item requires shipping. - name: sku description: The item's SKU (stock keeping unit). @@ -336,8 +336,6 @@ models: description: The price of the product variant. - name: product_id description: The unique numeric identifier for the product. - - name: is_requiring_shipping - description: This property is deprecated. Use the `requires_shipping` property on the InventoryItem resource instead. - name: sku description: A unique identifier for the product variant in the shop. Required in order to connect to a FulfillmentService. - name: is_taxable @@ -371,7 +369,7 @@ models: description: The ID associated with a refund in the refund table. - name: amount description: The amount of money included in the transaction. - - name: authorization + - name: authorization_code description: The authorization code associated with the transaction. - name: created_timestamp description: The date and time when the transaction was created. diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql index 32e943c..5c84f2c 100644 --- a/models/stg_shopify__customer.sql +++ b/models/stg_shopify__customer.sql @@ -1,10 +1,11 @@ -with source as ( +with base as ( - select * from {{ ref('stg_shopify__customer_tmp') }} + select * + from {{ ref('stg_shopify__customer_tmp') }} ), -renamed as ( +fields as ( select @@ -15,21 +16,46 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('customer_pass_through_columns') %} - , - {{ var('customer_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source + from base -) +), -select * from renamed +final as ( + + select + accepts_marketing as has_accepted_marketing, + email_marketing_consent_state, + created_at, + default_address_id, + email, + first_name, + id as customer_id, + last_name, + orders_count, + phone, + state as account_state, + tax_exempt as is_tax_exempt, + total_spent, + updated_at, + verified_email as is_verified_email, + note, + coalesce(accepts_marketing_updated_at, email_marketing_consent_consent_updated_at) as accepts_marketing_updated_at, + coalesce(marketing_opt_in_level, email_marketing_consent_opt_in_level) as marketing_opt_in_level, + lifetime_duration, + currency, + source_relation, + _fivetran_synced + + {{ fivetran_utils.fill_pass_through_columns('customer_pass_through_columns') }} + + from fields + +) +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__order.sql b/models/stg_shopify__order.sql index 17798dd..ab9194f 100644 --- a/models/stg_shopify__order.sql +++ b/models/stg_shopify__order.sql @@ -1,10 +1,11 @@ -with source as ( +with base as ( - select * from {{ ref('stg_shopify__order_tmp') }} + select * + from {{ ref('stg_shopify__order_tmp') }} ), -renamed as ( +fields as ( select @@ -15,20 +16,107 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('order_pass_through_columns') %} - , - {{ var('order_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source + from base + +), + +final as ( + + select + id as order_id, + processed_at, + updated_at, + user_id, + total_discounts, + total_discounts_set, + total_line_items_price, + total_line_items_price_set, + total_price, + total_price_set, + total_price_usd, + total_tax_set, + total_tax, + source_name, + subtotal_price, + taxes_included as has_taxes_included, + total_weight, + total_tip_received, + landing_site_base_url, + landing_site_ref, + location_id, + name, + note, + number, + order_number, + cancel_reason, + cancelled_at, + cart_token, + checkout_token, + closed_at, + created_at, + currency, + customer_id, + email, + financial_status, + fulfillment_status, + processing_method, + referring_site, + billing_address_address_1, + billing_address_address_2, + billing_address_city, + billing_address_company, + billing_address_country, + billing_address_country_code, + billing_address_first_name, + billing_address_last_name, + billing_address_latitude, + billing_address_longitude, + billing_address_name, + billing_address_phone, + billing_address_province, + billing_address_province_code, + billing_address_zip, + browser_ip, + buyer_accepts_marketing as has_buyer_accepted_marketing, + total_shipping_price_set, + shipping_address_address_1, + shipping_address_address_2, + shipping_address_city, + shipping_address_company, + shipping_address_country, + shipping_address_country_code, + shipping_address_first_name, + shipping_address_last_name, + shipping_address_latitude, + shipping_address_longitude, + shipping_address_name, + shipping_address_phone, + shipping_address_province, + shipping_address_province_code, + shipping_address_zip, + test as is_test_order, + token, + _fivetran_synced, + _fivetran_deleted, + app_id, + checkout_id, + client_details_user_agent, + customer_locale, + device_id, + order_status_url, + presentment_currency, + confirmed, + source_relation + + {{ fivetran_utils.fill_pass_through_columns('order_pass_through_columns') }} + from fields ) -select * from renamed \ No newline at end of file +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__order_adjustment.sql b/models/stg_shopify__order_adjustment.sql index 9570f35..a24d915 100644 --- a/models/stg_shopify__order_adjustment.sql +++ b/models/stg_shopify__order_adjustment.sql @@ -1,14 +1,14 @@ --To disable this model, set the shopify__using_order_adjustment variable within your dbt_project.yml file to False. {{ config(enabled=var('shopify__using_order_adjustment', True)) }} -with source as ( +with base as ( select * from {{ ref('stg_shopify__order_adjustment_tmp') }} ), -renamed as ( +fields as ( select {{ @@ -23,7 +23,26 @@ renamed as ( union_database_variable='shopify_union_databases') }} - from source + from base +), + +final as ( + + select + id as order_adjustment_id, + order_id, + refund_id, + amount, + amount_set, + tax_amount, + tax_amount_set, + kind, + reason, + source_relation, + _fivetran_synced + + from fields ) -select * from renamed +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__order_line.sql b/models/stg_shopify__order_line.sql index ab0297f..a52b6ef 100644 --- a/models/stg_shopify__order_line.sql +++ b/models/stg_shopify__order_line.sql @@ -1,10 +1,11 @@ -with source as ( +with base as ( - select * from {{ ref('stg_shopify__order_line_tmp') }} + select * + from {{ ref('stg_shopify__order_line_tmp') }} ), -renamed as ( +fields as ( select @@ -15,21 +16,59 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('order_line_pass_through_columns') %} - , - {{ var('order_line_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source + from base -) +), + +final as ( + + select + + fulfillable_quantity, + fulfillment_service, + fulfillment_status, + gift_card as is_gift_card, + grams, + id as order_line_id, + index, + name, + order_id, + pre_tax_price, + pre_tax_price_set, + price, + price_set, + product_id, + property_charge_interval_frequency, + property_for_shipping_jan_3_rd_2020, + property_shipping_interval_frequency, + property_shipping_interval_unit_type, + property_subscription_id, + quantity, + requires_shipping as is_shipping_required, + sku, + taxable as is_taxable, + tax_code, + title, + total_discount, + total_discount_set, + variant_id, + variant_title, + variant_inventory_management, + vendor, + properties, + source_relation, + _fivetran_synced + + {{ fivetran_utils.fill_pass_through_columns('order_line_pass_through_columns') }} -select * from renamed + from fields + +) +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__order_line_refund.sql b/models/stg_shopify__order_line_refund.sql index f1ad8bb..844f97e 100644 --- a/models/stg_shopify__order_line_refund.sql +++ b/models/stg_shopify__order_line_refund.sql @@ -1,13 +1,14 @@ --To disable this model, set the shopify__using_order_line_refund variable within your dbt_project.yml file to False. {{ config(enabled=var('shopify__using_order_line_refund', True)) }} -with source as ( +with base as ( - select * from {{ ref('stg_shopify__order_line_refund_tmp') }} + select * + from {{ ref('stg_shopify__order_line_refund_tmp') }} ), -renamed as ( +fields as ( select @@ -18,21 +19,35 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('order_line_refund_pass_through_columns') %} - , - {{ var('order_line_refund_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source + from base -) +), -select * from renamed +final as ( + + select + id as order_line_refund_id, + location_id, + order_line_id, + subtotal, + subtotal_set, + total_tax, + total_tax_set, + quantity, + refund_id, + restock_type, + _fivetran_synced, + source_relation + + {{ fivetran_utils.fill_pass_through_columns('order_line_refund_pass_through_columns') }} + + from fields +) +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__product.sql b/models/stg_shopify__product.sql index 6be960f..405acca 100644 --- a/models/stg_shopify__product.sql +++ b/models/stg_shopify__product.sql @@ -1,10 +1,11 @@ -with source as ( +with base as ( - select * from {{ ref('stg_shopify__product_tmp') }} + select * + from {{ ref('stg_shopify__product_tmp') }} ), -renamed as ( +fields as ( select @@ -15,21 +16,37 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('product_pass_through_columns') %} - , - {{ var('product_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source + from base -) +), -select * from renamed +final as ( + + select + created_at, + handle, + id as product_id, + product_type, + published_at, + published_scope, + title, + updated_at, + vendor, + status, + _fivetran_deleted as is_deleted, + _fivetran_synced, + source_relation + + {{ fivetran_utils.fill_pass_through_columns('product_pass_through_columns') }} + +from fields + +) +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__product_variant.sql b/models/stg_shopify__product_variant.sql index 13c0fb5..07f9c80 100644 --- a/models/stg_shopify__product_variant.sql +++ b/models/stg_shopify__product_variant.sql @@ -1,10 +1,11 @@ -with source as ( +with base as ( - select * from {{ ref('stg_shopify__product_variant_tmp') }} + select * + from {{ ref('stg_shopify__product_variant_tmp') }} ), -renamed as ( +fields as ( select @@ -15,20 +16,49 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('product_variant_pass_through_columns') %} - , - {{ var('product_variant_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source + from base + +), + +final as ( + + select + id as variant_id, + _fivetran_synced, + created_at, + updated_at, + product_id, + inventory_item_id, + image_id, + title, + price, + sku, + position, + inventory_policy, + compare_at_price, + fulfillment_service, + inventory_management, + taxable as is_taxable, + barcode, + grams, + inventory_quantity, + weight, + weight_unit, + option_1, + option_2, + option_3, + tax_code, + source_relation + + {{ fivetran_utils.fill_pass_through_columns('product_variant_pass_through_columns') }} + from fields ) -select * from renamed \ No newline at end of file +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__refund.sql b/models/stg_shopify__refund.sql index 4e7c6e1..ef550d6 100644 --- a/models/stg_shopify__refund.sql +++ b/models/stg_shopify__refund.sql @@ -1,14 +1,14 @@ --To disable this model, set the shopify__using_refund variable within your dbt_project.yml file to False. {{ config(enabled=var('shopify__using_refund', True)) }} -with source as ( +with base as ( select * from {{ ref('stg_shopify__refund_tmp') }} ), -renamed as ( +fields as ( select {{ @@ -23,7 +23,25 @@ renamed as ( union_database_variable='shopify_union_databases') }} - from source + from base +), + +final as ( + + select + created_at, + id as refund_id, + note, + order_id, + processed_at, + restock, + total_duties_set, + user_id, + _fivetran_synced, + source_relation + + from fields ) -select * from renamed +select * +from final \ No newline at end of file diff --git a/models/stg_shopify__transaction.sql b/models/stg_shopify__transaction.sql index 6dc51d2..91a94e0 100644 --- a/models/stg_shopify__transaction.sql +++ b/models/stg_shopify__transaction.sql @@ -1,10 +1,10 @@ -with source as ( +with base as ( select * from {{ ref('stg_shopify__transaction_tmp') }} ), -renamed as ( +fields as ( select @@ -15,21 +15,56 @@ renamed as ( ) }} - --The below script allows for pass through columns. - {% if var('transaction_pass_through_columns') %} - , - {{ var('transaction_pass_through_columns') | join (", ")}} - - {% endif %} - {{ fivetran_utils.source_relation( union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases') }} - from source - where not test + from base + +), + +final as ( + + select + id as transaction_id, + order_id, + refund_id, + amount, + created_at, + processed_at, + device_id, + gateway, + source_name, + message, + currency, + location_id, + parent_id, + payment_avs_result_code, + payment_credit_card_bin, + payment_cvv_result_code, + payment_credit_card_number, + payment_credit_card_company, + kind, + receipt, + currency_exchange_id, + currency_exchange_adjustment, + currency_exchange_original_amount, + currency_exchange_final_amount, + currency_exchange_currency, + error_code, + status, + user_id, + _fivetran_synced, + authorization_expires_at, + authorization_code, + source_relation + + {{ fivetran_utils.fill_pass_through_columns('transaction_pass_through_columns') }} + from fields + where not coalesce(test, false) ) -select * from renamed +select * +from final \ No newline at end of file diff --git a/models/tmp/stg_shopify__refund_tmp.sql b/models/tmp/stg_shopify__refund_tmp.sql index 80de149..d57c0ce 100644 --- a/models/tmp/stg_shopify__refund_tmp.sql +++ b/models/tmp/stg_shopify__refund_tmp.sql @@ -1,6 +1,15 @@ --To disable this model, set the shopify__using_refund variable within your dbt_project.yml file to False. {{ config(enabled=var('shopify__using_refund', True)) }} +{%- set source_relation = adapter.get_relation( + database=source('shopify', 'refund').database, + schema=source('shopify', 'refund').schema, + identifier=source('shopify', 'refund').name) -%} + +{% set table_exists=source_relation is not none %} + +{% if table_exists %} + {{ fivetran_utils.union_data( table_identifier='refund', @@ -12,4 +21,20 @@ union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) -}} \ No newline at end of file +}} + +{% else %} + +select + cast(null as {{ dbt.type_timestamp() }}) as _fivetran_synced, + cast(null as {{ dbt.type_timestamp() }}) as created_at, + cast(null as {{ dbt.type_numeric() }}) as id, + cast(null as {{ dbt.type_string() }}) as note, + cast(null as {{ dbt.type_numeric() }}) as order_id, + cast(null as {{ dbt.type_timestamp() }}) as processed_at, + cast(null as boolean) as restock, + cast(null as {{ dbt.type_string() }}) as total_duties_set, + cast(null as {{ dbt.type_numeric() }}) as user_id, + cast(null as {{ dbt.type_string() }}) as _dbt_source_relation + +{% endif %} \ No newline at end of file diff --git a/packages.yml b/packages.yml index 908f471..fd66aa3 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,8 @@ packages: -- package: fivetran/fivetran_utils - version: [">=0.4.0", "<0.5.0"] +# - package: fivetran/fivetran_utils +# version: [">=0.4.0", "<0.5.0"] +- git: https://github.com/fivetran/dbt_fivetran_utils.git + revision: releases/v0.4.latest + warn-unpinned: false - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"]