diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 3e08693..e6edf97 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -19,5 +19,5 @@ dbt deps dbt seed --target "$db" --full-refresh dbt run --target "$db" --full-refresh dbt test --target "$db" -dbt run --vars '{shopify_timezone: "America/New_York"}' --target "$db" --full-refresh +dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_all_metafields: true}' --target "$db" --full-refresh dbt test --target "$db" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2710741..55d3399 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ target/ dbt_modules/ logs/ - +env/ dbt_packages/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 63d7aab..c90fa23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# dbt_shopify v0.7.0 +# dbt_shopify v0.8.0 ## 🎉 Documentation and Feature Updates - Updated README documentation updates for easier navigation and setup of the dbt package - Included `shopify_[source_table_name]_identifier` variable within the Shopify source package for additional flexibility within the package when source tables are named differently. @@ -8,7 +8,12 @@ - Intermediate models that roll customer_ids up to emails: - `shopify__customer_email_rollup` - `shopify__emails__order_aggregates` +- Metafield support! This package now supports metafields for the collection, customer, order, product_image, product, product_variant, and shop objects. If enabled (see the [README](https://github.com/fivetran/dbt_shopify#adding-metafields) for more details), respective `shopify__[object]_metafields` models will materialize with **all** metafields defined within the `metafield` source table appended to the object. ([#50](https://github.com/fivetran/dbt_shopify/pull/50)) +## Under the Hood +- Addition of the calogica/dbt_expectations package for more robust testing. + +## dbt_shopify v0.7.0 ## 🚨 Breaking Changes 🚨: [PR #40](https://github.com/fivetran/dbt_shopify/pull/40) includes the following breaking changes: - Dispatch update for dbt-utils to dbt-core cross-db macros migration. Specifically `{{ dbt_utils. }}` have been updated to `{{ dbt. }}` for the below macros: diff --git a/README.md b/README.md index 7274fce..1e5ae66 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,21 @@ vars: product_variant_pass_through_columns: [] ``` +### Adding Metafields +In [May 2021](https://fivetran.com/docs/applications/shopify/changelog#may2021) the Shopify connector included support for the [metafield resource](https://shopify.dev/api/admin-rest/2023-01/resources/metafield). If you would like to take advantage of these metafields, this package offers corresponding mapping models which append these metafields to the respective source object for the following tables: collection, customer, order, product_image, product, product_variant, shop. If enabled, these models will materialize as `shopify__[object]_metafields` for each respective supported object. To enable these metafield mapping models, you may use the following configurations within your `dbt_project.yml`. +>**Note**: These metafield models will contain all the same records as the corresponding staging models with the exception of the metafield columns being added. To ensure there is no fanout, this package takes advantage of the `dbt_expectations.expect_table_row_count_to_equal_other_table` test to ensure the metafield models contain the same row count as the staging model. + +```yml +vars: + shopify_using_all_metafields: True ## False by default. Will enable ALL metafield models. FYI - This will override all other metafield variables. + shopify_using_collection_metafields: True ## False by default. Will enable ONLY the collection metafield model. + shopify_using_customer_metafields: True ## False by default. Will enable ONLY the customer metafield model. + shopify_using_order_metafields: True ## False by default. Will enable ONLY the order metafield model. + shopify_using_product_metafields: True ## False by default. Will enable ONLY the product metafield model. + shopify_using_product_image_metafields: True ## False by default. Will enable ONLY the product image metafield model. + shopify_using_product_variant_metafields: True ## False by default. Will enable ONLY the product variant metafield model. +``` + ### Changing the Build Schema By default this package will build the Shopify staging models within a schema titled ( + `_stg_shopify`) and the Shopify final models within a schema titled ( + `_shopify`) in your target database. If this is not where you would like your modeled Shopify data to be written to, add the following configuration to your `dbt_project.yml` file: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 0e70807..652bd02 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -39,7 +39,7 @@ vars: shopify_abandoned_checkout_discount_code_identifier: "shopify_abandoned_checkout_discount_code_data" shopify_order_discount_code_identifier: "shopify_order_discount_code_data" shopify_abandoned_checkout_shipping_line_identifier: "shopify_abandoned_checkout_shipping_line_data" - + dispatch: - macro_namespace: dbt_utils search_order: ['spark_utils', 'dbt_utils'] diff --git a/macros/get_metafields.sql b/macros/get_metafields.sql new file mode 100644 index 0000000..c00938c --- /dev/null +++ b/macros/get_metafields.sql @@ -0,0 +1,52 @@ +{% macro get_metafields(source_object, reference_value, lookup_object="stg_shopify__metafield", key_field="metafield_reference", key_value="value", reference_field="owner_resource") %} + +{% set pivot_fields = dbt_utils.get_column_values(table=ref(lookup_object), column=key_field, where="lower(" ~ reference_field ~ ") = lower('" ~ reference_value ~ "')") %} + +{% set source_columns = adapter.get_columns_in_relation(ref(source_object)) %} +{% set source_column_count = source_columns | length %} + +with source_table as ( + select * + from {{ ref(source_object) }} +) + +{% if pivot_fields is not none %}, +lookup_object as ( + select + *, + {{ dbt_utils.pivot( + column=key_field, + values=pivot_fields, + agg='', + then_value=key_value, + else_value="null", + quote_identifiers=false + ) + }} + from {{ ref(lookup_object) }} + where is_most_recent_record +), + +final as ( + select + {% for column in source_columns %} + source_table.{{ column.name }}{% if not loop.last %},{% endif %} + {% endfor %} + {% for fields in pivot_fields %} + , max(lookup_object.{{ dbt_utils.slugify(fields) }}) as metafield_{{ dbt_utils.slugify(fields) }} + {% endfor %} + from source_table + left join lookup_object + on lookup_object.{{ reference_field }}_id = source_table.{{ reference_value }}_id + and lookup_object.{{ reference_field }} = '{{ reference_value }}' + {{ dbt_utils.group_by(source_column_count) }} +) + +select * +from final +{% else %} + +select * +from source_table +{% endif %} +{% endmacro %} \ No newline at end of file diff --git a/models/metafields/shopify__collection_metafields.sql b/models/metafields/shopify__collection_metafields.sql new file mode 100644 index 0000000..d8130b9 --- /dev/null +++ b/models/metafields/shopify__collection_metafields.sql @@ -0,0 +1,6 @@ +{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_collection_metafields', False)) }} + +{{ get_metafields( + source_object = "stg_shopify__collection", + reference_value = 'collection') +}} \ No newline at end of file diff --git a/models/metafields/shopify__customer_metafields.sql b/models/metafields/shopify__customer_metafields.sql new file mode 100644 index 0000000..e88317a --- /dev/null +++ b/models/metafields/shopify__customer_metafields.sql @@ -0,0 +1,6 @@ +{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_customer_metafields', False)) }} + +{{ get_metafields( + source_object = "stg_shopify__customer", + reference_value = 'customer') +}} \ No newline at end of file diff --git a/models/metafields/shopify__order_metafields.sql b/models/metafields/shopify__order_metafields.sql new file mode 100644 index 0000000..35482e1 --- /dev/null +++ b/models/metafields/shopify__order_metafields.sql @@ -0,0 +1,6 @@ +{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_order_metafields', False)) }} + +{{ get_metafields( + source_object = "stg_shopify__order", + reference_value = 'order') +}} \ No newline at end of file diff --git a/models/metafields/shopify__product_image_metafields.sql b/models/metafields/shopify__product_image_metafields.sql new file mode 100644 index 0000000..bc2e3b7 --- /dev/null +++ b/models/metafields/shopify__product_image_metafields.sql @@ -0,0 +1,6 @@ +{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_product_image_metafields', False)) }} + +{{ get_metafields( + source_object = "stg_shopify__product_image", + reference_value = 'image') +}} \ No newline at end of file diff --git a/models/metafields/shopify__product_metafields.sql b/models/metafields/shopify__product_metafields.sql new file mode 100644 index 0000000..fb8f932 --- /dev/null +++ b/models/metafields/shopify__product_metafields.sql @@ -0,0 +1,6 @@ +{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_product_metafields', False)) }} + +{{ get_metafields( + source_object = "stg_shopify__product", + reference_value = 'product') +}} \ No newline at end of file diff --git a/models/metafields/shopify__product_variant_metafields.sql b/models/metafields/shopify__product_variant_metafields.sql new file mode 100644 index 0000000..44b5377 --- /dev/null +++ b/models/metafields/shopify__product_variant_metafields.sql @@ -0,0 +1,6 @@ +{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_product_variant_metafields', False)) }} + +{{ get_metafields( + source_object = "stg_shopify__product_variant", + reference_value = 'variant') +}} \ No newline at end of file diff --git a/models/metafields/shopify__shop_metafields.sql b/models/metafields/shopify__shop_metafields.sql new file mode 100644 index 0000000..6308b77 --- /dev/null +++ b/models/metafields/shopify__shop_metafields.sql @@ -0,0 +1,6 @@ +{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_shop_metafields', False)) }} + +{{ get_metafields( + source_object = "stg_shopify__shop", + reference_value = 'shop') +}} \ No newline at end of file diff --git a/models/metafields/shopify_metafields.yml b/models/metafields/shopify_metafields.yml new file mode 100644 index 0000000..62ff6d4 --- /dev/null +++ b/models/metafields/shopify_metafields.yml @@ -0,0 +1,44 @@ +version: 2 + +models: + - name: shopify__collection_metafields + description: Replica of the stg_shopify__collection model with the addition of metafields pivoted out from the stg_shopify__metafield model. + tests: + - dbt_expectations.expect_table_row_count_to_equal_other_table: + compare_model: ref("stg_shopify__collection") + + - name: shopify__customer_metafields + description: Replica of the stg_shopify__customer model with the addition of metafields pivoted out from the stg_shopify__metafield model. + tests: + - dbt_expectations.expect_table_row_count_to_equal_other_table: + compare_model: ref("stg_shopify__customer") + + - name: shopify__order_metafields + description: Replica of the stg_shopify__order model with the addition of metafields pivoted out from the stg_shopify__metafield model. + tests: + - dbt_expectations.expect_table_row_count_to_equal_other_table: + compare_model: ref("stg_shopify__order") + + - name: shopify__product_image_metafields + description: Replica of the stg_shopify__product_image model with the addition of metafields pivoted out from the stg_shopify__metafield model. + tests: + - dbt_expectations.expect_table_row_count_to_equal_other_table: + compare_model: ref("stg_shopify__product_image") + + - name: shopify__product_metafields + description: Replica of the stg_shopify__product model with the addition of metafields pivoted out from the stg_shopify__metafield model. + tests: + - dbt_expectations.expect_table_row_count_to_equal_other_table: + compare_model: ref("stg_shopify__product") + + - name: shopify__product_variant_metafields + description: Replica of the stg_shopify__product_variant model with the addition of metafields pivoted out from the stg_shopify__metafield model. + tests: + - dbt_expectations.expect_table_row_count_to_equal_other_table: + compare_model: ref("stg_shopify__product_variant") + + - name: shopify__shop_metafields + description: Replica of the stg_shopify__shop model with the addition of metafields pivoted out from the stg_shopify__metafield model. + tests: + - dbt_expectations.expect_table_row_count_to_equal_other_table: + compare_model: ref("stg_shopify__shop") \ No newline at end of file diff --git a/packages.yml b/packages.yml index 7f1a843..33e6487 100644 --- a/packages.yml +++ b/packages.yml @@ -3,7 +3,5 @@ packages: # version: [">=0.7.0", "<0.8.0"] - git: https://github.com/fivetran/dbt_shopify_source.git - revision: feature/revamp/transform-enhancements + revision: feature/package-revamp warn-unpinned: false - -# - local: ../Shopify/dbt_shopify_source