feature/metafields

.buildkite/scripts/run_models.sh

@@ -19,5 +19,5 @@ dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{shopify_timezone: "America/New_York"}' --target "$db" --full-refresh
+dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_all_metafields: true}' --target "$db" --full-refresh
 dbt test --target "$db"

.gitignore

@@ -2,5 +2,5 @@
 target/
 dbt_modules/
 logs/
-
+env/
 dbt_packages/

CHANGELOG.md

@@ -1,4 +1,4 @@
-# dbt_shopify v0.7.0
+# dbt_shopify v0.8.0
 ## 🎉 Documentation and Feature Updates
 - Updated README documentation updates for easier navigation and setup of the dbt package
 - Included `shopify_[source_table_name]_identifier` variable within the Shopify source package for additional flexibility within the package when source tables are named differently.
@@ -8,7 +8,12 @@
   - Intermediate models that roll customer_ids up to emails:
     - `shopify__customer_email_rollup`
     - `shopify__emails__order_aggregates`
+- Metafield support! This package now supports metafields for the collection, customer, order, product_image, product, product_variant, and shop objects. If enabled (see the [README](https://github.com/fivetran/dbt_shopify#adding-metafields) for more details), respective `shopify__[object]_metafields` models will materialize with **all** metafields defined within the `metafield` source table appended to the object. ([#50](https://github.com/fivetran/dbt_shopify/pull/50))
 
+## Under the Hood
+- Addition of the calogica/dbt_expectations package for more robust testing.
+
+## dbt_shopify v0.7.0
 ## 🚨 Breaking Changes 🚨:
 [PR #40](https://github.com/fivetran/dbt_shopify/pull/40) includes the following breaking changes:
 - Dispatch update for dbt-utils to dbt-core cross-db macros migration. Specifically `{{ dbt_utils.<macro> }}` have been updated to `{{ dbt.<macro> }}` for the below macros:

README.md

@@ -87,6 +87,21 @@ vars:
     product_variant_pass_through_columns: []
 ```
 
+### Adding Metafields
+In [May 2021](https://fivetran.com/docs/applications/shopify/changelog#may2021) the Shopify connector included support for the [metafield resource](https://shopify.dev/api/admin-rest/2023-01/resources/metafield). If you would like to take advantage of these metafields, this package offers corresponding mapping models which append these metafields to the respective source object for the following tables: collection, customer, order, product_image, product, product_variant, shop. If enabled, these models will materialize as `shopify__[object]_metafields` for each respective supported object. To enable these metafield mapping models, you may use the following configurations within your `dbt_project.yml`.
+>**Note**: These metafield models will contain all the same records as the corresponding staging models with the exception of the metafield columns being added. To ensure there is no fanout, this package takes advantage of the `dbt_expectations.expect_table_row_count_to_equal_other_table` test to ensure the metafield models contain the same row count as the staging model.
+
+```yml
+vars:
+  shopify_using_all_metafields: True ## False by default. Will enable ALL metafield models. FYI - This will override all other metafield variables.
+  shopify_using_collection_metafields: True ## False by default. Will enable ONLY the collection metafield model.
+  shopify_using_customer_metafields: True ## False by default. Will enable ONLY the customer metafield model.
+  shopify_using_order_metafields: True ## False by default. Will enable ONLY the order metafield model.
+  shopify_using_product_metafields: True ## False by default. Will enable ONLY the product metafield model.
+  shopify_using_product_image_metafields: True ## False by default. Will enable ONLY the product image metafield model.
+  shopify_using_product_variant_metafields: True ## False by default. Will enable ONLY the product variant metafield model.
+```
+
 ### Changing the Build Schema
 By default this package will build the Shopify staging models within a schema titled (<target_schema> + `_stg_shopify`) and the Shopify final models within a schema titled (<target_schema> + `_shopify`) in your target database. If this is not where you would like your modeled Shopify data to be written to, add the following configuration to your `dbt_project.yml` file:
 

integration_tests/dbt_project.yml

@@ -39,7 +39,7 @@ vars:
     shopify_abandoned_checkout_discount_code_identifier: "shopify_abandoned_checkout_discount_code_data"
     shopify_order_discount_code_identifier: "shopify_order_discount_code_data"
     shopify_abandoned_checkout_shipping_line_identifier: "shopify_abandoned_checkout_shipping_line_data"
-    
+
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']

macros/get_metafields.sql

@@ -0,0 +1,52 @@
+{% macro get_metafields(source_object, reference_value, lookup_object="stg_shopify__metafield", key_field="metafield_reference", key_value="value", reference_field="owner_resource") %}
+
+{% set pivot_fields = dbt_utils.get_column_values(table=ref(lookup_object), column=key_field, where="lower(" ~ reference_field ~ ") = lower('" ~ reference_value ~ "')") %}
+
+{% set source_columns = adapter.get_columns_in_relation(ref(source_object)) %}
+{% set source_column_count = source_columns | length %}
+
+with source_table as (
+    select *
+    from {{ ref(source_object) }}
+)
+
+{% if pivot_fields is not none %},
+lookup_object as (
+    select 
+        *,
+        {{ dbt_utils.pivot(
+                column=key_field, 
+                values=pivot_fields, 
+                agg='', 
+                then_value=key_value, 
+                else_value="null",
+                quote_identifiers=false
+                ) 
+        }}
+    from {{ ref(lookup_object) }}
+    where is_most_recent_record
+),
+
+final as (
+    select
+        {% for column in source_columns %}
+            source_table.{{ column.name }}{% if not loop.last %},{% endif %}
+        {% endfor %}
+        {% for fields in pivot_fields %}
+            , max(lookup_object.{{ dbt_utils.slugify(fields) }}) as metafield_{{ dbt_utils.slugify(fields) }}
+        {% endfor %}
+    from source_table
+    left join lookup_object 
+        on lookup_object.{{ reference_field }}_id = source_table.{{ reference_value }}_id
+        and lookup_object.{{ reference_field }} = '{{ reference_value }}'
+    {{ dbt_utils.group_by(source_column_count) }}
+)
+
+select *
+from final
+{% else %}
+
+select *
+from source_table
+{% endif %}
+{% endmacro %}

models/metafields/shopify__collection_metafields.sql

@@ -0,0 +1,6 @@
+{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_collection_metafields', False)) }}
+
+{{ get_metafields( 
+    source_object = "stg_shopify__collection", 
+    reference_value = 'collection') 
+}}

models/metafields/shopify__customer_metafields.sql

@@ -0,0 +1,6 @@
+{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_customer_metafields', False)) }}
+
+{{ get_metafields( 
+    source_object = "stg_shopify__customer", 
+    reference_value = 'customer') 
+}}

models/metafields/shopify__order_metafields.sql

@@ -0,0 +1,6 @@
+{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_order_metafields', False)) }}
+
+{{ get_metafields( 
+    source_object = "stg_shopify__order", 
+    reference_value = 'order') 
+}}

models/metafields/shopify__product_image_metafields.sql

@@ -0,0 +1,6 @@
+{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_product_image_metafields', False)) }}
+
+{{ get_metafields( 
+    source_object = "stg_shopify__product_image", 
+    reference_value = 'image') 
+}}

models/metafields/shopify__product_metafields.sql

@@ -0,0 +1,6 @@
+{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_product_metafields', False)) }}
+
+{{ get_metafields( 
+    source_object = "stg_shopify__product", 
+    reference_value = 'product') 
+}}

models/metafields/shopify__product_variant_metafields.sql

@@ -0,0 +1,6 @@
+{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_product_variant_metafields', False)) }}
+
+{{ get_metafields( 
+    source_object = "stg_shopify__product_variant", 
+    reference_value = 'variant') 
+}}

models/metafields/shopify__shop_metafields.sql

@@ -0,0 +1,6 @@
+{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_shop_metafields', False)) }}
+
+{{ get_metafields( 
+    source_object = "stg_shopify__shop", 
+    reference_value = 'shop') 
+}}

models/metafields/shopify_metafields.yml

@@ -0,0 +1,44 @@
+version: 2
+
+models:
+  - name: shopify__collection_metafields
+    description: Replica of the stg_shopify__collection model with the addition of metafields pivoted out from the stg_shopify__metafield model.
+    tests:
+      - dbt_expectations.expect_table_row_count_to_equal_other_table:
+          compare_model: ref("stg_shopify__collection")
+
+  - name: shopify__customer_metafields
+    description: Replica of the stg_shopify__customer model with the addition of metafields pivoted out from the stg_shopify__metafield model.
+    tests:
+      - dbt_expectations.expect_table_row_count_to_equal_other_table:
+          compare_model: ref("stg_shopify__customer")
+
+  - name: shopify__order_metafields
+    description: Replica of the stg_shopify__order model with the addition of metafields pivoted out from the stg_shopify__metafield model.
+    tests:
+      - dbt_expectations.expect_table_row_count_to_equal_other_table:
+          compare_model: ref("stg_shopify__order")
+
+  - name: shopify__product_image_metafields
+    description: Replica of the stg_shopify__product_image model with the addition of metafields pivoted out from the stg_shopify__metafield model.
+    tests:
+      - dbt_expectations.expect_table_row_count_to_equal_other_table:
+          compare_model: ref("stg_shopify__product_image")
+
+  - name: shopify__product_metafields
+    description: Replica of the stg_shopify__product model with the addition of metafields pivoted out from the stg_shopify__metafield model.
+    tests:
+      - dbt_expectations.expect_table_row_count_to_equal_other_table:
+          compare_model: ref("stg_shopify__product")
+
+  - name: shopify__product_variant_metafields
+    description: Replica of the stg_shopify__product_variant model with the addition of metafields pivoted out from the stg_shopify__metafield model.
+    tests:
+      - dbt_expectations.expect_table_row_count_to_equal_other_table:
+          compare_model: ref("stg_shopify__product_variant")
+
+  - name: shopify__shop_metafields
+    description: Replica of the stg_shopify__shop model with the addition of metafields pivoted out from the stg_shopify__metafield model.
+    tests:
+      - dbt_expectations.expect_table_row_count_to_equal_other_table:
+          compare_model: ref("stg_shopify__shop")

packages.yml

@@ -3,7 +3,5 @@ packages:
 #   version: [">=0.7.0", "<0.8.0"]
 
   - git: https://github.com/fivetran/dbt_shopify_source.git
-    revision: feature/revamp/transform-enhancements
-    warn-unpinned: false
-
-# - local: ../Shopify/dbt_shopify_source
+    revision: feature/metafields
+    warn-unpinned: false