Merge pull request #70 from fivetran/feature/general-ledger-unioning

Feature/General ledger unioning
fivetran · Jan 19, 2023 · 19d2384 · 19d2384
2 parents 8bd4697 + e785032
commit 19d2384
Show file tree

Hide file tree

Showing 9 changed files with 66 additions and 93 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@
 - Added `source_relation` field to all Quickbooks models to allow customers, if they have multiple Quickbooks connectors, to union them inside the package. [#62](https://github.com/fivetran/dbt_quickbooks/pull/62).
 - Added tests to all final models, particularly to test uniqueness across a combination of columns, including `source_relation`. [#62](https://github.com/fivetran/dbt_quickbooks/pull/62)
 - Modified `int_quickbooks__retained_earnings` intermediate model to accurately reflect `account_name` field, from "Net Income / Retained Earnings Adjustment" to "Net Income Adjustment". [#66](https://github.com/fivetran/dbt_quickbooks/pull/66)
+- Create `get_enabled_unioned_models` macro to make `quickbooks__general_ledger` more readable and provide more model flexibility. Materialized `double_entry_transactions` models in own schema as a table to ensure macro execution, created `quickbooks_intermediate` schema for customers to explore these models further. [#70](https://github.com/fivetran/dbt_quickbooks/pull/70)
 # dbt_quickbooks_source v0.6.0
 ## 🚨 Breaking Changes 🚨:
 [PR #51](https://github.com/fivetran/dbt_quickbooks/pull/51) includes the following breaking changes:

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,15 +1,16 @@
 config-version: 2
 name: 'quickbooks'
 
-version: '0.6.0'
+version: '0.7.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:
   quickbooks:
     +materialized: table
     +schema: quickbooks
     double_entry_transactions:
-      +materialized: ephemeral
+      +schema: quickbooks_intermediate
+      +materialized: table
     transaction_lines:
       +materialized: ephemeral
     intermediate:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/macros/get_enabled_unioned_models.sql b/macros/get_enabled_unioned_models.sql
@@ -0,0 +1,33 @@
+{% macro get_enabled_unioned_models(unioned_models = [ 
+    'bill',
+    'credit_memo',
+    'deposit',
+    'invoice',
+    'journal_entry',
+    'payment',
+    'refund_receipt',
+    'sales_receipt',
+    'transfer',
+    'vendor_credit']) %} 
+
+{% set enabled_unioned_models = [] %}
+
+{{ enabled_unioned_models.append(ref('int_quickbooks__purchase_double_entry')) }}
+
+{% for unioned_model in unioned_models %}  
+    {% if var('using_' ~ unioned_model, True) %}
+        {{ enabled_unioned_models.append(ref('int_quickbooks__' ~ unioned_model ~ '_double_entry')) }}
+    {% endif %}
+ {% endfor %}   
+
+{% if var('using_bill', True) %}
+    {{ enabled_unioned_models.append(ref('int_quickbooks__bill_payment_double_entry')) }}
+{% endif %}
+
+{% if var('using_credit_card_payment_txn', True) %}
+    {{ enabled_unioned_models.append(ref('int_quickbooks__credit_card_pymt_double_entry')) }}
+{% endif %}
+
+{{ return(enabled_unioned_models) }}
+
+{% endmacro %}
diff --git a/macros/quickbooks_macros.yml b/macros/quickbooks_macros.yml
@@ -0,0 +1,13 @@
+version: 2
+
+macros:
+  - name: get_enabled_unioned_models
+    description: >
+      This macro takes models from the `double_entry_transactions` folder and appends them together in an array of relations if these models are enabled. This will then be unioned together using `dbt_utils.union_relations` in the `quickbooks__general_ledger` model. `int_quickbooks__purchase_double_entry` is included by default. `int_quickbooks__bill_payment_double_entry` and `int_quickbooks__credit_card_pymt_double_entry` are appended separately because their `using_` variables do not match up like the other variables, which can't be modified.
+    arguments:
+      - name: unioned_models
+        type: list
+        description: List of strings referring to particular models to then be appended and eventually unioned if enabled.
+      - name: enabled_unioned_models
+        type: list
+        description: List of relations of models that are enabled to eventually be unioned into a general ledger model.
diff --git a/models/quickbooks.yml b/models/quickbooks.yml
@@ -295,8 +295,10 @@ models:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - unique_id
-            - source_relation 
+            - source_relation   
     columns:
+      - name: unique_id
+        description: Unique identifier of the general ledger line, dependent on `transaction_id`, `transaction_index`, `account_name, `transaction_type`, `transaction_source`. 
       - name: transaction_id
         description: Unique identifier of transaction. Each transaction will have an offsetting debit and credit record.
       - name: transaction_index

diff --git a/models/quickbooks__general_ledger.sql b/models/quickbooks__general_ledger.sql
@@ -1,7 +1,11 @@
-with gl_union as (
+with unioned_models as (
 
-    select
-        transaction_id,
+    {{ dbt_utils.union_relations(get_enabled_unioned_models()) }}
+),
+
+gl_union as (
+
+    select transaction_id,
         source_relation,
         index,
         transaction_date,
@@ -11,90 +15,8 @@ with gl_union as (
         account_id,
         class_id,
         transaction_type,
-        transaction_source
-    from {{ ref('int_quickbooks__purchase_double_entry') }}
-
-    {% if var('using_sales_receipt', True) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__sales_receipt_double_entry') }}
-    {% endif %}
-
-    {% if var('using_bill', True) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__bill_payment_double_entry') }}
-
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__bill_double_entry') }}
-    {% endif %}
-
-    {% if var('using_credit_memo', True) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__credit_memo_double_entry') }}
-    {% endif %}
-
-    {% if var('using_credit_card_payment_txn', False) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__credit_card_pymt_double_entry') }}
-    {% endif %}
-
-    {% if var('using_deposit', True) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__deposit_double_entry') }}
-    {% endif %}
-
-    {% if var('using_invoice', True) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__invoice_double_entry') }}
-    {% endif %}
-
-    {% if var('using_transfer', True) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__transfer_double_entry') }}
-    {% endif %}
-
-    {% if var('using_journal_entry', True) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__journal_entry_double_entry') }}
-    {% endif %}
-
-    {% if var('using_payment', True) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__payment_double_entry') }}
-    {% endif %}
-
-    {% if var('using_refund_receipt', True) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__refund_receipt_double_entry') }}
-    {% endif %}
-
-    {% if var('using_vendor_credit', True) %}
-    union all
-
-    select *
-    from {{ ref('int_quickbooks__vendor_credit_double_entry') }}
-    {% endif %}
+        transaction_source 
+    from unioned_models
 ),
 
 accounts as (
@@ -105,6 +27,7 @@ accounts as (
 
 
 adjusted_gl as (
+
     select
         {{ dbt_utils.generate_surrogate_key(['gl_union.transaction_id', 'gl_union.index', 'accounts.name', ' gl_union.transaction_type', 'gl_union.transaction_source']) }} as unique_id,
         gl_union.transaction_id,