Merge pull request #28 from fivetran/releases/v0.6.latest

Releases/v0.6.latest

.buildkite/pipeline.yml

@@ -56,4 +56,18 @@ steps:
             - "CI_REDSHIFT_DBT_PASS"
             - "CI_REDSHIFT_DBT_USER"
     commands: |
-      bash .buildkite/scripts/run_models.sh redshift
+      bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":databricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks

CHANGELOG.md

@@ -1,3 +1,11 @@
+# dbt_netsuite_source v0.6.1
+
+## Bug Fixes
+- In [v0.6.0](https://github.com/fivetran/dbt_netsuite_source/blob/main/CHANGELOG.md#dbt_netsuite_source-v060), we introduced pass through columns for the `vendors` and `items` tables. However, this update was only applied to the original `Netsuite` version of this package, and not `Netsuite2` as well. These pass through columns are accessible in Netsuite2 data models ([PR #27](https://github.com/fivetran/dbt_netsuite_source/pull/27/)).
+
+## 🎉 Feature Updates 🎉
+Now introducing...Databricks compatibility 🧱 ([PR #26](https://github.com/fivetran/dbt_netsuite_source/pull/26))
+
 # dbt_netsuite_source v0.6.0
 
 ## 🚨 Breaking Changes 🚨:

README.md

@@ -68,7 +68,16 @@ To use this dbt package, you must have At least either one Fivetran **Netsuite**
 - vendorcategory
 
 ### Database Compatibility
-This package is compatible with either a **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination.
+This package is compatible with either a **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination.
+
+### Databricks dispatch configuration
+If you are using a Databricks destination with this package, you must add the following (or a variation of the following) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively.
+```yml
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']
+```
+
 ## Step 2: Install the package
 Include the following netsuite_source package version in your `packages.yml` file.
 > TIP: Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
@@ -189,6 +198,9 @@ packages:
 
     - package: dbt-labs/dbt_utils
       version: [">=1.0.0", "<2.0.0"]
+
+    - package: dbt-labs/spark_utils
+      version: [">=0.3.0", "<0.4.0"]
 ```
 
 # 🙌 How is this package maintained and can I contribute?

dbt_project.yml

@@ -1,8 +1,7 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-
-version: '0.6.0'
+version: '0.6.1'
 
 models:
   netsuite_source:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: netsuite_source_integration_tests
+      schema: netsuite_source_integration_tests_1
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: netsuite_source_integration_tests
+      schema: netsuite_source_integration_tests_1
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: netsuite_source_integration_tests
+      schema: netsuite_source_integration_tests_1
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: netsuite_source_integration_tests
+      schema: netsuite_source_integration_tests_1
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: netsuite_source_integration_tests
+      schema: netsuite_source_integration_tests_1
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,12 +1,12 @@
 name: 'netsuite_source_integration_tests'
 
-version: '0.6.0'
+version: '0.6.1'
 
 profile: 'integration_tests'
 config-version: 2
 vars:
   netsuite_source:
-    netsuite_schema: netsuite_source_integration_tests
+    netsuite_schema: netsuite_source_integration_tests_1
     netsuite_data_model_override: netsuite
     netsuite_accounting_books_identifier: "netsuite_accounting_books_data"
     netsuite_accounting_periods_identifier: "netsuite_accounting_periods_data"
@@ -54,3 +54,7 @@ seeds:
     +column_types:
       ending: timestamp
       due_date: timestamp
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']

models/netsuite/src_netsuite.yml

@@ -4,7 +4,7 @@ version: 2
 sources:
   - name: netsuite
     schema: "{{ var('netsuite_schema', 'netsuite') }}" # you will need to customize if your schema name is not netsuite
-    database: "{{ var('netsuite_database', target.database) }}"
+    database: "{% if target.type != 'spark'%}{{ var('netsuite_database', target.database) }}{% endif %}"
     loader: fivetran
     loaded_at_field: _fivetran_synced
     config:

models/netsuite2/src_netsuite2.yml

@@ -3,7 +3,7 @@ version: 2
 sources:
   - name: netsuite2
     schema: "{{ var('netsuite_schema', 'netsuite') }}"
-    database: "{{ var('netsuite_database', target.database) }}"
+    database: "{% if target.type != 'spark'%}{{ var('netsuite_database', target.database) }}{% endif %}"
     loader: fivetran
     loaded_at_field: _fivetran_synced
     config:

models/netsuite2/stg_netsuite2__items.sql

@@ -40,6 +40,10 @@ final as (
         deferralaccount as deferred_expense_account_id,
         deferredrevenueaccount as deferred_revenue_account_id,
         parent as parent_item_id
+
+        --The below macro adds the fields defined within your items_pass_through_columns variable into the staging model
+        {{ fivetran_utils.fill_pass_through_columns('items_pass_through_columns') }}
+
     from fields
     where not coalesce(_fivetran_deleted, false)
 )

models/netsuite2/stg_netsuite2__vendors.sql

@@ -27,6 +27,10 @@ final as (
         companyname as company_name,
         datecreated as create_date_at,
         category as vendor_category_id
+
+        --The below macro adds the fields defined within your vendors_pass_through_columns variable into the staging model
+        {{ fivetran_utils.fill_pass_through_columns('vendors_pass_through_columns') }}
+
     from fields
     where not coalesce(_fivetran_deleted, false)
 )

packages.yml

@@ -1,3 +1,6 @@
 packages:
 - package: fivetran/fivetran_utils
   version: [">=0.4.0", "<0.5.0"]
+
+- package: dbt-labs/spark_utils
+  version: [">=0.3.0", "<0.4.0"]