Skip to content

Commit e76aba2

Browse files
fivetran-joemarkiewiczfivetran-joemarkiewicz
authored
Merge pull request #33 from fivetran/MagicBot/dbt-utils-cross-db-migration
Updates for dbt-utils to dbt-core cross-db macro migration
2 parents b7554b5 + 27c0f9e commit e76aba2

19 files changed

+309
-202
lines changed

‎.buildkite/hooks/pre-command

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
# Export secrets for Docker containers.
6+
# Restrict exposing secrets only to the steps that need them
7+
export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
8+
export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917")
9+
export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917")
10+
export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917")
11+
export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917")
12+
export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917")
13+
export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917")
14+
export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917")
15+
export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917")
16+
export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917")
17+
export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917")
18+
export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917")
19+
export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917")
20+
export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917")
21+
export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
22+
export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
23+
export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
24+
export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")

‎.buildkite/pipeline.yml

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
steps:
2+
- label: ":postgres: Run Tests - Postgres"
3+
key: "run-dbt-postgres"
4+
plugins:
5+
- docker#v3.13.0:
6+
image: "python:3.8"
7+
shell: [ "/bin/bash", "-e", "-c" ]
8+
environment:
9+
- "BASH_ENV=/tmp/.bashrc"
10+
- "CI_POSTGRES_DBT_DBNAME"
11+
- "CI_POSTGRES_DBT_HOST"
12+
- "CI_POSTGRES_DBT_PASS"
13+
- "CI_POSTGRES_DBT_USER"
14+
commands: |
15+
bash .buildkite/scripts/run_models.sh postgres
16+
17+
- label: ":snowflake-db: Run Tests - Snowflake"
18+
key: "run_dbt_snowflake"
19+
plugins:
20+
- docker#v3.13.0:
21+
image: "python:3.8"
22+
shell: [ "/bin/bash", "-e", "-c" ]
23+
environment:
24+
- "BASH_ENV=/tmp/.bashrc"
25+
- "CI_SNOWFLAKE_DBT_ACCOUNT"
26+
- "CI_SNOWFLAKE_DBT_DATABASE"
27+
- "CI_SNOWFLAKE_DBT_PASS"
28+
- "CI_SNOWFLAKE_DBT_ROLE"
29+
- "CI_SNOWFLAKE_DBT_USER"
30+
- "CI_SNOWFLAKE_DBT_WAREHOUSE"
31+
commands: |
32+
bash .buildkite/scripts/run_models.sh snowflake
33+
34+
- label: ":gcloud: Run Tests - BigQuery"
35+
key: "run_dbt_bigquery"
36+
plugins:
37+
- docker#v3.13.0:
38+
image: "python:3.8"
39+
shell: [ "/bin/bash", "-e", "-c" ]
40+
environment:
41+
- "BASH_ENV=/tmp/.bashrc"
42+
- "GCLOUD_SERVICE_KEY"
43+
commands: |
44+
bash .buildkite/scripts/run_models.sh bigquery
45+
46+
- label: ":amazon-redshift: Run Tests - Redshift"
47+
key: "run_dbt_redshift"
48+
plugins:
49+
- docker#v3.13.0:
50+
image: "python:3.8"
51+
shell: [ "/bin/bash", "-e", "-c" ]
52+
environment:
53+
- "BASH_ENV=/tmp/.bashrc"
54+
- "CI_REDSHIFT_DBT_DBNAME"
55+
- "CI_REDSHIFT_DBT_HOST"
56+
- "CI_REDSHIFT_DBT_PASS"
57+
- "CI_REDSHIFT_DBT_USER"
58+
commands: |
59+
bash .buildkite/scripts/run_models.sh redshift

‎.buildkite/scripts/run_models.sh

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
apt-get update
6+
apt-get install libsasl2-dev
7+
8+
python3 -m venv venv
9+
. venv/bin/activate
10+
pip install --upgrade pip setuptools
11+
pip install -r integration_tests/requirements.txt
12+
mkdir -p ~/.dbt
13+
cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
14+
15+
db=$1
16+
echo `pwd`
17+
cd integration_tests
18+
dbt deps
19+
dbt seed --target "$db" --full-refresh
20+
dbt run --target "$db" --full-refresh
21+
dbt test --target "$db"
22+
dbt run --vars '{xero__using_credit_note: false, xero__using_bank_transaction: false}' --target "$db" --full-refresh
23+
dbt test --target "$db"
24+
dbt run --target "$db" --full-refresh
25+
dbt test --target "$db"

‎.circleci/config.yml

-75
This file was deleted.

‎.github/pull_request_template.md

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
Pull Request
21
**Are you a current Fivetran customer?**
32
<!--- Please tell us your name, title and company -->
43

@@ -29,9 +28,9 @@ Pull Request
2928

3029
**How did you test the PR changes?**
3130
<!--- Proof of testing is required in order for the PR to be approved. -->
32-
<!--- To check a box, remove the space and insert an x in the box (eg. [x] CircleCi). -->
31+
<!--- To check a box, remove the space and insert an x in the box (eg. [x] Buildkite). -->
3332
<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
34-
- [ ] CircleCi <!--- CircleCi testing is only applicable to Fivetran employees. -->
33+
- [ ] Buildkite <!--- Buildkite testing is only applicable to Fivetran employees. -->
3534
- [ ] Local (please provide additional testing details below)
3635

3736
**Select which warehouse(s) were used to test the PR**

‎CHANGELOG.md

+39
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,42 @@
1+
# dbt_xero v0.5.0
2+
3+
## 🚨 Breaking Changes 🚨:
4+
[PR #33](https://github.com/fivetran/dbt_xero/pull/33) includes the following breaking changes:
5+
- Dispatch update for dbt-utils to dbt-core cross-db macros migration. Specifically `{{ dbt_utils.<macro> }}` have been updated to `{{ dbt.<macro> }}` for the below macros:
6+
- `any_value`
7+
- `bool_or`
8+
- `cast_bool_to_text`
9+
- `concat`
10+
- `date_trunc`
11+
- `dateadd`
12+
- `datediff`
13+
- `escape_single_quotes`
14+
- `except`
15+
- `hash`
16+
- `intersect`
17+
- `last_day`
18+
- `length`
19+
- `listagg`
20+
- `position`
21+
- `replace`
22+
- `right`
23+
- `safe_cast`
24+
- `split_part`
25+
- `string_literal`
26+
- `type_bigint`
27+
- `type_float`
28+
- `type_int`
29+
- `type_numeric`
30+
- `type_string`
31+
- `type_timestamp`
32+
- `array_append`
33+
- `array_concat`
34+
- `array_construct`
35+
- For `current_timestamp` and `current_timestamp_in_utc` macros, the dispatch AND the macro names have been updated to the below, respectively:
36+
- `dbt.current_timestamp_backcompat`
37+
- `dbt.current_timestamp_in_utc_backcompat`
38+
- Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
39+
140
# dbt_xero v0.4.2
241
## Bug Fix
342
- Fixes duplicate values in `net_amount` field in `xero__balance_sheet_report` when leveraging logic to union multiple Xero connectors

0 commit comments

Comments
 (0)