diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b3a76d3..bda965a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # dbt-utils v0.6.5 (unreleased) ## Features +* Add new `accepted_range` test ([#276](https://github.com/fishtown-analytics/dbt-utils/pull/276) [@joellabes](https://github.com/joellabes)) ## Fixes diff --git a/README.md b/README.md index c4d33cb1..87577239 100644 --- a/README.md +++ b/README.md @@ -418,6 +418,45 @@ An optional `quote_columns` parameter (`default=false`) can also be used if a co quote_columns: true ``` + +#### accepted_range ([source](macros/schema_tests/accepted_range.sql)) +This test checks that a column's values fall inside an expected range. Any combination of `min_value` and `max_value` is allowed, and the range can be inclusive or exclusive. Provide a `where` argument to filter to specific records only. + +In addition to comparisons to a scalar value, you can also compare to another column's values. Any data type that supports the `>` or `<` operators can be compared, so you could also run tests like checking that all order dates are in the past. + +Usage: +```yaml +version: 2 + +models: + - name: model_name + columns: + - name: user_id + tests: + - dbt_utils.accepted_range: + min_value: 0 + inclusive: false + + - name: account_created_at + tests: + - dbt_utils.accepted_range: + max_value: "getdate()" + #inclusive is true by default + + - name: num_returned_orders + tests: + - dbt_utils.accepted_range: + min_value: 0 + max_value: "num_orders" + + - name: num_web_sessions + tests: + - dbt_utils.accepted_range: + min_value: 0 + inclusive: false + where: "num_orders > 0" +``` + --- ### SQL helpers #### get_query_results_as_dict ([source](macros/sql/get_query_results_as_dict.sql)) diff --git a/integration_tests/data/schema_tests/data_test_accepted_range.csv b/integration_tests/data/schema_tests/data_test_accepted_range.csv new file mode 100644 index 00000000..38b90a50 --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_accepted_range.csv @@ -0,0 +1,3 @@ +id +-1 +11 \ No newline at end of file diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index 55de5a68..0026b957 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -101,3 +101,22 @@ models: combination_of_columns: - month - product + + - name: data_test_accepted_range + columns: + - name: id + tests: + - dbt_utils.accepted_range: + min_value: -1 + max_value: 11 + inclusive: true + + - dbt_utils.accepted_range: + min_value: -2 + max_value: 11.1 + inclusive: false + + - dbt_utils.accepted_range: + min_value: 0 + inclusive: true + where: "id <> -1" diff --git a/macros/schema_tests/accepted_range.sql b/macros/schema_tests/accepted_range.sql new file mode 100644 index 00000000..4c599f8a --- /dev/null +++ b/macros/schema_tests/accepted_range.sql @@ -0,0 +1,32 @@ +{% macro test_accepted_range(model, min_value = none, max_value = none, inclusive = true, where = "true") %} + +{%- set column_name = kwargs.get('column_name', kwargs.get('field')) -%} + +with meet_condition as( + select {{ column_name }} + from {{ model }} + where {{ where }} +), + +validation_errors as ( + select * + from meet_condition + where + -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds + 1 = 2 + + {%- if min_value is not none %} + -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. + or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} + {%- endif %} + + {%- if max_value is not none %} + -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule. + or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} + {%- endif %} +) + +select count(*) +from validation_errors + +{% endmacro %}