diff --git a/CHANGELOG.md b/CHANGELOG.md index 523ced05..45aa9316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Features * Add new `accepted_range` test ([#276](https://github.com/fishtown-analytics/dbt-utils/pull/276) [@joellabes](https://github.com/joellabes)) * Make `expression_is_true` work as a column test (code originally in [#226](https://github.com/fishtown-analytics/dbt-utils/pull/226/) from [@elliottohara](https://github.com/elliottohara), merged via [#313]) +* Add new schema test, `not_accepted_values` ([#284](https://github.com/fishtown-analytics/dbt-utils/pull/284) [@JavierMonton](https://github.com/JavierMonton)) ## Fixes * Handle booleans gracefully in the unpivot macro ([#305](https://github.com/fishtown-analytics/dbt-utils/pull/305) [@avishalom](https://github.com/avishalom)) @@ -24,7 +25,6 @@ - Bump `require-dbt-version` to `[">=0.18.0", "<0.20.0"]` to support dbt v0.19.0 ([#308](https://github.com/fishtown-analytics/dbt-utils/pull/308), [#309](https://github.com/fishtown-analytics/dbt-utils/pull/309)) - # dbt-utils v0.6.2 ## Fixes diff --git a/README.md b/README.md index 6b276567..6fac36cf 100644 --- a/README.md +++ b/README.md @@ -300,6 +300,22 @@ models: where: "_deleted = false" ``` +#### not_accepted_values ([source](macros/schema_tests/not_accepted_values.sql)) +This test validates that there are no rows that match the given values. + +Usage: +```yaml +version: 2 + +models: + - name: my_model + columns: + - name: city + tests: + - dbt_utils.not_accepted_values: + values: ['Barcelona', 'New York'] +``` + #### relationships_where ([source](macros/schema_tests/relationships_where.sql)) This test validates the referential integrity between two relations (same as the core relationships schema test) with an added predicate to filter out some rows from the test. This is useful to exclude records such as test entities, rows created in the last X minutes/hours to account for temporary gaps due to ETL limitations, etc. diff --git a/integration_tests/data/schema_tests/data_test_not_accepted_values.csv b/integration_tests/data/schema_tests/data_test_not_accepted_values.csv new file mode 100644 index 00000000..85b190f2 --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_not_accepted_values.csv @@ -0,0 +1,5 @@ +id,city +1,Barcelona +2,London +3,Paris +4,New York diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index cbc9cff9..0604be34 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -74,6 +74,13 @@ models: - dbt_utils.not_null_where: where: "_deleted = false" + - name: data_test_not_accepted_values + columns: + - name: city + tests: + - dbt_utils.not_accepted_values: + values: ['Madrid', 'Berlin'] + - name: data_test_relationships_where_table_2 columns: - name: id diff --git a/macros/schema_tests/not_accepted_values.sql b/macros/schema_tests/not_accepted_values.sql new file mode 100644 index 00000000..fa2f4d83 --- /dev/null +++ b/macros/schema_tests/not_accepted_values.sql @@ -0,0 +1,37 @@ +{% macro test_not_accepted_values(model, values) %} + +{% set column_name = kwargs.get('column_name', kwargs.get('field')) %} +{% set quote_values = kwargs.get('quote', True) %} + +with all_values as ( + + select distinct + {{ column_name }} as value_field + + from {{ model }} + +), + +validation_errors as ( + + select + value_field + + from all_values + where value_field in ( + {% for value in values -%} + {% if quote_values -%} + '{{ value }}' + {%- else -%} + {{ value }} + {%- endif -%} + {%- if not loop.last -%},{%- endif %} + {%- endfor %} + ) + +) + +select count(*) as validation_errors +from validation_errors + +{% endmacro %}