From 10bf5402ac934447bd5299359258af14ad9152bd Mon Sep 17 00:00:00 2001 From: JavierMonton Date: Fri, 25 Sep 2020 11:59:23 +0200 Subject: [PATCH 1/4] Schema Test - Not accepted values --- README.md | 16 ++++++++ .../data_test_not_accepted_values.csv | 5 +++ .../models/schema_tests/schema.yml | 7 ++++ macros/schema_tests/not_accepted_values.sql | 37 +++++++++++++++++++ 4 files changed, 65 insertions(+) create mode 100644 integration_tests/data/schema_tests/data_test_not_accepted_values.csv create mode 100644 macros/schema_tests/not_accepted_values.sql diff --git a/README.md b/README.md index cd325203..a0ac2808 100644 --- a/README.md +++ b/README.md @@ -300,6 +300,22 @@ models: where: "_deleted = false" ``` +#### not_accepted_values ([source](macros/schema_tests/not_accepted_values.sql)) +This test validates that there are no rows that match the given values. + +Usage: +```yaml +version: 2 + +models: + - name: my_model + columns: + - name: city + tests: + - dbt_utils.not_accepted_values: + values: ['Barcelona', 'New York'] +``` + #### relationships_where ([source](macros/schema_tests/relationships_where.sql)) This test validates the referential integrity between two relations (same as the core relationships schema test) with an added predicate to filter out some rows from the test. This is useful to exclude records such as test entities, rows created in the last X minutes/hours to account for temporary gaps due to ETL limitations, etc. diff --git a/integration_tests/data/schema_tests/data_test_not_accepted_values.csv b/integration_tests/data/schema_tests/data_test_not_accepted_values.csv new file mode 100644 index 00000000..8e9ed35e --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_not_accepted_values.csv @@ -0,0 +1,5 @@ +id,city +1,Barcelona +2,London +3,Paris +4,New York \ No newline at end of file diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index cbc9cff9..40cc328f 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -74,6 +74,13 @@ models: - dbt_utils.not_null_where: where: "_deleted = false" + - name: data_test_not_accepted_values + columns: + - name: city + tests: + - dbt_utils.not_accepted_values: + values: ['Barcelona', 'New York', 'Berlin'] + - name: data_test_relationships_where_table_2 columns: - name: id diff --git a/macros/schema_tests/not_accepted_values.sql b/macros/schema_tests/not_accepted_values.sql new file mode 100644 index 00000000..c430b89d --- /dev/null +++ b/macros/schema_tests/not_accepted_values.sql @@ -0,0 +1,37 @@ +{% macro test_not_accepted_values(model, values) %} + +{% set column_name = kwargs.get('column_name', kwargs.get('field')) %} +{% set quote_values = kwargs.get('quote', True) %} + +with all_values as ( + + select distinct + {{ column_name }} as value_field + + from {{ model }} + +), + +validation_errors as ( + + select + value_field + + from all_values + where value_field in ( + {% for value in values -%} + {% if quote_values -%} + '{{ value }}' + {%- else -%} + {{ value }} + {%- endif -%} + {%- if not loop.last -%},{%- endif %} + {%- endfor %} + ) + +) + +select count(*) as validation_errors +from validation_errors + +{% endmacro %} \ No newline at end of file From 689707d3df3bb2aeab54b8997226f9b7bba2c29e Mon Sep 17 00:00:00 2001 From: JavierMonton Date: Fri, 25 Sep 2020 12:14:20 +0200 Subject: [PATCH 2/4] Schema Test - Not accepted values, test fixed --- integration_tests/models/schema_tests/schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index 40cc328f..0604be34 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -79,7 +79,7 @@ models: - name: city tests: - dbt_utils.not_accepted_values: - values: ['Barcelona', 'New York', 'Berlin'] + values: ['Madrid', 'Berlin'] - name: data_test_relationships_where_table_2 columns: From a3f75be8221d636de79ffaeca4d2c64f1f8acaae Mon Sep 17 00:00:00 2001 From: JavierMonton Date: Fri, 25 Sep 2020 12:55:44 +0200 Subject: [PATCH 3/4] empty lines at the end --- .../data/schema_tests/data_test_not_accepted_values.csv | 2 +- macros/schema_tests/not_accepted_values.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/data/schema_tests/data_test_not_accepted_values.csv b/integration_tests/data/schema_tests/data_test_not_accepted_values.csv index 8e9ed35e..85b190f2 100644 --- a/integration_tests/data/schema_tests/data_test_not_accepted_values.csv +++ b/integration_tests/data/schema_tests/data_test_not_accepted_values.csv @@ -2,4 +2,4 @@ id,city 1,Barcelona 2,London 3,Paris -4,New York \ No newline at end of file +4,New York diff --git a/macros/schema_tests/not_accepted_values.sql b/macros/schema_tests/not_accepted_values.sql index c430b89d..fa2f4d83 100644 --- a/macros/schema_tests/not_accepted_values.sql +++ b/macros/schema_tests/not_accepted_values.sql @@ -34,4 +34,4 @@ validation_errors as ( select count(*) as validation_errors from validation_errors -{% endmacro %} \ No newline at end of file +{% endmacro %} From f60eebc3a0f122bd133b9fc55baa2be2f0a49755 Mon Sep 17 00:00:00 2001 From: JavierMonton Date: Wed, 14 Oct 2020 17:14:25 +0200 Subject: [PATCH 4/4] Changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 735ab6e5..7e2d9698 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ ## Features * Add new `accepted_range` test ([#276](https://github.com/fishtown-analytics/dbt-utils/pull/276) [@joellabes](https://github.com/joellabes)) * Make `expression_is_true` work as a column test (code originally in [#226](https://github.com/fishtown-analytics/dbt-utils/pull/226/) from [@elliottohara](https://github.com/elliottohara), merged via [#313]) -* Allow individual columns in star macro to be aliased (code originally in [#230](https://github.com/fishtown-analytics/dbt-utils/pull/230/) from [@elliottohara](https://github.com/elliottohara), merged via [#245]) +* Allow individual columns in star macro to be aliased (code originally in [#230](https://github.com/fishtown-analytics/dbt-utils/pull/230/) from [@elliottohara](https://github.com/elliottohara), merged via [#245]) +* Add new schema test, `not_accepted_values` ([#284](https://github.com/fishtown-analytics/dbt-utils/pull/284) [@JavierMonton](https://github.com/JavierMonton)) # dbt-utils v0.6.2