From 949d40d08d075e0ee01c4292856381633c61c8a8 Mon Sep 17 00:00:00 2001 From: Benjamin Ryon Date: Sat, 19 Dec 2020 00:28:22 -0800 Subject: [PATCH 1/2] Add zero length option to exclusive range test. --- README.md | 29 +++++++++++++++++++ ...exclusive_ranges_with_gaps_zero_length.csv | 7 +++++ .../models/schema_tests/schema.yml | 8 +++++ .../mutually_exclusive_ranges.sql | 20 +++++++++---- 4 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv diff --git a/README.md b/README.md index 7bd4a1c2..4ea96df9 100644 --- a/README.md +++ b/README.md @@ -326,6 +326,15 @@ models: upper_bound_column: ended_at partition_by: customer_id gaps: required + + # test that each customer can have subscriptions that start and end on the same date + - name: subscriptions + tests: + - dbt_utils.mutually_exclusive_ranges: + lower_bound_column: started_at + upper_bound_column: ended_at + partition_by: customer_id + zero_length: allowed ``` **Args:** * `lower_bound_column` (required): The name of the column that represents the @@ -337,6 +346,8 @@ upper value of the range. Must be not null. argument to indicate which column to partition by. `default=none` * `gaps` (optional): Whether there can be gaps are allowed between ranges. `default='allowed', one_of=['not_allowed', 'allowed', 'required']` +* `zero_length` (optional): Whether ranges can start and end on the same date. +`default='not_allowed', one_of=['not_allowed', 'allowed']` **Note:** Both `lower_bound_column` and `upper_bound_column` should be not null. If this is not the case in your data source, consider passing a coalesce function @@ -383,6 +394,24 @@ the lower bound of the next record (common for date ranges). | 2 | 3 | | 4 | 5 | +**Understanding the `zero_length` parameter:** +Here are a number of examples for each allowed `zero_length` parameter. +* `zero_length:not_allowed`: The upper bound of each record must be greater than its lower bound. + +| lower_bound | upper_bound | +|-------------|-------------| +| 0 | 1 | +| 1 | 2 | +| 2 | 3 | + +* `zero_length:allowed`: The upper bound of each record can be greater than or equal to its lower bound. + +| lower_bound | upper_bound | +|-------------|-------------| +| 0 | 1 | +| 2 | 2 | +| 3 | 4 | + #### unique_combination_of_columns ([source](macros/schema_tests/unique_combination_of_columns.sql)) This test confirms that the combination of columns is unique. For example, the combination of month and product is unique, however neither column is unique diff --git a/integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv b/integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv new file mode 100644 index 00000000..29330c8f --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv @@ -0,0 +1,7 @@ +subscription_id,valid_from,valid_to +3,2020-05-06,2020-05-07 +3,2020-05-08,2020-05-08 +3,2020-05-09,2020-05-10 +4,2020-06-06,2020-06-07 +4,2020-06-08,2020-06-08 +4,2020-06-09,2020-06-10 \ No newline at end of file diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index 0026b957..e9015a14 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -95,6 +95,14 @@ models: partition_by: subscription_id gaps: required + - name: data_test_mutually_exclusive_ranges_with_gaps_zero_length + tests: + - dbt_utils.mutually_exclusive_ranges: + lower_bound_column: valid_from + upper_bound_column: valid_to + partition_by: subscription_id + zero_length: allowed + - name: data_unique_combination_of_columns tests: - dbt_utils.unique_combination_of_columns: diff --git a/macros/schema_tests/mutually_exclusive_ranges.sql b/macros/schema_tests/mutually_exclusive_ranges.sql index fab228f5..9e0de820 100644 --- a/macros/schema_tests/mutually_exclusive_ranges.sql +++ b/macros/schema_tests/mutually_exclusive_ranges.sql @@ -1,4 +1,4 @@ -{% macro test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed') %} +{% macro test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length='not_allowed') %} {% if gaps == 'not_allowed' %} {% set allow_gaps_operator='=' %} @@ -13,7 +13,17 @@ {{ exceptions.raise_compiler_error( "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" ) }} - +{% endif %} +{% if zero_length == 'not_allowed' %} + {% set allow_zero_length_operator='<' %} + {% set allow_zero_length_operator_in_words='less_than' %} +{% elif zero_length == 'allowed' %} + {% set allow_zero_length_operator='<=' %} + {% set allow_zero_length_operator_in_words='less_than_or_equal_to' %} +{% else %} + {{ exceptions.raise_compiler_error( + "`zero_length` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed'] Got: '" ~ zero_length ~"'.'" + ) }} {% endif %} {% set partition_clause="partition by " ~ partition_by if partition_by else '' %} @@ -51,9 +61,9 @@ calc as ( -- Coalesce it to return an error on the null case (implicit assumption -- these columns are not_null) coalesce( - lower_bound < upper_bound, + lower_bound {{ allow_zero_length_operator }} upper_bound, false - ) as lower_bound_less_than_upper_bound, + ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. -- Coalesce it to handle null cases for the last record. @@ -75,7 +85,7 @@ validation_errors as ( where not( -- THE FOLLOWING SHOULD BE TRUE -- - lower_bound_less_than_upper_bound + lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound ) ) From fb0fa0f85e5f26dcc53ee1863a9a97ee00c606d0 Mon Sep 17 00:00:00 2001 From: Benjamin Ryon Date: Wed, 23 Dec 2020 13:28:55 -0800 Subject: [PATCH 2/2] Use boolean for zero range arg. --- README.md | 8 ++++---- integration_tests/models/schema_tests/schema.yml | 2 +- macros/schema_tests/mutually_exclusive_ranges.sql | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4ea96df9..d841b616 100644 --- a/README.md +++ b/README.md @@ -394,9 +394,9 @@ the lower bound of the next record (common for date ranges). | 2 | 3 | | 4 | 5 | -**Understanding the `zero_length` parameter:** -Here are a number of examples for each allowed `zero_length` parameter. -* `zero_length:not_allowed`: The upper bound of each record must be greater than its lower bound. +**Understanding the `zero_length_range_allowed` parameter:** +Here are a number of examples for each allowed `zero_length_range_allowed` parameter. +* `zero_length_range_allowed:false`: (default) The upper bound of each record must be greater than its lower bound. | lower_bound | upper_bound | |-------------|-------------| @@ -404,7 +404,7 @@ Here are a number of examples for each allowed `zero_length` parameter. | 1 | 2 | | 2 | 3 | -* `zero_length:allowed`: The upper bound of each record can be greater than or equal to its lower bound. +* `zero_length_range_allowed:true`: The upper bound of each record can be greater than or equal to its lower bound. | lower_bound | upper_bound | |-------------|-------------| diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index e9015a14..deae3f13 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -101,7 +101,7 @@ models: lower_bound_column: valid_from upper_bound_column: valid_to partition_by: subscription_id - zero_length: allowed + zero_length_range_allowed: true - name: data_unique_combination_of_columns tests: diff --git a/macros/schema_tests/mutually_exclusive_ranges.sql b/macros/schema_tests/mutually_exclusive_ranges.sql index 9e0de820..65007aad 100644 --- a/macros/schema_tests/mutually_exclusive_ranges.sql +++ b/macros/schema_tests/mutually_exclusive_ranges.sql @@ -1,4 +1,4 @@ -{% macro test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length='not_allowed') %} +{% macro test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} {% if gaps == 'not_allowed' %} {% set allow_gaps_operator='=' %} @@ -14,15 +14,15 @@ "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" ) }} {% endif %} -{% if zero_length == 'not_allowed' %} +{% if not zero_length_range_allowed %} {% set allow_zero_length_operator='<' %} {% set allow_zero_length_operator_in_words='less_than' %} -{% elif zero_length == 'allowed' %} +{% elif zero_length_range_allowed %} {% set allow_zero_length_operator='<=' %} {% set allow_zero_length_operator_in_words='less_than_or_equal_to' %} {% else %} {{ exceptions.raise_compiler_error( - "`zero_length` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed'] Got: '" ~ zero_length ~"'.'" + "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" ~ zero_length ~"'.'" ) }} {% endif %}