diff --git a/README.md b/README.md index 7bd4a1c2..d841b616 100644 --- a/README.md +++ b/README.md @@ -326,6 +326,15 @@ models: upper_bound_column: ended_at partition_by: customer_id gaps: required + + # test that each customer can have subscriptions that start and end on the same date + - name: subscriptions + tests: + - dbt_utils.mutually_exclusive_ranges: + lower_bound_column: started_at + upper_bound_column: ended_at + partition_by: customer_id + zero_length: allowed ``` **Args:** * `lower_bound_column` (required): The name of the column that represents the @@ -337,6 +346,8 @@ upper value of the range. Must be not null. argument to indicate which column to partition by. `default=none` * `gaps` (optional): Whether there can be gaps are allowed between ranges. `default='allowed', one_of=['not_allowed', 'allowed', 'required']` +* `zero_length` (optional): Whether ranges can start and end on the same date. +`default='not_allowed', one_of=['not_allowed', 'allowed']` **Note:** Both `lower_bound_column` and `upper_bound_column` should be not null. If this is not the case in your data source, consider passing a coalesce function @@ -383,6 +394,24 @@ the lower bound of the next record (common for date ranges). | 2 | 3 | | 4 | 5 | +**Understanding the `zero_length_range_allowed` parameter:** +Here are a number of examples for each allowed `zero_length_range_allowed` parameter. +* `zero_length_range_allowed:false`: (default) The upper bound of each record must be greater than its lower bound. + +| lower_bound | upper_bound | +|-------------|-------------| +| 0 | 1 | +| 1 | 2 | +| 2 | 3 | + +* `zero_length_range_allowed:true`: The upper bound of each record can be greater than or equal to its lower bound. + +| lower_bound | upper_bound | +|-------------|-------------| +| 0 | 1 | +| 2 | 2 | +| 3 | 4 | + #### unique_combination_of_columns ([source](macros/schema_tests/unique_combination_of_columns.sql)) This test confirms that the combination of columns is unique. For example, the combination of month and product is unique, however neither column is unique diff --git a/integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv b/integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv new file mode 100644 index 00000000..29330c8f --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv @@ -0,0 +1,7 @@ +subscription_id,valid_from,valid_to +3,2020-05-06,2020-05-07 +3,2020-05-08,2020-05-08 +3,2020-05-09,2020-05-10 +4,2020-06-06,2020-06-07 +4,2020-06-08,2020-06-08 +4,2020-06-09,2020-06-10 \ No newline at end of file diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index 0026b957..deae3f13 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -95,6 +95,14 @@ models: partition_by: subscription_id gaps: required + - name: data_test_mutually_exclusive_ranges_with_gaps_zero_length + tests: + - dbt_utils.mutually_exclusive_ranges: + lower_bound_column: valid_from + upper_bound_column: valid_to + partition_by: subscription_id + zero_length_range_allowed: true + - name: data_unique_combination_of_columns tests: - dbt_utils.unique_combination_of_columns: diff --git a/macros/schema_tests/mutually_exclusive_ranges.sql b/macros/schema_tests/mutually_exclusive_ranges.sql index fab228f5..65007aad 100644 --- a/macros/schema_tests/mutually_exclusive_ranges.sql +++ b/macros/schema_tests/mutually_exclusive_ranges.sql @@ -1,4 +1,4 @@ -{% macro test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed') %} +{% macro test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} {% if gaps == 'not_allowed' %} {% set allow_gaps_operator='=' %} @@ -13,7 +13,17 @@ {{ exceptions.raise_compiler_error( "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" ) }} - +{% endif %} +{% if not zero_length_range_allowed %} + {% set allow_zero_length_operator='<' %} + {% set allow_zero_length_operator_in_words='less_than' %} +{% elif zero_length_range_allowed %} + {% set allow_zero_length_operator='<=' %} + {% set allow_zero_length_operator_in_words='less_than_or_equal_to' %} +{% else %} + {{ exceptions.raise_compiler_error( + "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" ~ zero_length ~"'.'" + ) }} {% endif %} {% set partition_clause="partition by " ~ partition_by if partition_by else '' %} @@ -51,9 +61,9 @@ calc as ( -- Coalesce it to return an error on the null case (implicit assumption -- these columns are not_null) coalesce( - lower_bound < upper_bound, + lower_bound {{ allow_zero_length_operator }} upper_bound, false - ) as lower_bound_less_than_upper_bound, + ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. -- Coalesce it to handle null cases for the last record. @@ -75,7 +85,7 @@ validation_errors as ( where not( -- THE FOLLOWING SHOULD BE TRUE -- - lower_bound_less_than_upper_bound + lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound ) )