Skip to content

Commit

Permalink
fix: test_not_null_proportion support on athena (#19)
Browse files Browse the repository at this point in the history
  • Loading branch information
Jrmyy authored May 20, 2024
1 parent 4f92576 commit b3d2e12
Showing 1 changed file with 30 additions and 0 deletions.
30 changes: 30 additions & 0 deletions macros/dbt_utils/generic_tests/not_null_proportion.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{% macro athena__test_not_null_proportion(model, group_by_columns) %}

{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}
{% set at_least = kwargs.get('at_least', kwargs.get('arg')) %}
{% set at_most = kwargs.get('at_most', kwargs.get('arg', 1)) %}

{% if group_by_columns|length() > 0 %}
{% set select_gb_cols = group_by_columns|join(' ,') + ', ' %}
{% set groupby_gb_cols = 'group by ' + group_by_columns|join(',') %}
{% endif %}

with validation as (
select
{{select_gb_cols}}
sum(case when {{ column_name }} is null then 0 else 1 end) / cast(count(*) as double) as not_null_proportion
from {{ model }}
{{groupby_gb_cols}}
),
validation_errors as (
select
{{select_gb_cols}}
not_null_proportion
from validation
where not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }}
)
select
*
from validation_errors

{% endmacro %}

0 comments on commit b3d2e12

Please sign in to comment.