Initial sale view tests

ccao-data · Nov 15, 2023 · aa1ad67 · aa1ad67
1 parent 5899038
commit aa1ad67
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 49 deletions.
diff --git a/dbt/macros/row_count_by_group.sql b/dbt/macros/row_count_by_group.sql
@@ -6,11 +6,13 @@
 -- print the results of the query to stdout, which allows this macro to be used
 -- by scripts to return data.
 {% macro row_count_by_group(model, group_by, ordering="asc", print=False) %}
+    {%- set columns_csv = group_by | join(", ") %}
+
     {% set query %}
-        select count(*) as COUNT, {{ group_by }}
+        select count(*) as COUNT, {{ columns_csv }}
         from {{ model }}
-        group by {{ group_by }}
-        order by {{ group_by }} {{ ordering }}
+        group by {{ columns_csv }}
+        order by {{ columns_csv }} {{ ordering }}
     {% endset %}
 
     {% if print %}

diff --git a/dbt/models/default/schema/default.vw_pin_sale.yml b/dbt/models/default/schema/default.vw_pin_sale.yml
@@ -55,25 +55,38 @@ models:
         description: '{{ doc("shared_column_year") }}'
 
     tests:
-      # Number of sales for a given time period isn't suspicious
+      # Sales should be unique by PIN, year, and deed. Deeds can have
+      # multiple PINs
       - unique_combination_of_columns:
-          name: vw_pin_sale_reasonable_number_of_sales_per_year
+          name: vw_pin_sale_unique_by_pin_year_doc_no
           combination_of_columns:
             - pin
             - year
-          allowed_duplicates: 2
+            - doc_no
           config:
-            error_if: ">4032"
+            where: |
+              NOT sale_filter_same_sale_within_365
       # No sales for same price/pin within 12 months
       - unique_combination_of_columns:
-          name: vw_pin_sale_unique_price_pin_and_year
+          name: vw_pin_sale_unique_by_pin_year_price
           combination_of_columns:
             - pin
             - year
             - sale_price
-            - sale_filter_same_sale_within_365
           config:
-            error_if: ">2079"
+            error_if: ">1547"
+            where: |
+              NOT sale_filter_same_sale_within_365
+      # Monthly sales volume within expectations
+      - row_count_by_group:
+          name: vw_pin_sale_count_by_year_and_month
+          group_by:
+            - year
+            - MONTH(DATE_PARSE(SUBSTR(saledt, 1, 10), '%Y-%m-%d')) AS month
+          config:
+            where: |
+              NOT sale_filter_same_sale_within_365
+
       # TODO: Sale is validated (after sales validation has been added to
       # iasworld)
       # TODO: Validation is catching obvious outliers

diff --git a/dbt/models/reporting/schema.yml b/dbt/models/reporting/schema.yml
@@ -8,30 +8,30 @@ sources:
         tests:
           - expression_is_true:
               name: ratio_stats_no_nulls
-              expression: >-
+              expression: |
                 year IS NOT NULL
-                  AND triad IS NOT NULL
-                  AND geography_type IS NOT NULL
-                  AND property_group IS NOT NULL
-                  AND assessment_stage IS NOT NULL
-                  AND sale_year IS NOT NULL
+                AND triad IS NOT NULL
+                AND geography_type IS NOT NULL
+                AND property_group IS NOT NULL
+                AND assessment_stage IS NOT NULL
+                AND sale_year IS NOT NULL
           - expression_is_true:
               name: ratio_stats_metrics_are_sensible
-              expression: >-
+              expression: |
                 cod >= 0
-                  AND prd >= 0
-                  AND prb BETWEEN -1 AND 1
-                  AND mki >= 0
-                  AND triad IS NOT NULL
-                  AND geography_type IS NOT NULL
-                  AND property_group IS NOT NULL
-                  AND assessment_stage IS NOT NULL
-                  AND sale_year IS NOT NULL
+                AND prd >= 0
+                AND prb BETWEEN -1 AND 1
+                AND mki >= 0
+                AND triad IS NOT NULL
+                AND geography_type IS NOT NULL
+                AND property_group IS NOT NULL
+                AND assessment_stage IS NOT NULL
+                AND sale_year IS NOT NULL
           - expression_is_true:
               name: ratio_stats_within_directionality
-              expression: >-
+              expression: |
                 within_20_pct >= within_10_pct
-                  AND within_10_pct >= within_05_pct
+                AND within_10_pct >= within_05_pct
 
 models:
   - name: reporting.res_report_summary
@@ -55,11 +55,11 @@ models:
           - year
     - expression_is_true:
         name: res_report_summary_no_nulls
-        expression: >-
+        expression: |
           triad IS NOT NULL
-            AND geography_type IS NOT NULL
-            AND property_group IS NOT NULL
-            AND assessment_stage IS NOT NULL
+          AND geography_type IS NOT NULL
+          AND property_group IS NOT NULL
+          AND assessment_stage IS NOT NULL
         select_columns:
           - triad
           - geography_type
@@ -78,11 +78,11 @@ models:
             - class
       - expression_is_true:
           name: vw_assessment_roll_no_nulls
-          expression: >-
+          expression: |
             stage IS NOT NULL
-              AND township_name IS NOT NULL
-              AND triad IS NOT NULL
-              AND class IS NOT NULL
+            AND township_name IS NOT NULL
+            AND triad IS NOT NULL
+            AND class IS NOT NULL
 
   - name: reporting.vw_pin_most_recent_boundary
     description: '{{ doc("view_vw_pin_most_recent_boundary") }}'
@@ -123,11 +123,11 @@ models:
             - sale_price
       - expression_is_true:
           name: vw_ratio_stats_no_nulls
-          expression: >-
+          expression: |
             property_group IS NOT NULL
-              AND assessment_stage IS NOT NULL
-              AND triad IS NOT NULL
-              AND township_code IS NOT NULL
+            AND assessment_stage IS NOT NULL
+            AND triad IS NOT NULL
+            AND township_code IS NOT NULL
       - expression_is_true:
           name: vw_ratio_stats_sale_year_equals_year_minus_one
           expression: CAST(sale_year AS INTEGER) = CAST(year AS INTEGER) - 1
@@ -154,13 +154,13 @@ models:
           expression: rank BETWEEN 1 AND 5
       - expression_is_true:
           name: vw_top_5_no_nulls
-          expression: >-
+          expression: |
             year IS NOT NULL
-              AND township IS NOT NULL
-              AND class IS NOT NULL
-              AND rank IS NOT NULL
-              AND total_av IS NOT NULL
-              AND stage_used IS NOT NULL
+            AND township IS NOT NULL
+            AND class IS NOT NULL
+            AND rank IS NOT NULL
+            AND total_av IS NOT NULL
+            AND stage_used IS NOT NULL
 
   - name: reporting.vw_town_sale_history
     description: '{{ doc("view_vw_town_sale_history") }}'
@@ -173,8 +173,8 @@ models:
             - sale_year
       - expression_is_true:
           name: vw_town_sale_history_no_nulls
-          expression: >-
+          expression: |
             sale_year IS NOT NULL
-              AND property_group IS NOT NULL
-              AND geography_id IS NOT NULL
-              AND sale_median IS NOT NULL
+            AND property_group IS NOT NULL
+            AND geography_id IS NOT NULL
+            AND sale_median IS NOT NULL
diff --git a/dbt/tests/generic/test_group_row_counts.sql b/dbt/tests/generic/test_group_row_counts.sql
@@ -0,0 +1,8 @@
+-- Test that the a given expression returns a non-zero number of rows
+{% test group_row_counts(model, group_by, condition) %}
+
+    select *
+    from ({{ row_count_by_group(model, group_by) }})
+    where count {{ condition }}
+
+{% endtest %}