Add iasWorld dbt tests and namespace all tests (#236)

* Initial sale view tests * Add ADDN tests * Add ADPRVAL tests * Add ASMT_ALL tests * Revert "Initial sale view tests" This reverts commit aa1ad67. * Add new test macro for mismatched rows * Test mismatched classes in ASMT_ALL vs PARDAT * Update class comparison tests * Improve class match testing * Add more tests and rename all iasworld tests * Update test names and formatting * Add more DWELDAT char tests * Finish DWELDAT tests * Add HTPAR tests * Add LAND table tests * Add LEGDAT and address tests * Add more LEGDAT address checks * Add OWNDAT tests * Add PARDAT tests * Add OBY tests * Add SALES tests * Add tags for FP QC * Update test schemas * Truncate test name * Truncate test name * Update test name * Replace redundant conditions with YAML anchors * Fix missing test namespace * Drop extra linebreaks * Add test_qc tag to all iasWorld tables * Exclude test_qc tests from build_and_test_dbt workflow * Add note about trigger to test_dbt_models
ccao-data · Nov 21, 2023 · 3551103 · 3551103
1 parent 2027c68
commit 3551103
Show file tree

Hide file tree

Showing 51 changed files with 1,610 additions and 122 deletions.
diff --git a/.github/workflows/build_and_test_dbt.yaml b/.github/workflows/build_and_test_dbt.yaml
@@ -79,14 +79,14 @@ jobs:
           if [[ $MODIFIED_RESOURCES_ONLY == 'true' ]]; then
             if [[ $MANUALLY_DISPATCHED == 'true' ]]; then
               echo "Running tests on manually selected resources"
-              dbt test -t "$TARGET" -s ${{ inputs.models }} --defer --state "$STATE_DIR"
+              dbt test -t "$TARGET" -s ${{ inputs.models }} --exclude "tag:test_qc*" --defer --state "$STATE_DIR"
             else
               echo "Running tests on modified/new resources only"
-              dbt test -t "$TARGET" -s state:modified state:new --defer --state "$STATE_DIR"
+              dbt test -t "$TARGET" -s state:modified state:new --exclude "tag:test_qc*" --defer --state "$STATE_DIR"
             fi
           else
             echo "Running tests on all resources"
-            dbt test -t "$TARGET"
+            dbt test -t "$TARGET" --exclude "tag:test_qc*"
           fi
         working-directory: ${{ env.PROJECT_DIR }}
         shell: bash

diff --git a/.github/workflows/test_dbt_models.yaml b/.github/workflows/test_dbt_models.yaml
@@ -1,5 +1,8 @@
 name: test-dbt-models
 
+# This workflow is not scheduled or run on PRs because it is manually triggered
+# by the Data Department's sqoop data extraction process upon completion. See
+# https://github.com/ccao-data/service-sqoop-iasworld
 on: workflow_dispatch
 
 jobs:

diff --git a/dbt/models/default/schema/default.vw_card_res_char.yml b/dbt/models/default/schema/default.vw_card_res_char.yml
@@ -109,7 +109,7 @@ models:
 
     tests:
       - unique_combination_of_columns:
-          name: vw_card_res_char_unique_by_pin_card_and_year
+          name: default_vw_card_res_char_unique_by_pin_card_and_year
           combination_of_columns:
             - pin
             - year

diff --git a/dbt/models/default/schema/default.vw_pin_address.yml b/dbt/models/default/schema/default.vw_pin_address.yml
@@ -51,7 +51,7 @@ models:
     tests:
       # No superfluous whitespace
       - no_extra_whitespace:
-          name: vw_pin_address_no_extra_whitespace
+          name: default_vw_pin_address_no_extra_whitespace
           column_names:
             - prop_address_full
             - prop_address_city_name
@@ -66,7 +66,7 @@ models:
           config:
             error_if: ">900000"
       - unique_combination_of_columns:
-          name: vw_pin_address_unique_by_14_digit_pin_and_year
+          name: default_vw_pin_address_unique_by_14_digit_pin_and_year
           combination_of_columns:
             - pin
             - year

diff --git a/dbt/models/default/schema/default.vw_pin_appeal.yml b/dbt/models/default/schema/default.vw_pin_appeal.yml
@@ -45,7 +45,7 @@ models:
     tests:
       # If there is a `change`, the values should reflect this
       - expression_is_true:
-          name: vw_pin_appeal_change_matches_appeal_outcome
+          name: default_vw_pin_appeal_change_matches_appeal_outcome
           expression: >-
             {% set vars = [
               ('mailed_bldg', 'certified_bldg'),
@@ -94,7 +94,7 @@ models:
             error_if: ">266719"
       # `change` should be an enum
       - expression_is_true:
-          name: vw_pin_appeal_no_unexpected_change_values
+          name: default_vw_pin_appeal_no_unexpected_change_values
           expression: change is null or change in ('change', 'no change')
           select_columns:
             - pin

diff --git a/dbt/models/default/schema/default.vw_pin_condo_char.yml b/dbt/models/default/schema/default.vw_pin_condo_char.yml
@@ -83,7 +83,7 @@ models:
 
     tests:
       - unique_combination_of_columns:
-          name: vw_pin_condo_char_unique_by_14_digit_pin_and_year
+          name: default_vw_pin_condo_char_unique_by_14_digit_pin_and_year
           combination_of_columns:
             - pin
             - year

diff --git a/dbt/models/default/schema/default.vw_pin_history.yml b/dbt/models/default/schema/default.vw_pin_history.yml
@@ -70,7 +70,7 @@ models:
 
     tests:
       - unique_combination_of_columns:
-          name: vw_pin_history_unique_by_14_digit_pin_and_year
+          name: default_vw_pin_history_unique_by_14_digit_pin_and_year
           combination_of_columns:
             - pin
             - year
diff --git a/dbt/models/default/schema/default.vw_pin_sale.yml b/dbt/models/default/schema/default.vw_pin_sale.yml
@@ -57,7 +57,7 @@ models:
     tests:
       # Number of sales for a given time period isn't suspicious
       - unique_combination_of_columns:
-          name: vw_pin_sale_reasonable_number_of_sales_per_year
+          name: default_vw_pin_sale_reasonable_number_of_sales_per_year
           combination_of_columns:
             - pin
             - year
@@ -66,7 +66,7 @@ models:
             error_if: ">4032"
       # No sales for same price/pin within 12 months
       - unique_combination_of_columns:
-          name: vw_pin_sale_unique_price_pin_and_year
+          name: default_vw_pin_sale_unique_price_pin_and_year
           combination_of_columns:
             - pin
             - year

diff --git a/dbt/models/default/schema/default.vw_pin_universe.yml b/dbt/models/default/schema/default.vw_pin_universe.yml
@@ -233,18 +233,18 @@ models:
     tests:
       # Number of classes is consistent over time
       - count_is_consistent:
-          name: vw_pin_universe_class_count_is_consistent_by_year
+          name: default_vw_pin_universe_class_count_is_consistent_by_year
           group_column: year
           count_column: class
           config:
             error_if: ">24"
       # Number of towns is consistent over time
       - count_is_consistent:
-          name: vw_pin_universe_township_code_count_is_consistent_by_year
+          name: default_vw_pin_universe_town_count_is_consistent_by_year
           group_column: year
           count_column: township_code
       - unique_combination_of_columns:
-          name: vw_pin_universe_unique_by_14_digit_pin_and_year
+          name: default_vw_pin_universe_unique_by_14_digit_pin_and_year
           combination_of_columns:
             - pin
             - year

diff --git a/dbt/models/default/schema/default.vw_pin_value.yml b/dbt/models/default/schema/default.vw_pin_value.yml
@@ -42,7 +42,7 @@ models:
 
     tests:
       - unique_combination_of_columns:
-          name: vw_pin_value_unique_by_14_digit_pin_and_year
+          name: default_vw_pin_value_unique_by_14_digit_pin_and_year
           combination_of_columns:
             - pin
             - year
diff --git a/dbt/models/iasworld/docs.md b/dbt/models/iasworld/docs.md
@@ -38,6 +38,11 @@ Main assessment table, union of `iasworld.asmt` and `iasworld.asmt_hist`.
 Provides the latest assessed value for each PIN and used heavily in
 Data Department views.
 
+### Nuance
+
+- Unlike most `iasworld` tables, this table _does not_ use `cur = 'Y'` to
+  identify the most recent record (it uses `procname` instead).
+
 **Primary Key**: `jur`, `rolltype`, `valclass`, `valyear`, `distcode`,
 `seq` `taxyr`, `parid`, `card`
 {% enddocs %}

diff --git a/dbt/models/iasworld/schema/iasworld.aasysjur.yml b/dbt/models/iasworld/schema/iasworld.aasysjur.yml
@@ -3,6 +3,7 @@ sources:
     loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0')
     tags:
       - load_auto
+      - test_qc_iasworld
 
     tables:
       - name: aasysjur

diff --git a/dbt/models/iasworld/schema/iasworld.addn.yml b/dbt/models/iasworld/schema/iasworld.addn.yml
@@ -3,6 +3,7 @@ sources:
     loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0')
     tags:
       - load_auto
+      - test_qc_iasworld
       - type_res
 
     tables:
@@ -12,6 +13,13 @@ sources:
         columns:
           - name: area
             description: '{{ doc("column_area") }}'
+            tests:
+              - dbt_utils.accepted_range:
+                  name: iasworld_addn_area_between_0_and_1.5M
+                  min_value: 0
+                  max_value: 1500000
+                  config:
+                    where: cur = 'Y' AND deactivat IS NULL
           - name: area_sqm
             description: Area of this line in meters
           - name: areaid
@@ -20,20 +28,48 @@ sources:
             description: '{{ doc("column_areasum") }}'
           - name: card
             description: '{{ doc("column_card") }}'
+            tests:
+              - not_null:
+                  name: iasworld_addn_card_not_null
+              - dbt_utils.accepted_range:
+                  name: iasworld_addn_card_gte_1
+                  min_value: 1
           - name: cdu
             description: '{{ doc("column_cdu") }}'
           - name: chgrsn
             description: '{{ doc("column_chgrsn") }}'
           - name: class
             description: '{{ doc("shared_column_class") }}'
+            tests:
+              - relationships:
+                  name: iasworld_addn_class_in_ccao_class_dict
+                  to: source('ccao', 'class_dict')
+                  field: class_code
+                  config:
+                    where: |
+                      taxyr >= '2022'
+                      AND class != 'EX'
+                      AND cur = 'Y'
+                      AND deactivat IS NULL
           - name: convaddn
             description: Converted addition total
           - name: cur
             description: '{{ doc("column_cur") }}'
+            tests:
+              - accepted_values:
+                  name: iasworld_addn_cur_in_accepted_values
+                  values: ['Y', 'D']
           - name: deactivat
             description: '{{ doc("column_deactivat") }}'
           - name: depr
             description: Percent good from tables (based on age+cdu)
+            tests:
+              - dbt_utils.accepted_range:
+                  name: iasworld_addn_depr_between_0_and_100
+                  min_value: 0
+                  max_value: 100
+                  config:
+                    where: cur = 'Y' AND deactivat IS NULL
           - name: eff_area
             description: Support value per line
           - name: effageovr
@@ -64,6 +100,9 @@ sources:
             description: '{{ doc("column_jur") }}'
           - name: lline
             description: '{{ doc("shared_column_lline") }}'
+            tests:
+              - not_null:
+                  name: iasworld_addn_lline_not_null
           - name: lower
             description: Lower level addition code
           - name: mktadj
@@ -82,6 +121,13 @@ sources:
             description: '{{ doc("column_newconpct") }}'
           - name: parid
             description: '{{ doc("shared_column_pin") }}'
+            tests:
+              - not_null:
+                  name: iasworld_addn_parid_not_null
+              - relationships:
+                  name: iasworld_addn_parid_in_pardat_parid
+                  to: source('iasworld', 'pardat')
+                  field: parid
           - name: pctcomp
             description: '{{ doc("column_pctcomp") }}'
           - name: prodamage
@@ -126,10 +172,21 @@ sources:
             description: Second floor addition code
           - name: seq
             description: '{{ doc("shared_column_seq") }}'
+            tests:
+              - dbt_utils.sequential_values:
+                  name: iasworld_addn_seq_all_sequential_exist
+                  group_by_columns:
+                    - parid
+                    - taxyr
+                    - card
+                    - lline
           - name: status
             description: '{{ doc("column_status") }}'
           - name: taxyr
             description: '{{ doc("shared_column_year") }}'
+            tests:
+              - not_null:
+                  name: iasworld_addn_taxyr_not_null
           - name: third
             description: Third floor addition code
           - name: trans_id
@@ -152,9 +209,10 @@ sources:
             description: '{{ doc("column_whocalc") }}'
           - name: yrblt
             description: '{{ doc("column_yrblt") }}'
+
         tests:
           - unique_combination_of_columns:
-              name: addn_unique_by_parid_taxyr_card_lline
+              name: iasworld_addn_unique_by_parid_taxyr_card_lline
               combination_of_columns:
                 - parid
                 - taxyr

diff --git a/dbt/models/iasworld/schema/iasworld.addrindx.yml b/dbt/models/iasworld/schema/iasworld.addrindx.yml
@@ -3,6 +3,7 @@ sources:
     loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0')
     tags:
       - load_auto
+      - test_qc_iasworld
 
     tables:
       - name: addrindx