From 3551103f2be5e891673150dbf3c4815ec01551bb Mon Sep 17 00:00:00 2001 From: Dan Snow <31494343+dfsnow@users.noreply.github.com> Date: Tue, 21 Nov 2023 14:30:30 -0600 Subject: [PATCH] Add iasWorld dbt tests and namespace all tests (#236) * Initial sale view tests * Add ADDN tests * Add ADPRVAL tests * Add ASMT_ALL tests * Revert "Initial sale view tests" This reverts commit aa1ad673fd18e4d71dad96aa48546d4212b92f10. * Add new test macro for mismatched rows * Test mismatched classes in ASMT_ALL vs PARDAT * Update class comparison tests * Improve class match testing * Add more tests and rename all iasworld tests * Update test names and formatting * Add more DWELDAT char tests * Finish DWELDAT tests * Add HTPAR tests * Add LAND table tests * Add LEGDAT and address tests * Add more LEGDAT address checks * Add OWNDAT tests * Add PARDAT tests * Add OBY tests * Add SALES tests * Add tags for FP QC * Update test schemas * Truncate test name * Truncate test name * Update test name * Replace redundant conditions with YAML anchors * Fix missing test namespace * Drop extra linebreaks * Add test_qc tag to all iasWorld tables * Exclude test_qc tests from build_and_test_dbt workflow * Add note about trigger to test_dbt_models --- .github/workflows/build_and_test_dbt.yaml | 6 +- .github/workflows/test_dbt_models.yaml | 3 + .../schema/default.vw_card_res_char.yml | 2 +- .../default/schema/default.vw_pin_address.yml | 4 +- .../default/schema/default.vw_pin_appeal.yml | 4 +- .../schema/default.vw_pin_condo_char.yml | 2 +- .../default/schema/default.vw_pin_history.yml | 2 +- .../default/schema/default.vw_pin_sale.yml | 4 +- .../schema/default.vw_pin_universe.yml | 6 +- .../default/schema/default.vw_pin_value.yml | 2 +- dbt/models/iasworld/docs.md | 5 + .../iasworld/schema/iasworld.aasysjur.yml | 1 + dbt/models/iasworld/schema/iasworld.addn.yml | 60 ++- .../iasworld/schema/iasworld.addrindx.yml | 1 + .../iasworld/schema/iasworld.aprval.yml | 58 ++- .../iasworld/schema/iasworld.asmt_all.yml | 435 +++++++++++++++++- .../iasworld/schema/iasworld.asmt_hist.yml | 1 + dbt/models/iasworld/schema/iasworld.cname.yml | 1 + .../iasworld/schema/iasworld.comdat.yml | 89 +++- dbt/models/iasworld/schema/iasworld.comnt.yml | 1 + dbt/models/iasworld/schema/iasworld.cvleg.yml | 1 + dbt/models/iasworld/schema/iasworld.cvown.yml | 1 + .../iasworld/schema/iasworld.cvtran.yml | 1 + dbt/models/iasworld/schema/iasworld.dedit.yml | 1 + .../iasworld/schema/iasworld.dweldat.yml | 360 ++++++++++++++- dbt/models/iasworld/schema/iasworld.enter.yml | 1 + .../iasworld/schema/iasworld.exadmn.yml | 1 + dbt/models/iasworld/schema/iasworld.exapp.yml | 1 + .../iasworld/schema/iasworld.excode.yml | 1 + dbt/models/iasworld/schema/iasworld.exdet.yml | 1 + .../iasworld/schema/iasworld.htagnt.yml | 3 +- .../iasworld/schema/iasworld.htdates.yml | 1 + dbt/models/iasworld/schema/iasworld.htpar.yml | 61 ++- dbt/models/iasworld/schema/iasworld.land.yml | 94 +++- .../iasworld/schema/iasworld.legdat.yml | 89 +++- dbt/models/iasworld/schema/iasworld.lpmod.yml | 1 + .../iasworld/schema/iasworld.lpnbhd.yml | 1 + dbt/models/iasworld/schema/iasworld.oby.yml | 60 ++- .../iasworld/schema/iasworld.owndat.yml | 75 ++- .../iasworld/schema/iasworld.pardat.yml | 44 +- .../iasworld/schema/iasworld.permit.yml | 1 + dbt/models/iasworld/schema/iasworld.rcoby.yml | 1 + dbt/models/iasworld/schema/iasworld.sales.yml | 41 +- .../iasworld/schema/iasworld.splcom.yml | 1 + .../iasworld/schema/iasworld.valclass.yml | 1 + dbt/models/location/schema.yml | 6 +- dbt/models/reporting/schema.yml | 120 ++--- dbt/tests/generic/test_column_length.sql | 10 +- dbt/tests/generic/test_expression_is_true.sql | 4 +- .../generic/test_no_extra_whitespace.sql | 32 +- .../test_row_values_match_after_join.sql | 30 ++ 51 files changed, 1610 insertions(+), 122 deletions(-) create mode 100644 dbt/tests/generic/test_row_values_match_after_join.sql diff --git a/.github/workflows/build_and_test_dbt.yaml b/.github/workflows/build_and_test_dbt.yaml index 4a0a53f0b..706d9d241 100644 --- a/.github/workflows/build_and_test_dbt.yaml +++ b/.github/workflows/build_and_test_dbt.yaml @@ -79,14 +79,14 @@ jobs: if [[ $MODIFIED_RESOURCES_ONLY == 'true' ]]; then if [[ $MANUALLY_DISPATCHED == 'true' ]]; then echo "Running tests on manually selected resources" - dbt test -t "$TARGET" -s ${{ inputs.models }} --defer --state "$STATE_DIR" + dbt test -t "$TARGET" -s ${{ inputs.models }} --exclude "tag:test_qc*" --defer --state "$STATE_DIR" else echo "Running tests on modified/new resources only" - dbt test -t "$TARGET" -s state:modified state:new --defer --state "$STATE_DIR" + dbt test -t "$TARGET" -s state:modified state:new --exclude "tag:test_qc*" --defer --state "$STATE_DIR" fi else echo "Running tests on all resources" - dbt test -t "$TARGET" + dbt test -t "$TARGET" --exclude "tag:test_qc*" fi working-directory: ${{ env.PROJECT_DIR }} shell: bash diff --git a/.github/workflows/test_dbt_models.yaml b/.github/workflows/test_dbt_models.yaml index aceb6eb7f..9725ba63b 100644 --- a/.github/workflows/test_dbt_models.yaml +++ b/.github/workflows/test_dbt_models.yaml @@ -1,5 +1,8 @@ name: test-dbt-models +# This workflow is not scheduled or run on PRs because it is manually triggered +# by the Data Department's sqoop data extraction process upon completion. See +# https://github.com/ccao-data/service-sqoop-iasworld on: workflow_dispatch jobs: diff --git a/dbt/models/default/schema/default.vw_card_res_char.yml b/dbt/models/default/schema/default.vw_card_res_char.yml index 33036ddc0..2d6db405a 100644 --- a/dbt/models/default/schema/default.vw_card_res_char.yml +++ b/dbt/models/default/schema/default.vw_card_res_char.yml @@ -109,7 +109,7 @@ models: tests: - unique_combination_of_columns: - name: vw_card_res_char_unique_by_pin_card_and_year + name: default_vw_card_res_char_unique_by_pin_card_and_year combination_of_columns: - pin - year diff --git a/dbt/models/default/schema/default.vw_pin_address.yml b/dbt/models/default/schema/default.vw_pin_address.yml index ea0971bec..00c299d6b 100644 --- a/dbt/models/default/schema/default.vw_pin_address.yml +++ b/dbt/models/default/schema/default.vw_pin_address.yml @@ -51,7 +51,7 @@ models: tests: # No superfluous whitespace - no_extra_whitespace: - name: vw_pin_address_no_extra_whitespace + name: default_vw_pin_address_no_extra_whitespace column_names: - prop_address_full - prop_address_city_name @@ -66,7 +66,7 @@ models: config: error_if: ">900000" - unique_combination_of_columns: - name: vw_pin_address_unique_by_14_digit_pin_and_year + name: default_vw_pin_address_unique_by_14_digit_pin_and_year combination_of_columns: - pin - year diff --git a/dbt/models/default/schema/default.vw_pin_appeal.yml b/dbt/models/default/schema/default.vw_pin_appeal.yml index e1ded24bf..823815605 100644 --- a/dbt/models/default/schema/default.vw_pin_appeal.yml +++ b/dbt/models/default/schema/default.vw_pin_appeal.yml @@ -45,7 +45,7 @@ models: tests: # If there is a `change`, the values should reflect this - expression_is_true: - name: vw_pin_appeal_change_matches_appeal_outcome + name: default_vw_pin_appeal_change_matches_appeal_outcome expression: >- {% set vars = [ ('mailed_bldg', 'certified_bldg'), @@ -94,7 +94,7 @@ models: error_if: ">266719" # `change` should be an enum - expression_is_true: - name: vw_pin_appeal_no_unexpected_change_values + name: default_vw_pin_appeal_no_unexpected_change_values expression: change is null or change in ('change', 'no change') select_columns: - pin diff --git a/dbt/models/default/schema/default.vw_pin_condo_char.yml b/dbt/models/default/schema/default.vw_pin_condo_char.yml index 3f9bdc284..6bc48efac 100644 --- a/dbt/models/default/schema/default.vw_pin_condo_char.yml +++ b/dbt/models/default/schema/default.vw_pin_condo_char.yml @@ -83,7 +83,7 @@ models: tests: - unique_combination_of_columns: - name: vw_pin_condo_char_unique_by_14_digit_pin_and_year + name: default_vw_pin_condo_char_unique_by_14_digit_pin_and_year combination_of_columns: - pin - year diff --git a/dbt/models/default/schema/default.vw_pin_history.yml b/dbt/models/default/schema/default.vw_pin_history.yml index 8e7907c2e..cd84dec4f 100644 --- a/dbt/models/default/schema/default.vw_pin_history.yml +++ b/dbt/models/default/schema/default.vw_pin_history.yml @@ -70,7 +70,7 @@ models: tests: - unique_combination_of_columns: - name: vw_pin_history_unique_by_14_digit_pin_and_year + name: default_vw_pin_history_unique_by_14_digit_pin_and_year combination_of_columns: - pin - year diff --git a/dbt/models/default/schema/default.vw_pin_sale.yml b/dbt/models/default/schema/default.vw_pin_sale.yml index efda8a4a9..b477d76b5 100644 --- a/dbt/models/default/schema/default.vw_pin_sale.yml +++ b/dbt/models/default/schema/default.vw_pin_sale.yml @@ -57,7 +57,7 @@ models: tests: # Number of sales for a given time period isn't suspicious - unique_combination_of_columns: - name: vw_pin_sale_reasonable_number_of_sales_per_year + name: default_vw_pin_sale_reasonable_number_of_sales_per_year combination_of_columns: - pin - year @@ -66,7 +66,7 @@ models: error_if: ">4032" # No sales for same price/pin within 12 months - unique_combination_of_columns: - name: vw_pin_sale_unique_price_pin_and_year + name: default_vw_pin_sale_unique_price_pin_and_year combination_of_columns: - pin - year diff --git a/dbt/models/default/schema/default.vw_pin_universe.yml b/dbt/models/default/schema/default.vw_pin_universe.yml index b373a270f..05697db32 100644 --- a/dbt/models/default/schema/default.vw_pin_universe.yml +++ b/dbt/models/default/schema/default.vw_pin_universe.yml @@ -233,18 +233,18 @@ models: tests: # Number of classes is consistent over time - count_is_consistent: - name: vw_pin_universe_class_count_is_consistent_by_year + name: default_vw_pin_universe_class_count_is_consistent_by_year group_column: year count_column: class config: error_if: ">24" # Number of towns is consistent over time - count_is_consistent: - name: vw_pin_universe_township_code_count_is_consistent_by_year + name: default_vw_pin_universe_town_count_is_consistent_by_year group_column: year count_column: township_code - unique_combination_of_columns: - name: vw_pin_universe_unique_by_14_digit_pin_and_year + name: default_vw_pin_universe_unique_by_14_digit_pin_and_year combination_of_columns: - pin - year diff --git a/dbt/models/default/schema/default.vw_pin_value.yml b/dbt/models/default/schema/default.vw_pin_value.yml index d68d678fd..d542be7bd 100644 --- a/dbt/models/default/schema/default.vw_pin_value.yml +++ b/dbt/models/default/schema/default.vw_pin_value.yml @@ -42,7 +42,7 @@ models: tests: - unique_combination_of_columns: - name: vw_pin_value_unique_by_14_digit_pin_and_year + name: default_vw_pin_value_unique_by_14_digit_pin_and_year combination_of_columns: - pin - year diff --git a/dbt/models/iasworld/docs.md b/dbt/models/iasworld/docs.md index e49988865..8abde623a 100644 --- a/dbt/models/iasworld/docs.md +++ b/dbt/models/iasworld/docs.md @@ -38,6 +38,11 @@ Main assessment table, union of `iasworld.asmt` and `iasworld.asmt_hist`. Provides the latest assessed value for each PIN and used heavily in Data Department views. +### Nuance + +- Unlike most `iasworld` tables, this table _does not_ use `cur = 'Y'` to + identify the most recent record (it uses `procname` instead). + **Primary Key**: `jur`, `rolltype`, `valclass`, `valyear`, `distcode`, `seq` `taxyr`, `parid`, `card` {% enddocs %} diff --git a/dbt/models/iasworld/schema/iasworld.aasysjur.yml b/dbt/models/iasworld/schema/iasworld.aasysjur.yml index 51d33aa34..011dc8c9f 100644 --- a/dbt/models/iasworld/schema/iasworld.aasysjur.yml +++ b/dbt/models/iasworld/schema/iasworld.aasysjur.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: aasysjur diff --git a/dbt/models/iasworld/schema/iasworld.addn.yml b/dbt/models/iasworld/schema/iasworld.addn.yml index 5b6bb67d3..ff7720032 100644 --- a/dbt/models/iasworld/schema/iasworld.addn.yml +++ b/dbt/models/iasworld/schema/iasworld.addn.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld - type_res tables: @@ -12,6 +13,13 @@ sources: columns: - name: area description: '{{ doc("column_area") }}' + tests: + - dbt_utils.accepted_range: + name: iasworld_addn_area_between_0_and_1.5M + min_value: 0 + max_value: 1500000 + config: + where: cur = 'Y' AND deactivat IS NULL - name: area_sqm description: Area of this line in meters - name: areaid @@ -20,20 +28,48 @@ sources: description: '{{ doc("column_areasum") }}' - name: card description: '{{ doc("column_card") }}' + tests: + - not_null: + name: iasworld_addn_card_not_null + - dbt_utils.accepted_range: + name: iasworld_addn_card_gte_1 + min_value: 1 - name: cdu description: '{{ doc("column_cdu") }}' - name: chgrsn description: '{{ doc("column_chgrsn") }}' - name: class description: '{{ doc("shared_column_class") }}' + tests: + - relationships: + name: iasworld_addn_class_in_ccao_class_dict + to: source('ccao', 'class_dict') + field: class_code + config: + where: | + taxyr >= '2022' + AND class != 'EX' + AND cur = 'Y' + AND deactivat IS NULL - name: convaddn description: Converted addition total - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_addn_cur_in_accepted_values + values: ['Y', 'D'] - name: deactivat description: '{{ doc("column_deactivat") }}' - name: depr description: Percent good from tables (based on age+cdu) + tests: + - dbt_utils.accepted_range: + name: iasworld_addn_depr_between_0_and_100 + min_value: 0 + max_value: 100 + config: + where: cur = 'Y' AND deactivat IS NULL - name: eff_area description: Support value per line - name: effageovr @@ -64,6 +100,9 @@ sources: description: '{{ doc("column_jur") }}' - name: lline description: '{{ doc("shared_column_lline") }}' + tests: + - not_null: + name: iasworld_addn_lline_not_null - name: lower description: Lower level addition code - name: mktadj @@ -82,6 +121,13 @@ sources: description: '{{ doc("column_newconpct") }}' - name: parid description: '{{ doc("shared_column_pin") }}' + tests: + - not_null: + name: iasworld_addn_parid_not_null + - relationships: + name: iasworld_addn_parid_in_pardat_parid + to: source('iasworld', 'pardat') + field: parid - name: pctcomp description: '{{ doc("column_pctcomp") }}' - name: prodamage @@ -126,10 +172,21 @@ sources: description: Second floor addition code - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_addn_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr + - card + - lline - name: status description: '{{ doc("column_status") }}' - name: taxyr description: '{{ doc("shared_column_year") }}' + tests: + - not_null: + name: iasworld_addn_taxyr_not_null - name: third description: Third floor addition code - name: trans_id @@ -152,9 +209,10 @@ sources: description: '{{ doc("column_whocalc") }}' - name: yrblt description: '{{ doc("column_yrblt") }}' + tests: - unique_combination_of_columns: - name: addn_unique_by_parid_taxyr_card_lline + name: iasworld_addn_unique_by_parid_taxyr_card_lline combination_of_columns: - parid - taxyr diff --git a/dbt/models/iasworld/schema/iasworld.addrindx.yml b/dbt/models/iasworld/schema/iasworld.addrindx.yml index d814b6f7c..788116ff0 100644 --- a/dbt/models/iasworld/schema/iasworld.addrindx.yml +++ b/dbt/models/iasworld/schema/iasworld.addrindx.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: addrindx diff --git a/dbt/models/iasworld/schema/iasworld.aprval.yml b/dbt/models/iasworld/schema/iasworld.aprval.yml index 7fde5a05d..0c188eb14 100644 --- a/dbt/models/iasworld/schema/iasworld.aprval.yml +++ b/dbt/models/iasworld/schema/iasworld.aprval.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: aprval @@ -17,12 +18,35 @@ sources: description: Applied market value - name: aprbldg description: Appraised building value + tests: + - dbt_utils.accepted_range: + name: iasworld_aprval_aprbldg_between_0_and_1B + min_value: 0 + max_value: 1000000000 + config: + where: | + taxyr >= '2021' + AND cur = 'Y' + AND deactivat IS NULL - name: aprdate description: '`APRTOT` calc date' - name: aprland description: Appraised land value + tests: + - dbt_utils.accepted_range: + name: iasworld_aprval_aprland_between_0_and_1B + min_value: 0 + max_value: 1000000000 + config: &unique-conditions + where: cur = 'Y' AND deactivat IS NULL - name: aprtot description: '{{ doc("column_aprtot") }}' + tests: + - dbt_utils.accepted_range: + name: iasworld_aprval_aprtot_between_0_and_2B + min_value: 0 + max_value: 2000000000 + config: *unique-conditions - name: areasum description: '{{ doc("column_areasum") }}' - name: assmkt @@ -31,6 +55,12 @@ sources: description: Appraisal unit number - name: bldgval description: Building value + tests: + - dbt_utils.accepted_range: + name: iasworld_aprval_bldgval_between_0_and_1B + min_value: 0 + max_value: 1000000000 + config: *unique-conditions - name: comincval description: Commercial income value - name: commktsf @@ -65,6 +95,10 @@ sources: description: Total cost value - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_aprval_cur_in_accepted_values + values: ['Y', 'D'] - name: deactivat description: '{{ doc("column_deactivat") }}' - name: dwelval @@ -107,6 +141,12 @@ sources: description: Land allocation percent - name: landval description: Total land cost value + tests: + - dbt_utils.accepted_range: + name: iasworld_aprval_landval_between_0_and_1B + min_value: 0 + max_value: 1000000000 + config: *unique-conditions - name: lastupd description: Date of last change to a value field - name: mandate @@ -135,6 +175,13 @@ sources: description: '{{ doc("column_ovrmraval") }}' - name: parid description: '{{ doc("shared_column_pin") }}' + tests: + - not_null: + name: iasworld_aprval_parid_not_null + - relationships: + name: iasworld_aprval_parid_in_pardat_parid + to: source('iasworld', 'pardat') + field: parid - name: posttype description: Assessment posting type - name: ppcomval @@ -177,6 +224,12 @@ sources: description: Selected sale price - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_aprval_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr - name: spcflg description: Special processing flag - name: splitno @@ -189,6 +242,9 @@ sources: description: '{{ doc("column_status") }}' - name: taxyr description: '{{ doc("shared_column_year") }}' + tests: + - not_null: + name: iasworld_aprval_taxyr_not_null - name: tiebackbldg description: Sum of building value for all children in the income tieback group - name: tiebackland @@ -214,7 +270,7 @@ sources: tests: - unique_combination_of_columns: - name: aprval_unique_by_parid_taxyr + name: iasworld_aprval_unique_by_parid_taxyr combination_of_columns: - parid - taxyr diff --git a/dbt/models/iasworld/schema/iasworld.asmt_all.yml b/dbt/models/iasworld/schema/iasworld.asmt_all.yml index dda6fb551..10235335b 100644 --- a/dbt/models/iasworld/schema/iasworld.asmt_all.yml +++ b/dbt/models/iasworld/schema/iasworld.asmt_all.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: asmt_all @@ -15,8 +16,25 @@ sources: columns: - name: class description: '{{ doc("shared_column_class") }}' + tests: + - relationships: + name: iasworld_asmt_all_class_in_ccao_class_dict + to: source('ccao', 'class_dict') + field: class_code + config: + where: | + taxyr >= '2022' + AND class NOT IN ('EX', 'RR', '999') + AND NOT REGEXP_LIKE(class, '[0-9]{3}[A|B]') + AND rolltype != 'RR' + AND deactivat IS NULL + AND valclass IS NULL - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_asmt_all_cur_in_accepted_values + values: ['Y', 'D', 'N'] - name: deactivat description: '{{ doc("column_deactivat") }}' - name: distcode @@ -35,16 +53,41 @@ sources: description: Override value for taxes - name: parid description: '{{ doc("shared_column_pin") }}' + tests: + - not_null: + name: iasworld_asmt_all_parid_not_null + - relationships: + name: iasworld_asmt_all_parid_in_pardat_parid + to: source('iasworld', 'pardat') + field: parid - name: procdate description: '{{ doc("column_procdate") }}' - name: procname description: '{{ doc("column_procname") }}' + tests: + - accepted_values: + name: iasworld_asmt_all_procname_in_accepted_values + values: ['CCAOVALUE', 'CCAOFINAL', 'BORVALUE', 'CONVERT'] - name: reascd description: Reason code for change in assessed value - name: rolltype description: '{{ doc("column_rolltype") }}' - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_asmt_all_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr + - procname + config: + where: &unique-conditions | + rolltype != 'RR' + AND procname IN ('CCAOVALUE', 'CCAOFINAL', 'BORVALUE') + AND deactivat IS NULL + AND valclass IS NULL + error_if: ">15300" - name: splitno description: '{{ doc("column_splitno") }}' - name: status @@ -53,12 +96,381 @@ sources: description: Flag to indicate assessment record used in tax calculation - name: taxyr description: '{{ doc("shared_column_year") }}' + tests: + - not_null: + name: iasworld_asmt_all_taxyr_not_null - name: tottax description: Total tax - name: trans_id description: '{{ doc("column_trans_id") }}' - name: upd_status description: '{{ doc("column_upd_status") }}' + - name: val01 + # Test that all val* columns are not negative. Pulled from + # the following Inquire queries via Will C: + # - FP Checklist - Negative ASMT Values + # - FP Checklist - Non-EX, RR parcels with 0 land value + # - FP Checklist - Non-EX, RR parcels with 0 value + tests: + - dbt_utils.accepted_range: &test-non-negative + name: iasworld_asmt_all_val01_gte_0 + min_value: 0 + config: + where: &unique-conditions-and-recent | + taxyr >= '2023' + AND rolltype != 'RR' + AND procname IN ('CCAOVALUE', 'CCAOFINAL', 'BORVALUE') + AND deactivat IS NULL + AND valclass IS NULL + tags: + - test_qc_valuations_fp + - name: val02 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val02_gte_0 + <<: *test-non-negative + - name: val03 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val03_gte_0 + <<: *test-non-negative + - name: val04 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val04_gte_0 + <<: *test-non-negative + - name: val05 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val05_gte_0 + <<: *test-non-negative + - name: val06 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val06_gte_0 + <<: *test-non-negative + - name: val07 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val07_gte_0 + <<: *test-non-negative + - name: val08 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val08_gte_0 + <<: *test-non-negative + - name: val09 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val09_gte_0 + <<: *test-non-negative + - name: val10 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val10_gte_0 + <<: *test-non-negative + - name: val11 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val11_gte_0 + <<: *test-non-negative + - name: val12 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val12_gte_0 + <<: *test-non-negative + - name: val13 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val13_gte_0 + <<: *test-non-negative + - name: val14 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val14_gte_0 + <<: *test-non-negative + - name: val15 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val15_gte_0 + <<: *test-non-negative + - name: val16 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val16_gte_0 + <<: *test-non-negative + - name: val17 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val17_gte_0 + <<: *test-non-negative + - name: val18 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val18_gte_0 + <<: *test-non-negative + - name: val19 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val19_gte_0 + <<: *test-non-negative + - name: val20 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val20_gte_0 + <<: *test-non-negative + - name: val21 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val21_gte_0 + <<: *test-non-negative + - name: val22 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val22_gte_0 + <<: *test-non-negative + - name: val23 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val23_gte_0 + <<: *test-non-negative + - name: val24 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val24_gte_0 + <<: *test-non-negative + - name: val25 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val25_gte_0 + <<: *test-non-negative + - name: val26 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val26_gte_0 + <<: *test-non-negative + - name: val27 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val27_gte_0 + <<: *test-non-negative + - name: val28 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val28_gte_0 + <<: *test-non-negative + - name: val29 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val29_gte_0 + <<: *test-non-negative + - name: val30 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val30_gte_0 + <<: *test-non-negative + - name: val31 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val31_gte_0 + <<: *test-non-negative + - name: val32 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val32_gte_0 + <<: *test-non-negative + - name: val33 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val33_gte_0 + <<: *test-non-negative + - name: val34 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val34_gte_0 + <<: *test-non-negative + - name: val35 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val35_gte_0 + <<: *test-non-negative + - name: val36 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val36_gte_0 + <<: *test-non-negative + - name: val37 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val37_gte_0 + <<: *test-non-negative + - name: val38 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val38_gte_0 + <<: *test-non-negative + - name: val39 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val39_gte_0 + <<: *test-non-negative + - name: val40 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val40_gte_0 + <<: *test-non-negative + - name: val41 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val41_gte_0 + <<: *test-non-negative + - name: val42 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val42_gte_0 + <<: *test-non-negative + - name: val43 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val43_gte_0 + <<: *test-non-negative + - name: val44 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val44_gte_0 + <<: *test-non-negative + - name: val45 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val45_gte_0 + <<: *test-non-negative + - name: val46 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val46_gte_0 + <<: *test-non-negative + - name: val47 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val47_gte_0 + <<: *test-non-negative + - name: val48 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val48_gte_0 + <<: *test-non-negative + - name: val49 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val49_gte_0 + <<: *test-non-negative + - name: val50 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val50_gte_0 + <<: *test-non-negative + - name: val51 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val51_gte_0 + <<: *test-non-negative + - name: val52 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val52_gte_0 + <<: *test-non-negative + - name: val53 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val53_gte_0 + <<: *test-non-negative + - name: val54 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val54_gte_0 + <<: *test-non-negative + - name: val55 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val55_gte_0 + <<: *test-non-negative + - name: val56 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val56_gte_0 + <<: *test-non-negative + - name: val57 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val57_gte_0 + <<: *test-non-negative + - name: val58 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val58_gte_0 + <<: *test-non-negative + - name: val59 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val59_gte_0 + <<: *test-non-negative + - name: val60 + tests: + - dbt_utils.accepted_range: + name: iasworld_asmt_all_val60_gte_0 + <<: *test-non-negative + # Also from FP Checklist - Negative ASMT Values AND the view + # definition used for vw_pin_value + - name: valapr1 + tests: + - not_null: &test-val-non-null + name: iasworld_asmt_all_valapr1_not_null + config: + where: *unique-conditions-and-recent + error_if: ">702" + - dbt_utils.accepted_range: + name: iasworld_asmt_all_valapr1_gte_0 + <<: *test-non-negative + - name: valapr2 + tests: + - not_null: + name: iasworld_asmt_all_valapr2_not_null + <<: *test-val-non-null + - dbt_utils.accepted_range: + name: iasworld_asmt_all_valapr2_gte_0 + <<: *test-non-negative + - name: valapr3 + tests: + - not_null: + name: iasworld_asmt_all_valapr3_not_null + <<: *test-val-non-null + - dbt_utils.accepted_range: + name: iasworld_asmt_all_valapr3_gte_0 + <<: *test-non-negative + - name: valasm1 + tests: + - not_null: + name: iasworld_asmt_all_valasm1_not_null + <<: *test-val-non-null + - dbt_utils.accepted_range: + name: iasworld_asmt_all_valasm1_gte_0 + <<: *test-non-negative + - name: valasm2 + tests: + - not_null: + name: iasworld_asmt_all_valasm2_not_null + <<: *test-val-non-null + - dbt_utils.accepted_range: + name: iasworld_asmt_all_valasm2_gte_0 + <<: *test-non-negative + - name: valasm3 + tests: + - not_null: + name: iasworld_asmt_all_valasm3_not_null + <<: *test-val-non-null + - dbt_utils.accepted_range: + name: iasworld_asmt_all_valasm3_gte_0 + <<: *test-non-negative - name: valclass description: '{{ doc("column_valclass") }}' - name: valtax @@ -75,16 +487,25 @@ sources: description: '{{ doc("column_whocalc") }}' tests: + # Note that this test is NOT actually the unique primary key of + # this table, since there doesn't seem to BE a unique combination + # of identifying columns - unique_combination_of_columns: - name: asmt_all_unique_by_parid_procname_and_taxyr + name: iasworld_asmt_all_unique_by_parid_procname_and_taxyr combination_of_columns: - parid - procname - taxyr config: - where: >- - cur = 'Y' and - deactivat is null and - procname in ('CCAOVALUE', 'CCAOFINAL', 'BORVALUE') and - valclass is null - error_if: ">125" + where: *unique-conditions + error_if: ">15300" + - row_values_match_after_join: + name: iasworld_asmt_all_class_matches_pardat_class + column: substr(model.class, 1, 3) + external_model: source('iasworld', 'pardat') + external_column: substr(external_model.class, 1, 3) + join_columns: + - parid + - taxyr + config: + where: *unique-conditions diff --git a/dbt/models/iasworld/schema/iasworld.asmt_hist.yml b/dbt/models/iasworld/schema/iasworld.asmt_hist.yml index 4d47e3c32..8bfd95d7a 100644 --- a/dbt/models/iasworld/schema/iasworld.asmt_hist.yml +++ b/dbt/models/iasworld/schema/iasworld.asmt_hist.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: asmt_hist diff --git a/dbt/models/iasworld/schema/iasworld.cname.yml b/dbt/models/iasworld/schema/iasworld.cname.yml index af45e3137..9d21a0a7f 100644 --- a/dbt/models/iasworld/schema/iasworld.cname.yml +++ b/dbt/models/iasworld/schema/iasworld.cname.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: cname diff --git a/dbt/models/iasworld/schema/iasworld.comdat.yml b/dbt/models/iasworld/schema/iasworld.comdat.yml index 59531f424..91812c8b8 100644 --- a/dbt/models/iasworld/schema/iasworld.comdat.yml +++ b/dbt/models/iasworld/schema/iasworld.comdat.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld - type_ic tables: @@ -12,6 +13,13 @@ sources: columns: - name: adjfact description: '{{ doc("column_adjfact") }}' + tests: + - dbt_utils.accepted_range: + name: iasworld_comdat_adjfact_between_0_and_1 + min_value: 0 + max_value: 1 + config: + where: cur = 'Y' AND deactivat IS NULL - name: areasum description: '{{ doc("column_areasum") }}' - name: bld_modelid @@ -22,6 +30,13 @@ sources: description: Gross building value - name: bldgval description: Building value + tests: + - dbt_utils.accepted_range: + name: iasworld_comdat_bldgval_between_0_and_1B + min_value: 0 + max_value: 1000000000 + config: + where: cur = 'Y' AND deactivat IS NULL - name: bldnum description: Building number - name: busla @@ -30,22 +45,58 @@ sources: description: Exclude this record from income calculations - name: card description: '{{ doc("column_card") }}' + tests: + - not_null: + name: iasworld_comdat_card_not_null + - dbt_utils.accepted_range: + name: iasworld_comdat_card_gte_1 + min_value: 1 - name: cdu description: '{{ doc("column_cdu") }}' - name: chgrsn description: '{{ doc("column_chgrsn") }}' - name: class description: '{{ doc("shared_column_class") }}' + tests: + - relationships: + name: iasworld_comdat_class_in_ccao_class_dict + to: source('ccao', 'class_dict') + field: class_code + config: + where: | + taxyr >= '2022' + AND cur = 'Y' + AND NOT REGEXP_LIKE(class, '[0-9]{3}[A|B]') + AND class NOT IN ('EX', 'RR') + AND deactivat IS NULL - name: convbldg description: '{{ doc("column_convbldg") }}' + tests: + - dbt_utils.accepted_range: + name: iasworld_comdat_convbldg_between_0_and_1B + min_value: 0 + max_value: 1000000000 + config: + where: cur = 'Y' AND deactivat IS NULL - name: cubicft description: Cubic feet - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_comdat_cur_in_accepted_values + values: ['Y', 'D'] - name: deactivat description: '{{ doc("column_deactivat") }}' - name: depr description: Percent good from tables + tests: + - dbt_utils.accepted_range: + name: iasworld_comdat_depr_between_0_and_1 + min_value: 0 + max_value: 1 + config: + where: cur = 'Y' AND deactivat IS NULL - name: deprt description: '{{ doc("column_deprt") }}' - name: effageovr @@ -56,6 +107,13 @@ sources: description: '{{ doc("column_excess") }}' - name: exmppct description: '{{ doc("column_exmppct") }}' + tests: + - dbt_utils.accepted_range: + name: iasworld_comdat_exmppct_between_0_and_100 + min_value: 0 + max_value: 100 + config: + where: cur = 'Y' AND deactivat IS NULL - name: exmpval description: '{{ doc("column_exmpval") }}' - name: gfact @@ -130,6 +188,13 @@ sources: description: '{{ doc("column_salekey") }}' - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_comdat_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr + - card - name: status description: '{{ doc("column_status") }}' - name: structure @@ -158,11 +223,33 @@ sources: description: '{{ doc("column_whocalc") }}' - name: yrblt description: '{{ doc("shared_column_char_yrblt") }}' + tests: + - not_null: + name: iasworld_comdat_yrblt_not_null + - dbt_utils.accepted_range: + name: iasworld_comdat_yrblt_between_1850_and_now + min_value: 1850 + max_value: cast(year(current_date) as decimal) + config: + where: cur = 'Y' AND deactivat IS NULL tests: - unique_combination_of_columns: - name: comdat_unique_by_parid_card_taxyr + name: iasworld_comdat_unique_by_parid_card_taxyr combination_of_columns: - parid - taxyr - card + - row_values_match_after_join: + name: iasworld_comdat_class_matches_pardat_class + column: class + external_model: source('iasworld', 'pardat') + external_column: class + join_columns: + - parid + - taxyr + config: + where: | + taxyr >= '2023' + AND cur = 'Y' + AND deactivat IS NULL diff --git a/dbt/models/iasworld/schema/iasworld.comnt.yml b/dbt/models/iasworld/schema/iasworld.comnt.yml index 76983677a..0fffbb634 100644 --- a/dbt/models/iasworld/schema/iasworld.comnt.yml +++ b/dbt/models/iasworld/schema/iasworld.comnt.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: comnt diff --git a/dbt/models/iasworld/schema/iasworld.cvleg.yml b/dbt/models/iasworld/schema/iasworld.cvleg.yml index da5844e7f..2f9b23ead 100644 --- a/dbt/models/iasworld/schema/iasworld.cvleg.yml +++ b/dbt/models/iasworld/schema/iasworld.cvleg.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: cvleg diff --git a/dbt/models/iasworld/schema/iasworld.cvown.yml b/dbt/models/iasworld/schema/iasworld.cvown.yml index db5c99b1c..f876a9a84 100644 --- a/dbt/models/iasworld/schema/iasworld.cvown.yml +++ b/dbt/models/iasworld/schema/iasworld.cvown.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: cvown diff --git a/dbt/models/iasworld/schema/iasworld.cvtran.yml b/dbt/models/iasworld/schema/iasworld.cvtran.yml index 3e35eb24a..06d20879a 100644 --- a/dbt/models/iasworld/schema/iasworld.cvtran.yml +++ b/dbt/models/iasworld/schema/iasworld.cvtran.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: cvtran diff --git a/dbt/models/iasworld/schema/iasworld.dedit.yml b/dbt/models/iasworld/schema/iasworld.dedit.yml index a0cc1da14..f52999e91 100644 --- a/dbt/models/iasworld/schema/iasworld.dedit.yml +++ b/dbt/models/iasworld/schema/iasworld.dedit.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: dedit diff --git a/dbt/models/iasworld/schema/iasworld.dweldat.yml b/dbt/models/iasworld/schema/iasworld.dweldat.yml index 43f323f0b..997bcae0e 100644 --- a/dbt/models/iasworld/schema/iasworld.dweldat.yml +++ b/dbt/models/iasworld/schema/iasworld.dweldat.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld - type_res tables: @@ -42,6 +43,20 @@ sources: description: '{{ doc("column_areasum") }}' - name: attic description: '{{ doc("shared_column_char_attic_type") }}' + tests: + - not_null: + name: iasworld_dweldat_attic_not_null + # Excludes non-regression classes, as defined in Assessor's + # online PDF of all class definitions + config: &unique-conditions + where: | + cur = 'Y' + AND deactivat IS NULL + AND class NOT IN ('201', '213', '218', '219', '220', '221', '224', '225', '236', '240', '241', '290', '294', '297') + - accepted_values: + name: iasworld_dweldat_attic_in_accepted_values + values: ['1', '2', '3'] + config: *unique-conditions - name: atticarea description: Attic area - name: atticval @@ -60,6 +75,14 @@ sources: description: Building use - name: bsmt description: '{{ doc("shared_column_char_bsmt") }}' + tests: + - not_null: + name: iasworld_dweldat_bsmt_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_bsmt_in_accepted_values + values: ['1', '2', '3', '4'] + config: *unique-conditions - name: bsmtarea description: Unfinished basement area - name: bsmtcar @@ -70,6 +93,12 @@ sources: description: Claculated effective year - name: card description: '{{ doc("column_card") }}' + tests: + - not_null: + name: iasworld_dweldat_card_not_null + - dbt_utils.accepted_range: + name: iasworld_dweldat_card_gte_1 + min_value: 1 - name: catharea description: Cathedral ceiling area - name: cathval @@ -86,6 +115,17 @@ sources: description: '{{ doc("column_chgrsn") }}' - name: class description: '{{ doc("shared_column_class") }}' + tests: + - relationships: + name: iasworld_dweldat_class_in_ccao_class_dict + to: source('ccao', 'class_dict') + field: class_code + config: + where: | + taxyr >= '2022' + AND class NOT IN ('EX', 'RR') + AND cur = 'Y' + AND deactivat IS NULL - name: cline description: Condominium line - name: cndadjval @@ -116,6 +156,10 @@ sources: description: '{{ doc("column_convbldg") }}' - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_dweldat_cur_in_accepted_values + values: ['Y', 'D'] - name: deactivat description: '{{ doc("column_deactivat") }}' - name: degrem @@ -154,6 +198,14 @@ sources: description: '{{ doc("column_exmpval") }}' - name: extwall description: '{{ doc("shared_column_char_ext_wall") }}' + tests: + - not_null: + name: iasworld_dweldat_extwall_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_extwall_in_accepted_values + values: ['1', '2', '3', '4'] + config: *unique-conditions - name: extwall1 description: Exterior wall code - name: extwall1pct @@ -170,6 +222,15 @@ sources: description: Number additional plumbing fixtures - name: fixbath description: '{{ doc("shared_column_char_fbath") }}' + tests: + - not_null: + name: iasworld_dweldat_fixbath_not_null + config: *unique-conditions + - dbt_utils.accepted_range: + name: iasworld_dweldat_fixbath_between_1_and_16 + min_value: 1 + max_value: 16 + config: *unique-conditions - name: fixbath1 description: Number of full bathrooms on the first floor - name: fixbath2 @@ -200,6 +261,15 @@ sources: description: Number full bathrooms in upper Floor - name: fixhalf description: '{{ doc("shared_column_char_hbath") }}' + tests: + - not_null: + name: iasworld_dweldat_fixhalf_not_null + config: *unique-conditions + - dbt_utils.accepted_range: + name: iasworld_dweldat_fixhalf_between_0_and_6 + min_value: 0 + max_value: 6 + config: *unique-conditions - name: fixhalf1 description: Number of half baths on the first floor - name: fixhalf2 @@ -260,6 +330,14 @@ sources: description: '{{ doc("column_grpadj") }}' - name: heat description: '{{ doc("shared_column_char_heat") }}' + tests: + - not_null: + name: iasworld_dweldat_heat_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_heat_in_accepted_values + values: ['1', '2', '3', '4'] + config: *unique-conditions - name: heatarea description: Heat area - name: heatsys @@ -344,6 +422,13 @@ sources: description: '{{ doc("column_ovrrcnld") }}' - name: parid description: '{{ doc("shared_column_pin") }}' + tests: + - not_null: + name: iasworld_dweldat_parid_not_null + - relationships: + name: iasworld_dweldat_parid_in_pardat_parid + to: source('iasworld', 'pardat') + field: parid - name: pctcomplete description: Percent of construction completed - name: plumbgrade @@ -402,6 +487,38 @@ sources: description: Residential cost model - name: rmbed description: '{{ doc("shared_column_char_beds") }}' + tests: + - not_null: + name: iasworld_dweldat_rmbed_not_null + config: *unique-conditions + - dbt_utils.accepted_range: + name: iasworld_dweldat_rmbed_sf_between_1_and_15 + min_value: 1 + max_value: 15 + config: + where: | + class NOT IN ('211', '212') + AND cur = 'Y' + AND deactivat IS NULL + - dbt_utils.accepted_range: + name: iasworld_dweldat_rmbed_mf_between_1_and_24 + min_value: 1 + max_value: 24 + config: + where: | + class IN ('211', '212') + AND cur = 'Y' + AND deactivat IS NULL + - expression_is_true: + name: iasworld_dweldat_rmbed_lte_rmtot + expression: <= rmtot + select_columns: + - parid + - taxyr + - card + - rmtot + - rmbed + config: *unique-conditions - name: rmbed1 description: Number of bedrooms on the first floor - name: rmbed2 @@ -462,12 +579,63 @@ sources: description: Number of other rooms in the basement - name: rmtot description: '{{ doc("shared_column_char_rooms") }}' + tests: + - not_null: + name: iasworld_dweldat_rmtot_not_null + config: *unique-conditions + - dbt_utils.accepted_range: + name: iasworld_dweldat_rmtot_sf_between_1_and_40 + min_value: 1 + max_value: 40 + config: + where: | + class NOT IN ('211', '212') + AND cur = 'Y' + AND deactivat IS NULL + - dbt_utils.accepted_range: + name: iasworld_dweldat_rmtot_sf_between_1_and_50 + min_value: 1 + max_value: 50 + config: + where: | + class IN ('211', '212') + AND cur = 'Y' + AND deactivat IS NULL - name: salekey description: '{{ doc("column_salekey") }}' - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_dweldat_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr + - card - name: sfla description: '{{ doc("shared_column_char_bldg_sf") }}' + tests: + - not_null: + name: iasworld_dweldat_sfla_not_null + config: *unique-conditions + - dbt_utils.accepted_range: + name: iasworld_dweldat_sfla_sf_between_1_and_10000 + min_value: 1 + max_value: 10000 + config: + where: | + class NOT IN ('208', '209', '211', '212', '236', '297') + AND cur = 'Y' + AND deactivat IS NULL + - dbt_utils.accepted_range: + name: iasworld_dweldat_sfla_mf_between_1_and_20000 + min_value: 1 + max_value: 20000 + config: + where: | + class IN ('208', '209', '211', '212') + AND cur = 'Y' + AND deactivat IS NULL - name: shfact description: Story height factor - name: sidpct @@ -486,6 +654,19 @@ sources: description: '{{ doc("column_status") }}' - name: stories description: '{{ doc("shared_column_char_type_resd") }}' + tests: + - not_null: + name: iasworld_dweldat_stories_not_null + config: *unique-conditions + - expression_is_true: + name: iasworld_dweldat_stories_in_accepted_values + expression: 'IN (1.00, 2.00, 3.00, 4.00, 5.00, 9.90)' + select_columns: + - parid + - taxyr + - card + - stories + config: *unique-conditions - name: style description: Architectural style - name: subtval @@ -494,6 +675,9 @@ sources: description: '{{ doc("column_suppress") }}' - name: taxyr description: '{{ doc("shared_column_year") }}' + tests: + - not_null: + name: iasworld_dweldat_taxyr_not_null - name: trans_id description: '{{ doc("column_trans_id") }}' - name: trimval @@ -524,38 +708,163 @@ sources: description: '{{ doc("shared_column_char_site") }}' - name: user3 description: '{{ doc("shared_column_char_renovation") }}' + tests: + - accepted_values: + name: iasworld_dweldat_char_renovation_in_accepted_values + values: ['0', '1'] + config: *unique-conditions - name: user4 description: '{{ doc("shared_column_char_tp_dsgn") }}' + tests: + - not_null: + name: iasworld_dweldat_char_tp_dsgn_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_tp_dsgn_in_accepted_values + values: ['0', '1', '2', '3'] + config: *unique-conditions - name: user5 description: '{{ doc("shared_column_char_tp_plan") }}' + tests: + - not_null: + name: iasworld_dweldat_char_tp_plan_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_tp_plan_in_accepted_values + values: ['0', '1', '2'] + config: *unique-conditions - name: user6 description: '{{ doc("shared_column_char_attic_fnsh") }}' + tests: + - not_null: + name: iasworld_dweldat_char_attic_fnsh_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_attic_fnsh_in_accepted_values + values: ['1', '2', '3'] + config: *unique-conditions - name: user7 description: '{{ doc("shared_column_char_air") }}' + tests: + - not_null: + name: iasworld_dweldat_char_air_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_air_accepted_values + values: ['1', '2'] + config: *unique-conditions - name: user12 description: '{{ doc("shared_column_char_bsmt_fin") }}' + tests: + - not_null: + name: iasworld_dweldat_char_bsmt_fin_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_bsmt_fin_accepted_values + values: ['1', '2', '3'] + config: *unique-conditions - name: user13 description: '{{ doc("shared_column_char_roof_cnst") }}' + tests: + - not_null: + name: iasworld_dweldat_char_roof_cnst_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_roof_cnst_accepted_values + values: ['1', '2', '3', '4', '5', '6'] + config: *unique-conditions - name: user14 description: '{{ doc("shared_column_char_apts") }}' + tests: + - not_null: + name: iasworld_dweldat_char_apts_not_null + config: + where: | + class IN ('211', '212') + AND cur = 'Y' + AND deactivat IS NULL + - accepted_values: + name: iasworld_dweldat_char_apts_accepted_values + values: ['1', '2', '3', '4', '5', '6'] + config: + where: | + class IN ('211', '212') + AND cur = 'Y' + AND deactivat IS NULL - name: user15 description: '{{ doc("shared_column_char_use") }}' + tests: + - not_null: + name: iasworld_dweldat_char_use_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_use_accepted_values + values: ['1', '2'] + config: *unique-conditions - name: user17 description: Age. Deprecated, use `yrblt` - name: user20 description: '{{ doc("shared_column_char_ncu") }}' + tests: + - not_null: + name: iasworld_dweldat_char_ncu_not_null + config: *unique-conditions - name: user24 description: '{{ doc("shared_column_card_proration_rate") }}' + tests: + - not_null: + name: iasworld_dweldat_card_proration_rate_not_null + config: *unique-conditions - name: user30 description: '{{ doc("shared_column_char_porch") }}' + tests: + - not_null: + name: iasworld_dweldat_char_porch_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_porch_accepted_values + values: ['0', '1', '2'] + config: *unique-conditions - name: user31 description: '{{ doc("shared_column_char_gar_att") }}' + tests: + - not_null: + name: iasworld_dweldat_char_gar_att_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_gar_att_accepted_values + values: ['1', '2'] + config: *unique-conditions - name: user32 description: '{{ doc("shared_column_char_gar_area") }}' + tests: + - not_null: + name: iasworld_dweldat_char_gar_area_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_gar_area_accepted_values + values: ['1', '2'] + config: *unique-conditions - name: user33 description: '{{ doc("shared_column_char_gar_size") }}' + tests: + - not_null: + name: iasworld_dweldat_char_gar_size_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_gar_size_accepted_values + values: ['1', '2', '3', '4', '5', '6', '7', '8'] + config: *unique-conditions - name: user34 description: '{{ doc("shared_column_char_gar_cnst") }}' + tests: + - not_null: + name: iasworld_dweldat_char_gar_cnst_not_null + config: *unique-conditions + - accepted_values: + name: iasworld_dweldat_char_gar_cnst_accepted_values + values: ['1', '2', '3', '4'] + config: *unique-conditions - name: useramt description: RCN value of additions - name: userfact @@ -568,6 +877,15 @@ sources: description: Wall height factor - name: wbfp_o description: '{{ doc("shared_column_char_frpl") }}' + tests: + - not_null: + name: iasworld_dweldat_char_frpl_not_null + config: *unique-conditions + - dbt_utils.accepted_range: + name: iasworld_dweldat_char_frpl_between_0_and_6 + min_value: 0 + max_value: 6 + config: *unique-conditions - name: wbfp_pf description: Wood burning fireplace - prefab - name: wbfp_pfx @@ -592,13 +910,53 @@ sources: description: Windows year - name: yrblt description: '{{ doc("shared_column_char_yrblt") }}' + tests: + - not_null: + name: iasworld_dweldat_yrblt_not_null + config: *unique-conditions + - dbt_utils.accepted_range: + name: iasworld_dweldat_yrblt_between_1850_and_now + min_value: 1850 + max_value: cast(year(current_date) as decimal) + config: *unique-conditions - name: yrremod description: Year remodeled tests: - unique_combination_of_columns: - name: dweldat_unique_by_parid_card_taxyr + name: iasworld_dweldat_unique_by_parid_card_taxyr combination_of_columns: - parid - taxyr - card + - row_values_match_after_join: + name: iasworld_dweldat_class_matches_pardat_class + column: class + external_model: source('iasworld', 'pardat') + external_column: class + join_columns: + - parid + - taxyr + config: + where: | + taxyr >= '2023' + AND cur = 'Y' + AND deactivat IS NULL + - expression_is_true: + name: iasworld_dweldat_card_proration_rate_between_0_and_100 + expression: 'CAST(user24 AS decimal) BETWEEN 0.00 AND 100.00' + select_columns: + - parid + - taxyr + - card + - user24 + config: *unique-conditions + - expression_is_true: + name: iasworld_dweldat_char_ncu_between_0_and_5 + expression: 'CAST(user20 AS decimal) BETWEEN 0 AND 5' + select_columns: + - parid + - taxyr + - card + - user20 + config: *unique-conditions diff --git a/dbt/models/iasworld/schema/iasworld.enter.yml b/dbt/models/iasworld/schema/iasworld.enter.yml index fbbc28876..610f2a758 100644 --- a/dbt/models/iasworld/schema/iasworld.enter.yml +++ b/dbt/models/iasworld/schema/iasworld.enter.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: enter diff --git a/dbt/models/iasworld/schema/iasworld.exadmn.yml b/dbt/models/iasworld/schema/iasworld.exadmn.yml index 2877d01fa..5958c98b8 100644 --- a/dbt/models/iasworld/schema/iasworld.exadmn.yml +++ b/dbt/models/iasworld/schema/iasworld.exadmn.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: exadmn diff --git a/dbt/models/iasworld/schema/iasworld.exapp.yml b/dbt/models/iasworld/schema/iasworld.exapp.yml index 4cec3e61e..c36ba4b81 100644 --- a/dbt/models/iasworld/schema/iasworld.exapp.yml +++ b/dbt/models/iasworld/schema/iasworld.exapp.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: exapp diff --git a/dbt/models/iasworld/schema/iasworld.excode.yml b/dbt/models/iasworld/schema/iasworld.excode.yml index 895981fef..54f97b603 100644 --- a/dbt/models/iasworld/schema/iasworld.excode.yml +++ b/dbt/models/iasworld/schema/iasworld.excode.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: excode diff --git a/dbt/models/iasworld/schema/iasworld.exdet.yml b/dbt/models/iasworld/schema/iasworld.exdet.yml index 12c677c30..9c51d5f90 100644 --- a/dbt/models/iasworld/schema/iasworld.exdet.yml +++ b/dbt/models/iasworld/schema/iasworld.exdet.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: exdet diff --git a/dbt/models/iasworld/schema/iasworld.htagnt.yml b/dbt/models/iasworld/schema/iasworld.htagnt.yml index cc46f0b4c..e7335df7f 100644 --- a/dbt/models/iasworld/schema/iasworld.htagnt.yml +++ b/dbt/models/iasworld/schema/iasworld.htagnt.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: htagnt @@ -118,6 +119,6 @@ sources: tests: - unique_combination_of_columns: - name: htagnt_unique_by_agent + name: iasworld_htagnt_unique_by_agent combination_of_columns: - agent diff --git a/dbt/models/iasworld/schema/iasworld.htdates.yml b/dbt/models/iasworld/schema/iasworld.htdates.yml index 305f1c784..716373cdb 100644 --- a/dbt/models/iasworld/schema/iasworld.htdates.yml +++ b/dbt/models/iasworld/schema/iasworld.htdates.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: htdates diff --git a/dbt/models/iasworld/schema/iasworld.htpar.yml b/dbt/models/iasworld/schema/iasworld.htpar.yml index b428fca34..778e9b5da 100644 --- a/dbt/models/iasworld/schema/iasworld.htpar.yml +++ b/dbt/models/iasworld/schema/iasworld.htpar.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: htpar @@ -65,6 +66,10 @@ sources: description: Attorney jurisdiction - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_htpar_cur_in_accepted_values + values: ['Y', 'D'] - name: curbldg description: Current building value - name: curland @@ -95,6 +100,14 @@ sources: description: Hearing reason code - name: hrstatus description: '{{ doc("shared_column_appeal_status") }}' + tests: + - accepted_values: + name: iasworld_htpar_hrstatus_in_accepted_values + values: ['C', 'O', 'P', 'X'] + config: &unique-conditions + where: | + cur = 'Y' + AND deactivat IS NULL - name: httimejur description: Schedule jurisdiction - name: iasw_id @@ -149,6 +162,13 @@ sources: description: Owner phone number - name: parid description: '{{ doc("shared_column_pin") }}' + tests: + - not_null: + name: iasworld_htpar_parid_not_null + - relationships: + name: iasworld_htpar_parid_in_pardat_parid + to: source('iasworld', 'pardat') + field: parid - name: petemail description: Petitioner email address - name: petfax @@ -203,6 +223,14 @@ sources: description: Schedule time - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_htpar_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr + - caseno + - subkey - name: status description: '{{ doc("column_status") }}' - name: stipbldg @@ -219,6 +247,9 @@ sources: description: Unique number to allow multiple hearings per level - name: taxyr description: '{{ doc("shared_column_year") }}' + tests: + - not_null: + name: iasworld_htpar_taxyr_not_null - name: trans_id description: '{{ doc("column_trans_id") }}' - name: txpyval @@ -227,6 +258,11 @@ sources: description: '{{ doc("column_upd_status") }}' - name: user38 description: '{{ doc("shared_column_appeal_type") }}' + tests: + - accepted_values: + name: iasworld_htpar_user38_in_accepted_values + values: ['RS', 'IC', 'CC', 'CE', 'CI', 'CO', 'CV', 'IM', 'LD', 'OM', 'RC'] + config: *unique-conditions - name: user89 description: '{{ doc("shared_column_reason_code") }}' - name: user100 @@ -243,12 +279,33 @@ sources: tests: - unique_combination_of_columns: - name: htpar_unique_by_parid_caseno_taxyr_subkey + name: iasworld_htpar_unique_by_parid_caseno_taxyr_subkey combination_of_columns: - parid - caseno - taxyr - subkey config: - where: cur = 'Y' and deactivat is null + <<: *unique-conditions error_if: ">2" + - no_extra_whitespace: + name: iasworld_htpar_address_columns_no_whitespace + column_names: + - own1 + # - addr1 # almost every value in this field has a trailing space + - addr2 + - addr3 + - cpaddr1 + - cpaddr2 + - cpaddr3 + allow_interior_space: true + config: *unique-conditions + - expression_is_true: + name: iasworld_htpar_cpaddr3_length_lte_5 + expression: | + LENGTH(cpaddr3) = 5 + OR (LENGTH(cpaddr3) = 10 AND cpaddr3 LIKE '%-%') + select_columns: + - parid + - taxyr + - cpaddr3 diff --git a/dbt/models/iasworld/schema/iasworld.land.yml b/dbt/models/iasworld/schema/iasworld.land.yml index c4f9279f5..9e6b5289c 100644 --- a/dbt/models/iasworld/schema/iasworld.land.yml +++ b/dbt/models/iasworld/schema/iasworld.land.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld - type_land tables: @@ -12,6 +13,10 @@ sources: columns: - name: acres description: '{{ doc("column_acres") }}' + tests: + - dbt_utils.expression_is_true: + name: iasworld_land_acres_is_sf_transformed + expression: '* 43560 BETWEEN sf - 5 AND sf + 5' - name: adjfact description: '{{ doc("column_adjfact") }}' - name: agflg @@ -36,6 +41,17 @@ sources: description: '{{ doc("column_chgrsn") }}' - name: class description: '{{ doc("shared_column_class") }}' + tests: + - relationships: + name: iasworld_land_class_in_ccao_class_dict + to: source('ccao', 'class_dict') + field: class_code + config: + where: | + cur = 'Y' + AND class NOT IN ('EX', 'RR', '999') + AND NOT REGEXP_LIKE(class, '[0-9]{3}[A|B]') + AND deactivat IS NULL - name: code description: Code - name: comres @@ -44,6 +60,10 @@ sources: description: Land value at conversion - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_land_cur_in_accepted_values + values: ['Y', 'D'] - name: deactivat description: '{{ doc("column_deactivat") }}' - name: depfact @@ -76,6 +96,15 @@ sources: description: 'Influence Code #2' - name: influ description: Influence percent (+/-) + tests: + - dbt_utils.accepted_range: + name: iasworld_land_influ_between_neg_100_and_pos_100 + min_value: -100 + max_value: 100 + config: + where: + cur = 'Y' + AND deactivat IS NULL - name: influ2 description: Influence percent (+/-) - name: jur @@ -86,6 +115,16 @@ sources: description: Land code model - name: lline description: '{{ doc("shared_column_lline") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_land_lline_is_sequential + group_by_columns: + - parid + - taxyr + config: + where: + cur = 'Y' + AND deactivat IS NULL - name: lmod description: Land location model number - name: locfact @@ -132,8 +171,25 @@ sources: description: Override price - name: parid description: '{{ doc("shared_column_pin") }}' + tests: + - not_null: + name: iasworld_land_parid_not_null + - relationships: + name: iasworld_land_parid_in_pardat_parid + to: source('iasworld', 'pardat') + field: parid - name: price description: Price + tests: + - dbt_utils.accepted_range: + name: iasworld_land_price_gte_1 + min_value: 1 + config: + where: + cur = 'Y' + AND taxyr >= '2023' + AND class NOT IN ('EX', 'RR', '999') + AND deactivat IS NULL - name: recnr description: '{{ doc("column_recnr") }}' - name: salekey @@ -142,8 +198,27 @@ sources: description: Schedule model number - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_land_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr + - lline + config: + where: + cur = 'Y' + AND deactivat IS NULL - name: sf description: '{{ doc("shared_column_char_land_sf") }}' + tests: + - dbt_utils.accepted_range: + name: iasworld_land_sf_gte_1 + min_value: 1 + config: + where: + cur = 'Y' + AND deactivat IS NULL - name: sfact description: S factor - name: sizefact @@ -164,6 +239,9 @@ sources: description: '{{ doc("column_suppress") }}' - name: taxyr description: '{{ doc("shared_column_year") }}' + tests: + - not_null: + name: iasworld_land_taxyr_not_null - name: topofact description: Topography Factor - name: trans_id @@ -189,8 +267,22 @@ sources: tests: - unique_combination_of_columns: - name: land_unique_by_parid_lline_taxyr + name: iasworld_land_unique_by_parid_lline_taxyr combination_of_columns: - parid - lline - taxyr + - row_values_match_after_join: + name: iasworld_land_class_matches_pardat_class + column: substr(model.class, 1, 1) + external_model: source('iasworld', 'pardat') + external_column: substr(external_model.class, 1, 1) + join_columns: + - parid + - taxyr + config: + where: | + taxyr >= '2022' + AND class NOT IN ('EX', 'RR') + AND cur = 'Y' + AND deactivat IS NULL diff --git a/dbt/models/iasworld/schema/iasworld.legdat.yml b/dbt/models/iasworld/schema/iasworld.legdat.yml index a970637f8..4bb6f9faa 100644 --- a/dbt/models/iasworld/schema/iasworld.legdat.yml +++ b/dbt/models/iasworld/schema/iasworld.legdat.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: legdat @@ -19,6 +20,14 @@ sources: description: '{{ doc("column_adradd") }}' - name: adrdir description: '{{ doc("shared_column_prop_address_street_dir") }}' + tests: + - accepted_values: + name: iasworld_legdat_adrdir_in_accepted_values + values: ['N', 'S', 'E', 'W', 'NE', 'NW', 'SE', 'SW'] + config: &unique-conditions + where: + cur = 'Y' + AND deactivat IS NULL - name: adrid description: '{{ doc("column_adrid") }}' - name: adrno @@ -55,6 +64,10 @@ sources: description: '{{ doc("column_conveyno") }}' - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_legdat_cur_in_accepted_values + values: ['Y', 'D'] - name: deactivat description: '{{ doc("column_deactivat") }}' - name: floorno @@ -81,6 +94,13 @@ sources: description: Number of lots - name: parid description: '{{ doc("shared_column_pin") }}' + tests: + - not_null: + name: iasworld_legdat_parid_not_null + - relationships: + name: iasworld_legdat_parid_in_pardat_parid + to: source('iasworld', 'pardat') + field: parid - name: postalcode description: '{{ doc("column_postalcode") }}' - name: procdate @@ -105,6 +125,13 @@ sources: description: Consolidated school district value percentage - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_legdat_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr + config: *unique-conditions - name: sortkey description: Block, lot and qualifier, reformatted for sorting - name: splitno @@ -115,6 +142,11 @@ sources: description: Star credit flag - name: statecode description: '{{ doc("shared_column_prop_address_state") }}' + tests: + - accepted_values: + name: iasworld_legdat_statecode_in_accepted_values + values: ['IL'] + config: *unique-conditions - name: status description: '{{ doc("column_status") }}' - name: strcd @@ -147,6 +179,9 @@ sources: description: Tax district 6 - name: taxyr description: '{{ doc("shared_column_year") }}' + tests: + - not_null: + name: iasworld_legdat_taxyr_not_null - name: trans_id description: '{{ doc("column_trans_id") }}' - name: unitdesc @@ -157,6 +192,13 @@ sources: description: '{{ doc("column_upd_status") }}' - name: user1 description: '{{ doc("shared_column_township_code") }}' + tests: + - not_null: + name: iasworld_legdat_user1_not_null + - relationships: + name: iasworld_legdat_user1_in_spatial_township_township_code + to: source('spatial', 'township') + field: township_code - name: wen description: '{{ doc("shared_column_updated_at") }}' - name: wencalc @@ -173,12 +215,57 @@ sources: description: Z coordinate - name: zip1 description: '{{ doc("shared_column_prop_address_zipcode_1") }}' + tests: + - dbt_utils.not_accepted_values: + name: iasworld_legdat_zip1_has_no_00000_values + values: ['00000'] + config: *unique-conditions - name: zip2 description: '{{ doc("shared_column_prop_address_zipcode_2") }}' + tests: + - dbt_utils.not_accepted_values: + name: iasworld_legdat_zip2_has_no_0000_values + values: ['0000'] + config: *unique-conditions tests: - unique_combination_of_columns: - name: legdat_unique_by_parid_taxyr + name: iasworld_legdat_unique_by_parid_taxyr combination_of_columns: - parid - taxyr + # Not using column_length here because adrno needs to be cast first + - expression_is_true: + name: iasworld_legdat_adrno_length_lte_5 + expression: LENGTH(CAST(adrno AS varchar)) <= 5 + select_columns: + - parid + - taxyr + - adrno + config: *unique-conditions + - column_length: + name: iasworld_legdat_adrsuf_zip1_taxdist_length_lte_5 + columns: + - adrsuf + - zip1 + - taxdist + length: 5 + additional_select_columns: + - parid + - taxyr + - no_extra_whitespace: + name: iasworld_legdat_address_columns_no_extra_whitespace + column_names: + - adrpre + - adrdir + - adrstr + - adrsuf + - adrsuf2 + - unitdesc + - unitno + - cityname + - statecode + - zip1 + - zip2 + allow_interior_space: true + config: *unique-conditions diff --git a/dbt/models/iasworld/schema/iasworld.lpmod.yml b/dbt/models/iasworld/schema/iasworld.lpmod.yml index 70d27c523..fab0cbd3c 100644 --- a/dbt/models/iasworld/schema/iasworld.lpmod.yml +++ b/dbt/models/iasworld/schema/iasworld.lpmod.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: lpmod diff --git a/dbt/models/iasworld/schema/iasworld.lpnbhd.yml b/dbt/models/iasworld/schema/iasworld.lpnbhd.yml index 045f23c39..fdd7bb398 100644 --- a/dbt/models/iasworld/schema/iasworld.lpnbhd.yml +++ b/dbt/models/iasworld/schema/iasworld.lpnbhd.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: lpnbhd diff --git a/dbt/models/iasworld/schema/iasworld.oby.yml b/dbt/models/iasworld/schema/iasworld.oby.yml index 3c05ff8cf..06ade2537 100644 --- a/dbt/models/iasworld/schema/iasworld.oby.yml +++ b/dbt/models/iasworld/schema/iasworld.oby.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld - type_condo tables: @@ -20,6 +21,14 @@ sources: description: '{{ doc("column_altbasedate") }}' - name: area description: '{{ doc("column_area") }}' + tests: + - dbt_utils.accepted_range: + name: iasworld_oby_area_gte_1 + min_value: 1 + config: &unique-conditions + where: | + cur = 'Y' + AND deactivat IS NULL - name: areaid description: '{{ doc("column_areaid") }}' - name: areasum @@ -28,6 +37,12 @@ sources: description: '{{ doc("column_bld_modelid") }}' - name: card description: '{{ doc("column_card") }}' + tests: + - dbt_utils.accepted_range: + name: iasworld_oby_card_between_1_and_100 + min_value: 1 + max_value: 100 + config: *unique-conditions - name: cdpct description: Cost and design percent - name: chgrsn @@ -56,6 +71,10 @@ sources: description: Cubic feet - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_oby_cur_in_accepted_values + values: ['Y', 'D'] - name: curmult description: Current multiplier - name: deactivat @@ -100,6 +119,12 @@ sources: description: Factor used to calculate `RCNADJ` - name: lline description: '{{ doc("shared_column_lline") }}' + tests: + - dbt_utils.accepted_range: + name: iasworld_oby_lline_between_1_and_100 + min_value: 1 + max_value: 100 + config: *unique-conditions - name: locmult description: Local multiplier - name: lumpadj @@ -186,6 +211,14 @@ sources: description: '{{ doc("column_salekey") }}' - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_oby_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr + - card + - lline - name: status description: '{{ doc("column_status") }}' - name: stories @@ -194,6 +227,9 @@ sources: description: '{{ doc("column_suppress") }}' - name: taxyr description: '{{ doc("shared_column_year") }}' + tests: + - not_null: + name: iasworld_oby_taxyr_not_null - name: trans_id description: '{{ doc("column_trans_id") }}' - name: units @@ -220,14 +256,36 @@ sources: description: '{{ doc("column_whocalc") }}' - name: yrblt description: '{{ doc("shared_column_char_yrblt") }}' + tests: + - not_null: + name: iasworld_oby_yrblt_not_null + config: *unique-conditions + - dbt_utils.accepted_range: + name: iasworld_oby_yrblt_between_1850_and_now + min_value: 1850 + max_value: cast(year(current_date) as decimal) + config: *unique-conditions - name: yrrem description: Year remodeled tests: - unique_combination_of_columns: - name: oby_unique_by_parid_card_lline_taxyr + name: iasworld_oby_unique_by_parid_card_lline_taxyr combination_of_columns: - parid - card - lline - taxyr + - row_values_match_after_join: + name: iasworld_oby_class_matches_pardat_class + column: substr(model.class, 1, 1) + external_model: source('iasworld', 'pardat') + external_column: substr(external_model.class, 1, 1) + join_columns: + - parid + - taxyr + config: + where: | + taxyr >= '2022' + AND cur = 'Y' + AND deactivat IS NULL diff --git a/dbt/models/iasworld/schema/iasworld.owndat.yml b/dbt/models/iasworld/schema/iasworld.owndat.yml index f10094865..e3230e65d 100644 --- a/dbt/models/iasworld/schema/iasworld.owndat.yml +++ b/dbt/models/iasworld/schema/iasworld.owndat.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: owndat @@ -59,6 +60,10 @@ sources: description: '{{ doc("column_country") }}' - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_owndat_cur_in_accepted_values + values: ['Y', 'D'] - name: deactivat description: '{{ doc("column_deactivat") }}' - name: delinstalltype @@ -137,6 +142,13 @@ sources: description: '{{ doc("column_page") }}' - name: parid description: '{{ doc("shared_column_pin") }}' + tests: + - not_null: + name: iasworld_owndat_parid_not_null + - relationships: + name: iasworld_owndat_parid_in_pardat_parid + to: source('iasworld', 'pardat') + field: parid - name: partial description: '{{ doc("column_partial") }}' - name: pctown @@ -153,6 +165,16 @@ sources: description: '{{ doc("column_salekey") }}' - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_owndat_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr + config: &unique-conditions + where: | + cur = 'Y' + AND deactivat IS NULL - name: site description: '{{ doc("column_site") }}' - name: skip_addr_validation @@ -185,12 +207,63 @@ sources: description: '{{ doc("column_whocalc") }}' - name: zip1 description: '{{ doc("shared_column_mail_address_zipcode_1") }}' + tests: + - dbt_utils.not_accepted_values: + name: iasworld_owndat_zip1_has_no_00000_values + values: ['00000'] + config: *unique-conditions - name: zip2 description: '{{ doc("shared_column_mail_address_zipcode_2") }}' + tests: + - dbt_utils.not_accepted_values: + name: iasworld_owndat_zip2_has_no_0000_values + values: ['0000'] + config: *unique-conditions tests: - unique_combination_of_columns: - name: owndat_unique_by_parid_taxyr + name: iasworld_owndat_unique_by_parid_taxyr combination_of_columns: - parid - taxyr + - no_extra_whitespace: + name: iasworld_owndat_address_columns_no_extra_whitespace + column_names: + - own1 + - own2 + - addr1 + - addr2 + - addr3 + - addr4 + - addr5 + - adrpre + - adrdir + - adrstr + - adrsuf + - unitdesc + - unitno + - cityname + - statecode + - zip1 + - zip2 + - user27 + allow_interior_space: true + config: *unique-conditions + - expression_is_true: + name: iasworld_owndat_addr1_address_is_well_formed + expression: | + CASE + WHEN LENGTH(REGEXP_EXTRACT(addr1, '^[N|S|E|W]{0,2}\s{0,1}([0-9]{1,5})',1)) <= 5 THEN TRUE + WHEN REGEXP_LIKE(UPPER(addr1), '^[PO BOX|P\.O\. BOX|P O BOX|BOX|P O BX|PO BX|PO BX|P.O.BOX]') THEN TRUE + WHEN REGEXP_LIKE(UPPER(addr1), '^[C\/O|ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE]') THEN TRUE + ELSE FALSE + END + select_columns: + - parid + - taxyr + - addr1 + config: + where: | + cur = 'Y' + AND deactivat IS NULL + AND addr1 IS NOT NULL diff --git a/dbt/models/iasworld/schema/iasworld.pardat.yml b/dbt/models/iasworld/schema/iasworld.pardat.yml index 74bd4c46f..f9b6a35f0 100644 --- a/dbt/models/iasworld/schema/iasworld.pardat.yml +++ b/dbt/models/iasworld/schema/iasworld.pardat.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: pardat @@ -69,8 +70,24 @@ sources: description: '{{ doc("column_cityname") }}' - name: class description: '{{ doc("shared_column_class") }}' + tests: + - relationships: + name: iasworld_pardat_class_in_ccao_class_dict + to: source('ccao', 'class_dict') + field: class_code + config: + where: | + taxyr >= '2022' + AND class NOT IN ('EX', 'RR') + AND NOT REGEXP_LIKE(class, '[0-9]{3}[A|B]') + AND cur = 'Y' + AND deactivat IS NULL - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_pardat_cur_in_accepted_values + values: ['Y', 'D'] - name: deactivat description: '{{ doc("column_deactivat") }}' - name: farminc @@ -171,6 +188,16 @@ sources: description: Security field - name: seq description: '{{ doc("shared_column_seq") }}' + tests: + - dbt_utils.sequential_values: + name: iasworld_pardat_seq_all_sequential_exist + group_by_columns: + - parid + - taxyr + config: &unique-conditions + where: | + cur = 'Y' + AND deactivat IS NULL - name: skip_addr_validation description: '{{ doc("column_skip_addr_validation") }}' - name: splitno @@ -240,7 +267,7 @@ sources: tests: - column_is_subset_of_external_column: - name: pardat_nbhd_matches_spatial_town_nbhd + name: iasworld_pardat_nbhd_matches_spatial_town_nbhd column: nbhd external_model: spatial.neighborhood external_column: town_nbhd @@ -250,10 +277,19 @@ sources: config: # Codes ending in 999 are dummy codes used for some purpose, # although we do not yet know what it is - where: (taxyr between '2010' and '2021') and (nbhd not like '%999') - error_if: ">1992" + where: | + (taxyr BETWEEN '2010' AND CAST(YEAR(NOW()) AS varchar)) + AND (nbhd NOT LIKE '%999') - unique_combination_of_columns: - name: pardat_unique_by_parid_taxyr + name: iasworld_pardat_unique_by_parid_taxyr combination_of_columns: - parid - taxyr + - expression_is_true: + name: iasworld_pardat_adrno_length_lte_5 + expression: LENGTH(CAST(adrno AS varchar)) <= 5 + select_columns: + - parid + - taxyr + - adrno + config: *unique-conditions diff --git a/dbt/models/iasworld/schema/iasworld.permit.yml b/dbt/models/iasworld/schema/iasworld.permit.yml index 9b2b15f1c..26c062bdc 100644 --- a/dbt/models/iasworld/schema/iasworld.permit.yml +++ b/dbt/models/iasworld/schema/iasworld.permit.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: permit diff --git a/dbt/models/iasworld/schema/iasworld.rcoby.yml b/dbt/models/iasworld/schema/iasworld.rcoby.yml index 9d0ef9269..357b52c6b 100644 --- a/dbt/models/iasworld/schema/iasworld.rcoby.yml +++ b/dbt/models/iasworld/schema/iasworld.rcoby.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: rcoby diff --git a/dbt/models/iasworld/schema/iasworld.sales.yml b/dbt/models/iasworld/schema/iasworld.sales.yml index 466c91ed6..d1ea23ecc 100644 --- a/dbt/models/iasworld/schema/iasworld.sales.yml +++ b/dbt/models/iasworld/schema/iasworld.sales.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: sales @@ -29,6 +30,10 @@ sources: description: County final date - name: cur description: '{{ doc("column_cur") }}' + tests: + - accepted_values: + name: iasworld_sales_cur_in_accepted_values + values: ['Y', 'D'] - name: deactivat description: '{{ doc("column_deactivat") }}' - name: deed_ref @@ -45,6 +50,15 @@ sources: description: Document image number for the sale deed - name: instrtyp description: '{{ doc("shared_column_deed_type") }}' + tests: + - accepted_values: + name: iasworld_sales_instrtyp_in_accepted_values + values: ['01', '02', '03', '04', '05', '06', 'B'] + config: &unique-conditions + where: | + substr(saledt, 1, 4) >= '2011' + AND cur = 'Y' + AND deactivat IS NULL - name: instruno description: '{{ doc("shared_column_document_number") }}' - name: jur @@ -79,8 +93,22 @@ sources: description: Time adjusted sale parent name - name: parid description: '{{ doc("shared_column_pin") }}' + tests: + - not_null: + name: iasworld_sales_parid_not_null + - relationships: + name: iasworld_sales_parid_in_pardat_parid + to: source('iasworld', 'pardat') + field: parid + config: *unique-conditions - name: price description: '{{ doc("shared_column_sale_price") }}' + tests: + - dbt_utils.accepted_range: + name: iasworld_sales_price_between_0_and_1B + min_value: 0 + max_value: 1000000000 + config: *unique-conditions - name: recorddt description: Recording date - name: saledt @@ -146,9 +174,16 @@ sources: tests: - unique_combination_of_columns: - name: sales_unique_by_parid_instruno + name: iasworld_sales_unique_by_parid_instruno combination_of_columns: - parid - instruno - config: - where: substr(saledt, 1, 4) >= '2023' + config: *unique-conditions + - expression_is_true: + name: iasworld_sales_saledt_lte_now + expression: DATE_PARSE(SUBSTR(saledt, 1, 10), '%Y-%m-%d') <= current_date + select_columns: + - parid + - saledt + - instruno + config: *unique-conditions diff --git a/dbt/models/iasworld/schema/iasworld.splcom.yml b/dbt/models/iasworld/schema/iasworld.splcom.yml index 65844148d..f9756792d 100644 --- a/dbt/models/iasworld/schema/iasworld.splcom.yml +++ b/dbt/models/iasworld/schema/iasworld.splcom.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: splcom diff --git a/dbt/models/iasworld/schema/iasworld.valclass.yml b/dbt/models/iasworld/schema/iasworld.valclass.yml index 0c0103453..4f4f9179d 100644 --- a/dbt/models/iasworld/schema/iasworld.valclass.yml +++ b/dbt/models/iasworld/schema/iasworld.valclass.yml @@ -3,6 +3,7 @@ sources: loaded_at_field: date_parse(wen, '%Y-%m-%d %H:%i:%s.0') tags: - load_auto + - test_qc_iasworld tables: - name: valclass diff --git a/dbt/models/location/schema.yml b/dbt/models/location/schema.yml index c3860a671..ccb2136c6 100644 --- a/dbt/models/location/schema.yml +++ b/dbt/models/location/schema.yml @@ -239,13 +239,13 @@ models: tests: # Unique by 10-digit PIN and year - unique_combination_of_columns: - name: vw_pin10_location_unique_by_10_digit_pin_and_year + name: location_vw_pin10_location_unique_by_keys combination_of_columns: - pin10 - year - # GeoIDs are the correct length + # GEOIDs are the correct length - column_length: - name: vw_pin10_location_seven_digit_geoids_are_the_correct_length + name: location_vw_pin10_location_7_digit_ids_are_correct_length length: 7 columns: - census_place_geoid diff --git a/dbt/models/reporting/schema.yml b/dbt/models/reporting/schema.yml index 42d5fb7b2..9e3d6383b 100644 --- a/dbt/models/reporting/schema.yml +++ b/dbt/models/reporting/schema.yml @@ -7,31 +7,31 @@ sources: description: '{{ doc("table_ratio_stats") }}' tests: - expression_is_true: - name: ratio_stats_no_nulls - expression: >- + name: reporting_ratio_stats_no_nulls + expression: | year IS NOT NULL - AND triad IS NOT NULL - AND geography_type IS NOT NULL - AND property_group IS NOT NULL - AND assessment_stage IS NOT NULL - AND sale_year IS NOT NULL + AND triad IS NOT NULL + AND geography_type IS NOT NULL + AND property_group IS NOT NULL + AND assessment_stage IS NOT NULL + AND sale_year IS NOT NULL - expression_is_true: - name: ratio_stats_metrics_are_sensible - expression: >- + name: reporting_ratio_stats_metrics_are_sensible + expression: | cod >= 0 - AND prd >= 0 - AND prb BETWEEN -1 AND 1 - AND mki >= 0 - AND triad IS NOT NULL - AND geography_type IS NOT NULL - AND property_group IS NOT NULL - AND assessment_stage IS NOT NULL - AND sale_year IS NOT NULL + AND prd >= 0 + AND prb BETWEEN -1 AND 1 + AND mki >= 0 + AND triad IS NOT NULL + AND geography_type IS NOT NULL + AND property_group IS NOT NULL + AND assessment_stage IS NOT NULL + AND sale_year IS NOT NULL - expression_is_true: - name: ratio_stats_within_directionality - expression: >- + name: reporting_ratio_stats_within_directionality + expression: | within_20_pct >= within_10_pct - AND within_10_pct >= within_05_pct + AND within_10_pct >= within_05_pct models: - name: reporting.res_report_summary @@ -41,25 +41,25 @@ models: - daily tests: - unique_combination_of_columns: - name: res_report_summary_unique_by_geography_id_property_group_assessment_stage_year + name: reporting_res_report_summary_unique_by_keys combination_of_columns: - geography_id - property_group - assessment_stage - year - expression_is_true: - name: res_report_summary_sale_year_equals_year_minus_one + name: reporting_res_report_summary_sale_year_eq_year_minus_one expression: CAST(sale_year AS INTEGER) = CAST(year AS INTEGER) - 1 select_columns: - sale_year - year - expression_is_true: - name: res_report_summary_no_nulls - expression: >- + name: reporting_res_report_summary_no_nulls + expression: | triad IS NOT NULL - AND geography_type IS NOT NULL - AND property_group IS NOT NULL - AND assessment_stage IS NOT NULL + AND geography_type IS NOT NULL + AND property_group IS NOT NULL + AND assessment_stage IS NOT NULL select_columns: - triad - geography_type @@ -70,25 +70,25 @@ models: description: '{{ doc("view_vw_assessment_roll") }}' tests: - unique_combination_of_columns: - name: vw_assessment_roll_unique_by_year_stage_township_name_class + name: reporting_vw_assessment_roll_unique_by_keys combination_of_columns: - year - stage - township_name - class - expression_is_true: - name: vw_assessment_roll_no_nulls - expression: >- + name: reporting_vw_assessment_roll_no_nulls + expression: | stage IS NOT NULL - AND township_name IS NOT NULL - AND triad IS NOT NULL - AND class IS NOT NULL + AND township_name IS NOT NULL + AND triad IS NOT NULL + AND class IS NOT NULL - name: reporting.vw_pin_most_recent_boundary description: '{{ doc("view_vw_pin_most_recent_boundary") }}' tests: - unique_combination_of_columns: - name: vw_pin_most_recent_boundary_unique_by_pin10_municipality_name + name: reporting_vw_pin_most_recent_boundary_unique_by_keys combination_of_columns: - pin10 - municipality_name @@ -97,7 +97,7 @@ models: description: '{{ doc("view_vw_pin_most_recent_sale") }}' tests: - unique_combination_of_columns: - name: vw_pin_most_recent_sale_unique_by_pin + name: reporting_vw_pin_most_recent_sale_unique_by_pin combination_of_columns: - pin # These two tests make sure that the most recent sale view 1) contains @@ -105,37 +105,37 @@ models: # sale view should contain one observation for each parcel in the county # regardless of whether it has recently sold or not. - value_is_present: - name: vw_pin_most_recent_sale_null + name: reporting_vw_pin_most_recent_sale_null expression: sale_price IS NULL - value_is_present: - name: vw_pin_most_recent_sale_not_null + name: reporting_vw_pin_most_recent_sale_not_null expression: sale_price IS NOT NULL - name: reporting.vw_ratio_stats description: '{{ doc("view_vw_ratio_stats") }}' tests: - unique_combination_of_columns: - name: vw_ratio_stats_unique_by_pin_year_assessment_stage_sale_price + name: reporting_vw_ratio_stats_unique_by_keys combination_of_columns: - pin - year - assessment_stage - sale_price - expression_is_true: - name: vw_ratio_stats_no_nulls - expression: >- + name: reporting_vw_ratio_stats_no_nulls + expression: | property_group IS NOT NULL - AND assessment_stage IS NOT NULL - AND triad IS NOT NULL - AND township_code IS NOT NULL + AND assessment_stage IS NOT NULL + AND triad IS NOT NULL + AND township_code IS NOT NULL - expression_is_true: - name: vw_ratio_stats_sale_year_equals_year_minus_one + name: reporting_vw_ratio_stats_sale_year_equals_year_minus_one expression: CAST(sale_year AS INTEGER) = CAST(year AS INTEGER) - 1 select_columns: - sale_year - year - expression_is_true: - name: vw_ratio_stats_ratio_greater_than_zero + name: reporting_vw_ratio_stats_ratio_greater_than_zero expression: ratio >= 0 select_columns: - ratio @@ -144,37 +144,37 @@ models: description: '{{ doc("view_vw_top_5") }}' tests: - unique_combination_of_columns: - name: vw_top_5_unique_by_rank_parid_year + name: reporting_vw_top_5_unique_by_keys combination_of_columns: - rank - parid - year - expression_is_true: - name: vw_top_5_rank_between_1_and_5 + name: reporting_vw_top_5_rank_between_1_and_5 expression: rank BETWEEN 1 AND 5 - expression_is_true: - name: vw_top_5_no_nulls - expression: >- + name: reporting_vw_top_5_no_nulls + expression: | year IS NOT NULL - AND township IS NOT NULL - AND class IS NOT NULL - AND rank IS NOT NULL - AND total_av IS NOT NULL - AND stage_used IS NOT NULL + AND township IS NOT NULL + AND class IS NOT NULL + AND rank IS NOT NULL + AND total_av IS NOT NULL + AND stage_used IS NOT NULL - name: reporting.vw_town_sale_history description: '{{ doc("view_vw_town_sale_history") }}' tests: - unique_combination_of_columns: - name: vw_town_sale_history_unique_by_geography_id_property_group_sale_year + name: reporting_vw_town_sale_history_unique_by_keys combination_of_columns: - geography_id - property_group - sale_year - expression_is_true: - name: vw_town_sale_history_no_nulls - expression: >- + name: reporting_vw_town_sale_history_no_nulls + expression: | sale_year IS NOT NULL - AND property_group IS NOT NULL - AND geography_id IS NOT NULL - AND sale_median IS NOT NULL + AND property_group IS NOT NULL + AND geography_id IS NOT NULL + AND sale_median IS NOT NULL diff --git a/dbt/tests/generic/test_column_length.sql b/dbt/tests/generic/test_column_length.sql index f39d889eb..63e30c6ab 100644 --- a/dbt/tests/generic/test_column_length.sql +++ b/dbt/tests/generic/test_column_length.sql @@ -11,14 +11,14 @@ {%- set columns_csv = additional_select_columns | join(", ") %} {%- set length_columns = [] %} - {% for column in columns %} + {%- for column in columns %} {%- set length_columns = length_columns.append([column, "len_" + column]) %} - {% endfor %} + {%- endfor %} {%- set select_columns = additional_select_columns %} - {% set filter_conditions = [] %} + {%- set filter_conditions = [] %} - {% for column, length_column in length_columns %} + {%- for column, length_column in length_columns %} {%- set select_columns = select_columns.append( "length(" + column + ") as " + length_column ) %} @@ -31,7 +31,7 @@ + length | string + ")" ) %} - {% endfor %} + {%- endfor %} {%- set columns_csv = select_columns | join(", ") %} {%- set filter_conditions_str = filter_conditions | join(" or ") %} diff --git a/dbt/tests/generic/test_expression_is_true.sql b/dbt/tests/generic/test_expression_is_true.sql index 44bdb9334..609fe9ed7 100644 --- a/dbt/tests/generic/test_expression_is_true.sql +++ b/dbt/tests/generic/test_expression_is_true.sql @@ -4,10 +4,10 @@ -- an optional `select_columns` option representing an array of columns to -- select for failing rows. If no `select_columns` array is provided, defaults -- to selecting 1 for failing rows. -{% test expression_is_true(model, expression, select_columns=[1]) %} +{% test expression_is_true(model, expression, column_name, select_columns=[1]) %} {%- set columns_csv = select_columns | join(", ") %} select {{ columns_csv }} from {{ model }} - where not ({{ expression }}) + where not ({{ column_name }} {{ expression }}) {% endtest %} diff --git a/dbt/tests/generic/test_no_extra_whitespace.sql b/dbt/tests/generic/test_no_extra_whitespace.sql index d7db28f68..30c30e107 100644 --- a/dbt/tests/generic/test_no_extra_whitespace.sql +++ b/dbt/tests/generic/test_no_extra_whitespace.sql @@ -1,19 +1,29 @@ -- Test that one or more string columns do not contain extraneous whitespace -{% test no_extra_whitespace(model, column_names) %} +{% test no_extra_whitespace(model, column_names, allow_interior_space=false) %} {%- set columns = column_names | join(", ") %} {%- set conditions_list = [] %} - {% for column_name in column_names %} - {%- set conditions_list = conditions_list.append( - "(" - + column_name - + " like '% %' or " - + column_name - + " like '% ' or " - + column_name - + " like ' %')" - ) %} + {%- for column_name in column_names %} + {%- if allow_interior_space %} + {%- set conditions_list = conditions_list.append( + "(" + + column_name + + " like '% ' or " + + column_name + + " like ' %')" + ) %} + {%- else %} + {%- set conditions_list = conditions_list.append( + "(" + + column_name + + " like '% %' or " + + column_name + + " like '% ' or " + + column_name + + " like ' %')" + ) %} + {% endif %} {%- endfor %} {%- set conditions = conditions_list | join(" or ") %} diff --git a/dbt/tests/generic/test_row_values_match_after_join.sql b/dbt/tests/generic/test_row_values_match_after_join.sql new file mode 100644 index 000000000..40a148f49 --- /dev/null +++ b/dbt/tests/generic/test_row_values_match_after_join.sql @@ -0,0 +1,30 @@ +-- Test that row values match after joining two tables. Row values can be +-- a subset of the values in the joined table, i.e. if the class of PINA +-- from tableA is '212' and the class of PINA from tableB is '211' and '212', +-- then tableA matches (returns no rows). +{% test row_values_match_after_join( + model, column, external_model, external_column, join_columns=[] +) %} + + {%- set join_columns_csv = join_columns | join(", ") -%} + + {%- if "." in column -%} {%- set model_col = column -%} + {%- else -%} {%- set model_col = "model" ~ "." ~ column -%} + {%- endif -%} + + {%- if "." in external_column -%} {%- set external_model_col = external_column -%} + {%- else -%} + {%- set external_model_col = "external_model" ~ "." ~ external_column -%} + {%- endif -%} + + select + {{ join_columns_csv }}, + array_agg({{ model_col }}) as model_col, + array_agg({{ external_model_col }}) as external_model_col + from {{ external_model }} as external_model + join (select * from {{ model }}) as model using ({{ join_columns_csv }}) + group by {{ join_columns_csv }} + having + sum(case when {{ external_model_col }} = {{ model_col }} then 1 else 0 end) = 0 + +{% endtest %}