From 5601744bdebfb3f99d3df27745b7e9cbd456aa74 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Mon, 23 Jan 2023 13:26:12 -0600 Subject: [PATCH 1/4] add base class for merge exclude tests --- .../test_incremental_merge_exclude_columns.py | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py diff --git a/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py b/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py new file mode 100644 index 00000000000..d5adc51a257 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py @@ -0,0 +1,127 @@ +import pytest +from dbt.tests.util import run_dbt, check_relations_equal +from collections import namedtuple + + +models__merge_exclude_columns_sql = """ +{{ config( + materialized = 'incremental', + unique_key = 'id', + incremental_strategy='merge', + merge_exclude_columns=['msg'] +) }} + +{% if not is_incremental() %} + +select 1 as id, 'hello' as msg, 'blue' as color +union all +select 2 as id, 'goodbye' as msg, 'red' as color + +{% else %} + +select 1 as id, 'hey' as msg, 'blue' as color +union all +select 2 as id, 'yo' as msg, 'green' as color +union all +select 3 as id, 'anyway' as msg, 'purple' as color + +{% endif %} +""" + +seeds__expected_merge_exclude_columns_csv = """id,msg,color +1,hello,blue +2,goodbye,green +3,anyway,purple +""" + +ResultHolder = namedtuple( + "ResultHolder", + [ + "seed_count", + "model_count", + "seed_rows", + "inc_test_model_count", + "opt_model_count", + "relation", + ], +) + + +class BaseMergeExcludeColumns: + @pytest.fixture(scope="class") + def models(self): + return {"merge_exclude_columns.sql": models__merge_exclude_columns_sql} + + @pytest.fixture(scope="class") + def seeds(self): + return {"expected_merge_exclude_columns.csv": seeds__expected_merge_exclude_columns_csv} + + def update_incremental_model(self, incremental_model): + """update incremental model after the seed table has been updated""" + model_result_set = run_dbt(["run", "--select", incremental_model]) + return len(model_result_set) + + def get_test_fields( + self, project, seed, incremental_model, update_sql_file, opt_model_count=None + ): + + seed_count = len(run_dbt(["seed", "--select", seed, "--full-refresh"])) + + model_count = len(run_dbt(["run", "--select", incremental_model, "--full-refresh"])) + # pass on kwarg + relation = incremental_model + # update seed in anticipation of incremental model update + row_count_query = "select * from {}.{}".format(project.test_schema, seed) + # project.run_sql_file(Path("seeds") / Path(update_sql_file + ".sql")) + seed_rows = len(project.run_sql(row_count_query, fetch="all")) + + # propagate seed state to incremental model according to unique keys + inc_test_model_count = self.update_incremental_model(incremental_model=incremental_model) + + return ResultHolder( + seed_count, model_count, seed_rows, inc_test_model_count, opt_model_count, relation + ) + + def check_scenario_correctness(self, expected_fields, test_case_fields, project): + """Invoke assertions to verify correct build functionality""" + # 1. test seed(s) should build afresh + assert expected_fields.seed_count == test_case_fields.seed_count + # 2. test model(s) should build afresh + assert expected_fields.model_count == test_case_fields.model_count + # 3. seeds should have intended row counts post update + assert expected_fields.seed_rows == test_case_fields.seed_rows + # 4. incremental test model(s) should be updated + assert expected_fields.inc_test_model_count == test_case_fields.inc_test_model_count + # 5. extra incremental model(s) should be built; optional since + # comparison may be between an incremental model and seed + if expected_fields.opt_model_count and test_case_fields.opt_model_count: + assert expected_fields.opt_model_count == test_case_fields.opt_model_count + # 6. result table should match intended result set (itself a relation) + check_relations_equal( + project.adapter, [expected_fields.relation, test_case_fields.relation] + ) + + def get_expected_fields(self, relation, seed_rows, opt_model_count=None): + return ResultHolder( + seed_count=1, + model_count=1, + inc_test_model_count=1, + seed_rows=seed_rows, + opt_model_count=opt_model_count, + relation=relation, + ) + + # no unique_key test + def test__merge_exclude_columns(self, project): + """seed should match model after two incremental runs""" + + expected_fields = self.get_expected_fields( + relation="expected_merge_exclude_columns", seed_rows=3 + ) + test_case_fields = self.get_test_fields( + project, + seed="expected_merge_exclude_columns", + incremental_model="merge_exclude_columns", + update_sql_file=None, + ) + self.check_scenario_correctness(expected_fields, test_case_fields, project) From 90c183d409b1dc7667c562e755d7d79797735218 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Mon, 23 Jan 2023 13:28:27 -0600 Subject: [PATCH 2/4] changie <33 --- .changes/unreleased/Fixes-20230123-132814.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20230123-132814.yaml diff --git a/.changes/unreleased/Fixes-20230123-132814.yaml b/.changes/unreleased/Fixes-20230123-132814.yaml new file mode 100644 index 00000000000..f05bac4571a --- /dev/null +++ b/.changes/unreleased/Fixes-20230123-132814.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: add merge_exclude_columns adapter tests +time: 2023-01-23T13:28:14.808748-06:00 +custom: + Author: dave-connors-3 + Issue: "6699" From 1a3eb776d72666d1402df7450c82676192fa8500 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Tue, 7 Feb 2023 08:37:05 -0600 Subject: [PATCH 3/4] remove comments --- .../incremental/test_incremental_merge_exclude_columns.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py b/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py index d5adc51a257..22ea1653dcb 100644 --- a/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py +++ b/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py @@ -68,11 +68,11 @@ def get_test_fields( seed_count = len(run_dbt(["seed", "--select", seed, "--full-refresh"])) model_count = len(run_dbt(["run", "--select", incremental_model, "--full-refresh"])) - # pass on kwarg + relation = incremental_model # update seed in anticipation of incremental model update row_count_query = "select * from {}.{}".format(project.test_schema, seed) - # project.run_sql_file(Path("seeds") / Path(update_sql_file + ".sql")) + seed_rows = len(project.run_sql(row_count_query, fetch="all")) # propagate seed state to incremental model according to unique keys @@ -111,7 +111,6 @@ def get_expected_fields(self, relation, seed_rows, opt_model_count=None): relation=relation, ) - # no unique_key test def test__merge_exclude_columns(self, project): """seed should match model after two incremental runs""" From 25b74c47416b92db8e6cea9965c3bbffad21e4e9 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Tue, 7 Feb 2023 14:18:40 -0600 Subject: [PATCH 4/4] add comments to sql, remove and clarify contents of resultholder --- .../test_incremental_merge_exclude_columns.py | 36 +++++++------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py b/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py index 22ea1653dcb..db958f1eda4 100644 --- a/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py +++ b/tests/adapter/dbt/tests/adapter/incremental/test_incremental_merge_exclude_columns.py @@ -13,12 +13,16 @@ {% if not is_incremental() %} +-- data for first invocation of model + select 1 as id, 'hello' as msg, 'blue' as color union all select 2 as id, 'goodbye' as msg, 'red' as color {% else %} +-- data for subsequent incremental update + select 1 as id, 'hey' as msg, 'blue' as color union all select 2 as id, 'yo' as msg, 'green' as color @@ -41,7 +45,6 @@ "model_count", "seed_rows", "inc_test_model_count", - "opt_model_count", "relation", ], ) @@ -61,9 +64,7 @@ def update_incremental_model(self, incremental_model): model_result_set = run_dbt(["run", "--select", incremental_model]) return len(model_result_set) - def get_test_fields( - self, project, seed, incremental_model, update_sql_file, opt_model_count=None - ): + def get_test_fields(self, project, seed, incremental_model, update_sql_file): seed_count = len(run_dbt(["seed", "--select", seed, "--full-refresh"])) @@ -78,9 +79,7 @@ def get_test_fields( # propagate seed state to incremental model according to unique keys inc_test_model_count = self.update_incremental_model(incremental_model=incremental_model) - return ResultHolder( - seed_count, model_count, seed_rows, inc_test_model_count, opt_model_count, relation - ) + return ResultHolder(seed_count, model_count, seed_rows, inc_test_model_count, relation) def check_scenario_correctness(self, expected_fields, test_case_fields, project): """Invoke assertions to verify correct build functionality""" @@ -92,31 +91,22 @@ def check_scenario_correctness(self, expected_fields, test_case_fields, project) assert expected_fields.seed_rows == test_case_fields.seed_rows # 4. incremental test model(s) should be updated assert expected_fields.inc_test_model_count == test_case_fields.inc_test_model_count - # 5. extra incremental model(s) should be built; optional since - # comparison may be between an incremental model and seed - if expected_fields.opt_model_count and test_case_fields.opt_model_count: - assert expected_fields.opt_model_count == test_case_fields.opt_model_count - # 6. result table should match intended result set (itself a relation) + # 5. result table should match intended result set (itself a relation) check_relations_equal( project.adapter, [expected_fields.relation, test_case_fields.relation] ) - def get_expected_fields(self, relation, seed_rows, opt_model_count=None): - return ResultHolder( + def test__merge_exclude_columns(self, project): + """seed should match model after two incremental runs""" + + expected_fields = ResultHolder( seed_count=1, model_count=1, inc_test_model_count=1, - seed_rows=seed_rows, - opt_model_count=opt_model_count, - relation=relation, + seed_rows=3, + relation="expected_merge_exclude_columns", ) - def test__merge_exclude_columns(self, project): - """seed should match model after two incremental runs""" - - expected_fields = self.get_expected_fields( - relation="expected_merge_exclude_columns", seed_rows=3 - ) test_case_fields = self.get_test_fields( project, seed="expected_merge_exclude_columns",