From 38ed3952c6a599e3bb398d82bf89d9c17f9de709 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 16 Mar 2023 11:58:55 -0500 Subject: [PATCH 1/5] add test for enforcing contracts on incremental materializations --- dev-requirements.txt | 4 +- tests/functional/adapter/test_constraints.py | 53 ++++++++++++++++++-- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 4a982392e..3be7bee14 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@er/ct-2293-incremental-mat#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@er/ct-2293-incremental-mat#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index ae3d874d2..39b9ae34a 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -3,7 +3,11 @@ from dbt.tests.adapter.constraints.test_constraints import ( BaseTableConstraintsColumnsEqual, BaseViewConstraintsColumnsEqual, - BaseConstraintsRuntimeEnforcement + BaseIncrementalConstraintsColumnsEqual, + BaseConstraintsRuntimeDdlEnforcement, + BaseConstraintsRollback, + BaseIncrementalConstraintsRuntimeDdlEnforcement, + BaseIncrementalConstraintsRollback, ) from dbt.tests.adapter.constraints.fixtures import ( my_model_sql, @@ -11,11 +15,13 @@ my_model_wrong_name_sql, my_model_view_wrong_order_sql, my_model_view_wrong_name_sql, + my_model_incremental_wrong_order_sql, + my_model_incremental_wrong_name_sql, model_schema_yml, ) _expected_sql_bigquery = """ -create or replace table {0} ( +create or replace table ( id integer not null, color string, date_day string @@ -85,7 +91,17 @@ def models(self): } -class TestBigQueryConstraintsRuntimeEnforcement(BaseConstraintsRuntimeEnforcement): +class TestBigQueryIncrementalConstraintsColumnsEqual(BigQueryColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, + "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } + + +class SetupBigQueryRuntimeDdlEnforcement: @pytest.fixture(scope="class") def models(self): return { @@ -95,9 +111,36 @@ def models(self): @pytest.fixture(scope="class") def expected_sql(self, project): - relation = relation_from_name(project.adapter, "my_model") - return _expected_sql_bigquery.format(relation) + return _expected_sql_bigquery + + +class SetupBigQueryConstraintsRollback: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": constraints_yml, + } @pytest.fixture(scope="class") def expected_error_messages(self): return ["Required field id cannot be null"] + + +class TestBigQueryTableConstraintsRuntimeDdlEnforcement(SetupBigQueryRuntimeDdlEnforcement, BaseConstraintsRuntimeDdlEnforcement): + pass + + +class TestBigQueryTableConstraintsRollback(SetupBigQueryConstraintsRollback, BaseConstraintsRollback): + pass + + +class TestBigQueryIncrementalConstraintsRuntimeDdlEnforcement( + SetupBigQueryRuntimeDdlEnforcement, + BaseIncrementalConstraintsRuntimeDdlEnforcement +): + pass + + +class TestBigQueryIncrementalConstraintsRollback(SetupBigQueryConstraintsRollback, BaseIncrementalConstraintsRollback): + pass From ddd29e2238451f331399070deccad552130ae002 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 16 Mar 2023 12:01:22 -0500 Subject: [PATCH 2/5] cleanup formatting --- tests/functional/adapter/test_constraints.py | 30 ++++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 39b9ae34a..8adfc9ef7 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -71,7 +71,10 @@ def data_types(self, int_type, string_type): ] -class TestBigQueryTableConstraintsColumnsEqual(BigQueryColumnEqualSetup, BaseTableConstraintsColumnsEqual): +class TestBigQueryTableConstraintsColumnsEqual( + BigQueryColumnEqualSetup, + BaseTableConstraintsColumnsEqual +): @pytest.fixture(scope="class") def models(self): return { @@ -81,7 +84,10 @@ def models(self): } -class TestBigQueryViewConstraintsColumnsEqual(BigQueryColumnEqualSetup, BaseViewConstraintsColumnsEqual): +class TestBigQueryViewConstraintsColumnsEqual( + BigQueryColumnEqualSetup, + BaseViewConstraintsColumnsEqual +): @pytest.fixture(scope="class") def models(self): return { @@ -91,7 +97,10 @@ def models(self): } -class TestBigQueryIncrementalConstraintsColumnsEqual(BigQueryColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual): +class TestBigQueryIncrementalConstraintsColumnsEqual( + BigQueryColumnEqualSetup, + BaseIncrementalConstraintsColumnsEqual +): @pytest.fixture(scope="class") def models(self): return { @@ -127,11 +136,17 @@ def expected_error_messages(self): return ["Required field id cannot be null"] -class TestBigQueryTableConstraintsRuntimeDdlEnforcement(SetupBigQueryRuntimeDdlEnforcement, BaseConstraintsRuntimeDdlEnforcement): +class TestBigQueryTableConstraintsRuntimeDdlEnforcement( + SetupBigQueryRuntimeDdlEnforcement, + BaseConstraintsRuntimeDdlEnforcement +): pass -class TestBigQueryTableConstraintsRollback(SetupBigQueryConstraintsRollback, BaseConstraintsRollback): +class TestBigQueryTableConstraintsRollback( + SetupBigQueryConstraintsRollback, + BaseConstraintsRollback +): pass @@ -142,5 +157,8 @@ class TestBigQueryIncrementalConstraintsRuntimeDdlEnforcement( pass -class TestBigQueryIncrementalConstraintsRollback(SetupBigQueryConstraintsRollback, BaseIncrementalConstraintsRollback): +class TestBigQueryIncrementalConstraintsRollback( + SetupBigQueryConstraintsRollback, + BaseIncrementalConstraintsRollback +): pass From 7508e69b2fc29b04088ac2725a272e35e80587e4 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 21 Mar 2023 11:32:18 -0500 Subject: [PATCH 3/5] fix test files --- tests/functional/adapter/test_constraints.py | 35 +++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 8adfc9ef7..57d9dff83 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -110,7 +110,7 @@ def models(self): } -class SetupBigQueryRuntimeDdlEnforcement: +class SetupBigQueryConstraintsRollback: @pytest.fixture(scope="class") def models(self): return { @@ -119,28 +119,23 @@ def models(self): } @pytest.fixture(scope="class") - def expected_sql(self, project): - return _expected_sql_bigquery + def expected_error_messages(self): + return ["Required field id cannot be null"] -class SetupBigQueryConstraintsRollback: +class TestBigQueryTableConstraintsRuntimeDdlEnforcement( + BaseConstraintsRuntimeDdlEnforcement +): @pytest.fixture(scope="class") def models(self): return { - "my_model.sql": my_model_sql, + "my_model.sql": my_model_wrong_order_sql, "constraints_schema.yml": constraints_yml, } @pytest.fixture(scope="class") - def expected_error_messages(self): - return ["Required field id cannot be null"] - - -class TestBigQueryTableConstraintsRuntimeDdlEnforcement( - SetupBigQueryRuntimeDdlEnforcement, - BaseConstraintsRuntimeDdlEnforcement -): - pass + def expected_sql(self, project): + return _expected_sql_bigquery class TestBigQueryTableConstraintsRollback( @@ -151,10 +146,18 @@ class TestBigQueryTableConstraintsRollback( class TestBigQueryIncrementalConstraintsRuntimeDdlEnforcement( - SetupBigQueryRuntimeDdlEnforcement, BaseIncrementalConstraintsRuntimeDdlEnforcement ): - pass + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_incremental_wrong_order_sql, + "constraints_schema.yml": constraints_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self, project): + return _expected_sql_bigquery class TestBigQueryIncrementalConstraintsRollback( From be4f536294fefa9fcac8aefc4a61f7b74a506915 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 21 Mar 2023 12:33:30 -0500 Subject: [PATCH 4/5] add incremental model --- tests/functional/adapter/test_constraints.py | 41 +++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 57d9dff83..266b97b0d 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -11,11 +11,12 @@ ) from dbt.tests.adapter.constraints.fixtures import ( my_model_sql, + my_incremental_model_sql, my_model_wrong_order_sql, - my_model_wrong_name_sql, my_model_view_wrong_order_sql, - my_model_view_wrong_name_sql, my_model_incremental_wrong_order_sql, + my_model_wrong_name_sql, + my_model_view_wrong_name_sql, my_model_incremental_wrong_name_sql, model_schema_yml, ) @@ -110,19 +111,6 @@ def models(self): } -class SetupBigQueryConstraintsRollback: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "constraints_schema.yml": constraints_yml, - } - - @pytest.fixture(scope="class") - def expected_error_messages(self): - return ["Required field id cannot be null"] - - class TestBigQueryTableConstraintsRuntimeDdlEnforcement( BaseConstraintsRuntimeDdlEnforcement ): @@ -139,11 +127,18 @@ def expected_sql(self, project): class TestBigQueryTableConstraintsRollback( - SetupBigQueryConstraintsRollback, BaseConstraintsRollback ): - pass + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": constraints_yml, + } + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["Required field id cannot be null"] class TestBigQueryIncrementalConstraintsRuntimeDdlEnforcement( BaseIncrementalConstraintsRuntimeDdlEnforcement @@ -161,7 +156,15 @@ def expected_sql(self, project): class TestBigQueryIncrementalConstraintsRollback( - SetupBigQueryConstraintsRollback, BaseIncrementalConstraintsRollback ): - pass + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": constraints_yml, + } + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["Required field id cannot be null"] \ No newline at end of file From 58466db11f8e6f2565d29503cae4bc09d89b3a16 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 27 Mar 2023 14:13:13 -0500 Subject: [PATCH 5/5] reset dev reqs --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3be7bee14..4a982392e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@er/ct-2293-incremental-mat#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@er/ct-2293-incremental-mat#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor