From ef436db69d3a374b92e1f1a960b270efb846e79d Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 1 Feb 2024 13:00:47 -0500 Subject: [PATCH 01/11] first pass: test unit testing on snowflake --- .../snowflake/macros/utils/safe_cast.sql | 3 ++- dev-requirements.txt | 3 ++- .../adapter/unit_testing/test_unit_testing.py | 27 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/functional/adapter/unit_testing/test_unit_testing.py diff --git a/dbt/include/snowflake/macros/utils/safe_cast.sql b/dbt/include/snowflake/macros/utils/safe_cast.sql index 65f9265a2..27a748329 100644 --- a/dbt/include/snowflake/macros/utils/safe_cast.sql +++ b/dbt/include/snowflake/macros/utils/safe_cast.sql @@ -1,3 +1,4 @@ {% macro snowflake__safe_cast(field, type) %} - try_cast({{field}} as {{type}}) + {% set field_as_sting = dbt.string_literal(field) if field is number else field %} + try_cast({{field_as_sting}} as {{type}}) {% endmacro %} diff --git a/dev-requirements.txt b/dev-requirements.txt index c19c587f3..fb4be52be 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,8 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-adapters.git@unit-testing-case-insensitive-comparisons +git+https://github.com/dbt-labs/dbt-adapters.git@unit-testing-case-insensitive-comparisons#subdirectory=dbt-tests-adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py new file mode 100644 index 000000000..046cda1e9 --- /dev/null +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -0,0 +1,27 @@ +import pytest + +from dbt.tests.adapter.unit_testing.test_unit_testing import BaseUnitTestingTypes + + +class TestSnowflakeUnitTestingTypes(BaseUnitTestingTypes): + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["'12345'", "12345"], + ["'string'", "string"], + ["true", "true"], + ["DATE '2020-01-02'", "2020-01-02"], + ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["'2013-11-03 00:00:00-0'::TIMESTAMPTZ", "2013-11-03 00:00:00-0"], + ["TO_NUMBER('3', 10, 9)", "3"], + # ["3::VARIANT", "3"], + # [ + # """TO_JSON(PARSE_JSON('{"bar": "baz", "balance": 7.77, "active": false}'))""", + # """'TO_JSON(PARSE_JSON('{"bar": "baz", "balance": 7.77, "active": false}'))'""", + # ], + # TODO: support complex types + # ["ARRAY['a','b','c']", """'{"a", "b", "c"}'"""], + # ["ARRAY[1,2,3]", """'{1, 2, 3}'"""], + ] From 694c515d433edef51974715ed92077fc568a379d Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 1 Feb 2024 13:11:47 -0500 Subject: [PATCH 02/11] array and json support --- .../adapter/unit_testing/test_unit_testing.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index 046cda1e9..73d67e47a 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -17,11 +17,10 @@ def data_types(self): ["'2013-11-03 00:00:00-0'::TIMESTAMPTZ", "2013-11-03 00:00:00-0"], ["TO_NUMBER('3', 10, 9)", "3"], # ["3::VARIANT", "3"], - # [ - # """TO_JSON(PARSE_JSON('{"bar": "baz", "balance": 7.77, "active": false}'))""", - # """'TO_JSON(PARSE_JSON('{"bar": "baz", "balance": 7.77, "active": false}'))'""", - # ], - # TODO: support complex types - # ["ARRAY['a','b','c']", """'{"a", "b", "c"}'"""], - # ["ARRAY[1,2,3]", """'{1, 2, 3}'"""], + [ + "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", + "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", + ], + ["['a','b','c']", "['a','b','c']"], + ["[1,2,3]", "[1, 2, 3]"], ] From a2e1987184f3c8c22cfcbc7dfeb04607cf01bbdf Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 1 Feb 2024 13:22:55 -0500 Subject: [PATCH 03/11] variant support --- dbt/include/snowflake/macros/utils/safe_cast.sql | 8 ++++++-- .../functional/adapter/unit_testing/test_unit_testing.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dbt/include/snowflake/macros/utils/safe_cast.sql b/dbt/include/snowflake/macros/utils/safe_cast.sql index 27a748329..fee4cc8d9 100644 --- a/dbt/include/snowflake/macros/utils/safe_cast.sql +++ b/dbt/include/snowflake/macros/utils/safe_cast.sql @@ -1,4 +1,8 @@ {% macro snowflake__safe_cast(field, type) %} - {% set field_as_sting = dbt.string_literal(field) if field is number else field %} - try_cast({{field_as_sting}} as {{type}}) + {% if (type|upper != "VARIANT") -%} + {% set field_as_sting = dbt.string_literal(field) if field is number else field %} + try_cast({{field_as_sting}} as {{type}}) + {% else -%} + cast({{field}} as {{type}}) + {% endif -%} {% endmacro %} diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index 73d67e47a..6fb8cc93b 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -9,6 +9,7 @@ def data_types(self): # sql_value, yaml_value return [ ["1", "1"], + ["2.0", "2.0"], ["'12345'", "12345"], ["'string'", "string"], ["true", "true"], @@ -16,7 +17,7 @@ def data_types(self): ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], ["'2013-11-03 00:00:00-0'::TIMESTAMPTZ", "2013-11-03 00:00:00-0"], ["TO_NUMBER('3', 10, 9)", "3"], - # ["3::VARIANT", "3"], + ["3::VARIANT", "3"], [ "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", From 9a2bfe840d13bad2a2ef9aae69151268852155a5 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 1 Feb 2024 17:12:14 -0500 Subject: [PATCH 04/11] refactor to implement cast --- dbt/include/snowflake/macros/utils/cast.sql | 9 +++++++++ dbt/include/snowflake/macros/utils/safe_cast.sql | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 dbt/include/snowflake/macros/utils/cast.sql diff --git a/dbt/include/snowflake/macros/utils/cast.sql b/dbt/include/snowflake/macros/utils/cast.sql new file mode 100644 index 000000000..212361a6d --- /dev/null +++ b/dbt/include/snowflake/macros/utils/cast.sql @@ -0,0 +1,9 @@ +{% macro snowflake__cast(field, type) %} + {% if (type|upper == "GEOGRAPHY") -%} + to_geography({{field}} as {{type}}) + {% elif (type|upper == "GEOMETRY") -%} + to_geometry({{field}} as {{type}}) + {% else -%} + cast({{field}} as {{type}}) + {% endif -%} +{% endmacro %} diff --git a/dbt/include/snowflake/macros/utils/safe_cast.sql b/dbt/include/snowflake/macros/utils/safe_cast.sql index fee4cc8d9..acb36c107 100644 --- a/dbt/include/snowflake/macros/utils/safe_cast.sql +++ b/dbt/include/snowflake/macros/utils/safe_cast.sql @@ -3,6 +3,6 @@ {% set field_as_sting = dbt.string_literal(field) if field is number else field %} try_cast({{field_as_sting}} as {{type}}) {% else -%} - cast({{field}} as {{type}}) + {{ adapter.dispatch('cast', 'dbt')(field, type) }} {% endif -%} {% endmacro %} From cd2513d68ae6b28c83cb0cb85926077b5ee193fd Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 1 Feb 2024 17:32:40 -0500 Subject: [PATCH 05/11] geography and geometry types + cast refactor --- dbt/include/snowflake/macros/utils/cast.sql | 4 ++-- dbt/include/snowflake/macros/utils/safe_cast.sql | 6 +++++- tests/functional/adapter/unit_testing/test_unit_testing.py | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/dbt/include/snowflake/macros/utils/cast.sql b/dbt/include/snowflake/macros/utils/cast.sql index 212361a6d..3da7c3aec 100644 --- a/dbt/include/snowflake/macros/utils/cast.sql +++ b/dbt/include/snowflake/macros/utils/cast.sql @@ -1,8 +1,8 @@ {% macro snowflake__cast(field, type) %} {% if (type|upper == "GEOGRAPHY") -%} - to_geography({{field}} as {{type}}) + to_geography({{field}}) {% elif (type|upper == "GEOMETRY") -%} - to_geometry({{field}} as {{type}}) + to_geometry({{field}}) {% else -%} cast({{field}} as {{type}}) {% endif -%} diff --git a/dbt/include/snowflake/macros/utils/safe_cast.sql b/dbt/include/snowflake/macros/utils/safe_cast.sql index acb36c107..0ce6a1088 100644 --- a/dbt/include/snowflake/macros/utils/safe_cast.sql +++ b/dbt/include/snowflake/macros/utils/safe_cast.sql @@ -1,7 +1,11 @@ {% macro snowflake__safe_cast(field, type) %} - {% if (type|upper != "VARIANT") -%} + {% if type|upper == "VARIANT" -%} {% set field_as_sting = dbt.string_literal(field) if field is number else field %} try_cast({{field_as_sting}} as {{type}}) + {% elif type|upper == "GEOMETRY" -%} + try_to_geometry({{field}}) + {% elif type|upper == "GEOGRAPHY" -%} + try_to_geography({{field}}) {% else -%} {{ adapter.dispatch('cast', 'dbt')(field, type) }} {% endif -%} diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index 6fb8cc93b..2a6b3a7f0 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -18,6 +18,8 @@ def data_types(self): ["'2013-11-03 00:00:00-0'::TIMESTAMPTZ", "2013-11-03 00:00:00-0"], ["TO_NUMBER('3', 10, 9)", "3"], ["3::VARIANT", "3"], + ["TO_GEOMETRY('POINT(1820.12 890.56)')", "POINT(1820.12 890.56)"], + ["TO_GEOGRAPHY('POINT(-122.35 37.55)')", "POINT(-122.35 37.55)"], [ "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", From e63874a31f89f0c468fb02a65ba848ec4d8b29ff Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 1 Feb 2024 22:47:40 -0500 Subject: [PATCH 06/11] fix variant logic in safe_cast --- dbt/include/snowflake/macros/utils/safe_cast.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dbt/include/snowflake/macros/utils/safe_cast.sql b/dbt/include/snowflake/macros/utils/safe_cast.sql index 0ce6a1088..e4d88c51e 100644 --- a/dbt/include/snowflake/macros/utils/safe_cast.sql +++ b/dbt/include/snowflake/macros/utils/safe_cast.sql @@ -1,11 +1,12 @@ {% macro snowflake__safe_cast(field, type) %} - {% if type|upper == "VARIANT" -%} - {% set field_as_sting = dbt.string_literal(field) if field is number else field %} - try_cast({{field_as_sting}} as {{type}}) - {% elif type|upper == "GEOMETRY" -%} + {% if type|upper == "GEOMETRY" -%} try_to_geometry({{field}}) {% elif type|upper == "GEOGRAPHY" -%} try_to_geography({{field}}) + {% elif type|upper != "VARIANT" -%} + {#-- Snowflake try_cast does not support casting to variant, and expects the field as a string --#} + {% set field_as_sting = dbt.string_literal(field) if field is number else field %} + try_cast({{field_as_sting}} as {{type}}) {% else -%} {{ adapter.dispatch('cast', 'dbt')(field, type) }} {% endif -%} From 3e4dd2a9ddbd4f69677da6176707aaea8e7711ca Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 5 Feb 2024 17:14:22 -0500 Subject: [PATCH 07/11] typo --- dbt/include/snowflake/macros/utils/safe_cast.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/snowflake/macros/utils/safe_cast.sql b/dbt/include/snowflake/macros/utils/safe_cast.sql index e4d88c51e..6ff4e351e 100644 --- a/dbt/include/snowflake/macros/utils/safe_cast.sql +++ b/dbt/include/snowflake/macros/utils/safe_cast.sql @@ -5,8 +5,8 @@ try_to_geography({{field}}) {% elif type|upper != "VARIANT" -%} {#-- Snowflake try_cast does not support casting to variant, and expects the field as a string --#} - {% set field_as_sting = dbt.string_literal(field) if field is number else field %} - try_cast({{field_as_sting}} as {{type}}) + {% set field_as_string = dbt.string_literal(field) if field is number else field %} + try_cast({{field_as_string}} as {{type}}) {% else -%} {{ adapter.dispatch('cast', 'dbt')(field, type) }} {% endif -%} From bae866de290b7f4ba32b8a4631ee4c99d47c6c37 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 5 Feb 2024 17:35:19 -0500 Subject: [PATCH 08/11] TestSnowflakeUnitTestCaseInsensitivity --- tests/functional/adapter/unit_testing/test_unit_testing.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index 2a6b3a7f0..8e6479916 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -1,6 +1,7 @@ import pytest from dbt.tests.adapter.unit_testing.test_unit_testing import BaseUnitTestingTypes +from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity class TestSnowflakeUnitTestingTypes(BaseUnitTestingTypes): @@ -27,3 +28,7 @@ def data_types(self): ["['a','b','c']", "['a','b','c']"], ["[1,2,3]", "[1, 2, 3]"], ] + + +class TestSnowflakeUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass From 0e35e7b4cef29e57652360d9d1614c76a33a9cbf Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 5 Feb 2024 17:48:20 -0500 Subject: [PATCH 09/11] changelog entry --- .changes/unreleased/Features-20240205-174816.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20240205-174816.yaml diff --git a/.changes/unreleased/Features-20240205-174816.yaml b/.changes/unreleased/Features-20240205-174816.yaml new file mode 100644 index 000000000..5cf6d41f2 --- /dev/null +++ b/.changes/unreleased/Features-20240205-174816.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support primative types + object, variant, array in snowflake for unit testing" +time: 2024-02-05T17:48:16.118398-05:00 +custom: + Author: michelleark + Issue: "898" From 9af4fee1e2f3cdd615a8352ad1b4ce0bca4c145c Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 6 Feb 2024 16:38:11 -0500 Subject: [PATCH 10/11] TestSnowflakeUnitTestInvalidInput --- tests/functional/adapter/unit_testing/test_unit_testing.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index 8e6479916..b97be0ac2 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -1,7 +1,8 @@ import pytest -from dbt.tests.adapter.unit_testing.test_unit_testing import BaseUnitTestingTypes +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity +from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput class TestSnowflakeUnitTestingTypes(BaseUnitTestingTypes): @@ -32,3 +33,7 @@ def data_types(self): class TestSnowflakeUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): pass + + +class TestSnowflakeUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass From 925f76fdd6fdc52280957290eda90603c9196bb3 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 8 Feb 2024 22:26:36 -0500 Subject: [PATCH 11/11] restore dev-requirements --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index fb4be52be..db49d0497 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,8 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git@unit-testing-case-insensitive-comparisons -git+https://github.com/dbt-labs/dbt-adapters.git@unit-testing-case-insensitive-comparisons#subdirectory=dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-adapters.git +git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor