From 977b8414ecf15312e9835f6054fbec214e9f9d1f Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 6 Sep 2022 09:27:24 -0700 Subject: [PATCH 01/35] Consolidate date macros into dates.sql --- .../global_project/macros/adapters/dates.sql | 45 +++++++++++++++++++ .../macros/adapters/freshness.sql | 10 ----- .../materializations/snapshots/strategies.sql | 13 ------ 3 files changed, 45 insertions(+), 23 deletions(-) create mode 100644 core/dbt/include/global_project/macros/adapters/dates.sql diff --git a/core/dbt/include/global_project/macros/adapters/dates.sql b/core/dbt/include/global_project/macros/adapters/dates.sql new file mode 100644 index 00000000000..f20967d2bab --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/dates.sql @@ -0,0 +1,45 @@ +{%- macro current_timestamp() -%} + {{ adapter.dispatch('current_timestamp', 'dbt')() }} +{%- endmacro -%} + +{% macro default__current_timestamp() %} + current_timestamp +{% endmacro %} + +{%- macro current_timestamp_in_utc() -%} + {{ adapter.dispatch('current_timestamp_in_utc', 'dbt')() }} +{%- endmacro -%} + +{% macro default__current_timestamp_in_utc() %} + {{ default__current_timestamp() }} at time zone 'utc' +{% endmacro %} + +{%- macro snapshot_get_time() -%} + {{ adapter.dispatch('snapshot_get_time', 'dbt')() }} +{%- endmacro -%} + +{% macro default__snapshot_get_time() %} + {{ current_timestamp() }} +{% endmacro %} + +{%- macro convert_timezone(timestamp, target_tz, source_tz) -%} + {%- if not target_tz is string -%} + {{ exceptions.raise_compiler_error("'target_tz' must be a string") }} + {%- else -%} + {{ adapter.dispatch('convert_timezone', 'dbt') (column, target_tz, source_tz) }} + {%- endif -%} + +{%- endmacro -%} + +{%- macro default__convert_timezone(column, target_tz, source_tz) -%} + {%- if not source_tz -%} + {{ column }} at time zone '{{ target_tz }}' + {%- else -%} + {{ column }} at time zone '{{ source_tz }}' at time zone '{{ target_tz }}' + {%- endif -%} +{%- endmacro -%} + +{%- macro now(tz=None) -%} +{{ convert_timezone(current_timestamp_in_utc(), tz) }} +{%- endmacro -%} + diff --git a/core/dbt/include/global_project/macros/adapters/freshness.sql b/core/dbt/include/global_project/macros/adapters/freshness.sql index fe8e7d7a1fe..6a5bd79d1d0 100644 --- a/core/dbt/include/global_project/macros/adapters/freshness.sql +++ b/core/dbt/include/global_project/macros/adapters/freshness.sql @@ -1,13 +1,3 @@ -{% macro current_timestamp() -%} - {{ adapter.dispatch('current_timestamp', 'dbt')() }} -{%- endmacro %} - -{% macro default__current_timestamp() -%} - {{ exceptions.raise_not_implemented( - 'current_timestamp macro not implemented for adapter '+adapter.type()) }} -{%- endmacro %} - - {% macro collect_freshness(source, loaded_at_field, filter) %} {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql b/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index 74e0f3c19a2..c78a7529a42 100644 --- a/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql +++ b/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -46,19 +46,6 @@ {%- endfor -%}) {%- endmacro %} - -{# - Get the current time cross-db -#} -{% macro snapshot_get_time() -%} - {{ adapter.dispatch('snapshot_get_time', 'dbt')() }} -{%- endmacro %} - -{% macro default__snapshot_get_time() -%} - {{ current_timestamp() }} -{%- endmacro %} - - {# Core strategy definitions #} From b0a6eb4bca213d4e85b57db0b36d3fc2a509706b Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 14 Sep 2022 09:24:36 -0700 Subject: [PATCH 02/35] rename to timestamps.sql --- .../global_project/macros/adapters/{dates.sql => timestamps.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename core/dbt/include/global_project/macros/adapters/{dates.sql => timestamps.sql} (100%) diff --git a/core/dbt/include/global_project/macros/adapters/dates.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql similarity index 100% rename from core/dbt/include/global_project/macros/adapters/dates.sql rename to core/dbt/include/global_project/macros/adapters/timestamps.sql From bd4b70576929ac425657e6375f0de6444dfd2629 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 14 Sep 2022 09:56:38 -0700 Subject: [PATCH 03/35] fix whitespace + add changie --- .changes/unreleased/Features-20220914-095625.yaml | 7 +++++++ .../include/global_project/macros/adapters/timestamps.sql | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20220914-095625.yaml diff --git a/.changes/unreleased/Features-20220914-095625.yaml b/.changes/unreleased/Features-20220914-095625.yaml new file mode 100644 index 00000000000..61196532985 --- /dev/null +++ b/.changes/unreleased/Features-20220914-095625.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Add standard timestamps.sql +time: 2022-09-14T09:56:25.97818-07:00 +custom: + Author: colin-rogers-dbt + Issue: "5521" + PR: "5838" diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index f20967d2bab..894ea3f20b8 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -42,4 +42,3 @@ {%- macro now(tz=None) -%} {{ convert_timezone(current_timestamp_in_utc(), tz) }} {%- endmacro -%} - From 580d70f6e314070ecc761de9ba0b55671f932fcd Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 19 Sep 2022 18:28:56 -0700 Subject: [PATCH 04/35] cleanup macros and add testing --- core/dbt/dbt-oss.code-workspace | 35 +++++++++++++ .../macros/adapters/timestamps.sql | 50 ++++++++++++++++--- core/dbt/tests/util.py | 11 +++- .../dbt/include/postgres/macros/adapters.sql | 15 ------ .../include/postgres/macros/timestamps.sql | 12 +++++ .../functional/timestamps/test_timestamps.py | 29 +++++++++++ 6 files changed, 129 insertions(+), 23 deletions(-) create mode 100644 core/dbt/dbt-oss.code-workspace create mode 100644 plugins/postgres/dbt/include/postgres/macros/timestamps.sql create mode 100644 tests/functional/timestamps/test_timestamps.py diff --git a/core/dbt/dbt-oss.code-workspace b/core/dbt/dbt-oss.code-workspace new file mode 100644 index 00000000000..8d05b957a6a --- /dev/null +++ b/core/dbt/dbt-oss.code-workspace @@ -0,0 +1,35 @@ +{ + "folders": [ + { + "path": "../.." + }, + { + "path": "../../../dbt-utils" + }, + { + "path": "../../../dbt-redshift" + }, + { + "path": "../../../dbt-bigquery" + }, + { + "path": "../../../dbt-snowflake" + }, + { + "path": "../../../dbt-spark" + } + ], + "settings": { + "makefile.extensionOutputFolder": "./.vscode", + "pytestersuits.pythonDefaultTestLibrary": "pytest", + "python.autoComplete.extraPaths": [], + "python.analysis.autoSearchPaths": true, + "python.analysis.extraPaths": [ + "core", + "dbt-core/core" + ], + "python.analysis.indexing": true, + "python.analysis.inlayHints.functionReturnTypes": true, + "python.analysis.inlayHints.variableTypes": true + } +} \ No newline at end of file diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index 894ea3f20b8..baf396f65b1 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -11,7 +11,8 @@ {%- endmacro -%} {% macro default__current_timestamp_in_utc() %} - {{ default__current_timestamp() }} at time zone 'utc' + {{ convert_timezone(target_tz="UTC", + timestamp=current_timestamp())}} {% endmacro %} {%- macro snapshot_get_time() -%} @@ -22,23 +23,58 @@ {{ current_timestamp() }} {% endmacro %} -{%- macro convert_timezone(timestamp, target_tz, source_tz) -%} +{%- macro convert_timezone(source_tz, target_tz, timestamp) -%} {%- if not target_tz is string -%} {{ exceptions.raise_compiler_error("'target_tz' must be a string") }} {%- else -%} - {{ adapter.dispatch('convert_timezone', 'dbt') (column, target_tz, source_tz) }} + {{ adapter.dispatch('convert_timezone', 'dbt') (source_tz, target_tz, timestamp) }} {%- endif -%} {%- endmacro -%} -{%- macro default__convert_timezone(column, target_tz, source_tz) -%} +{%- macro default__convert_timezone(source_tz, target_tz, timestamp) -%} {%- if not source_tz -%} - {{ column }} at time zone '{{ target_tz }}' + {{ timestamp }} at time zone '{{ target_tz }}' {%- else -%} - {{ column }} at time zone '{{ source_tz }}' at time zone '{{ target_tz }}' + {{ timestamp }} at time zone '{{ source_tz }}' at time zone '{{ target_tz }}' {%- endif -%} {%- endmacro -%} {%- macro now(tz=None) -%} -{{ convert_timezone(current_timestamp_in_utc(), tz) }} + {%- if not source_tz -%} + {{ current_timestamp() }} + {%- else -%} + {{ convert_timezone(current_timestamp_in_utc(), tz) }} + {%- endif -%} {%- endmacro -%} + + +--------------------------------------------- + +/* {# + DEPRECATED: DO NOT USE IN NEW PROJECTS + + This is ONLY to handle the fact that Snowflake + Postgres had functionally + different implementations of {{ dbt.current_timestamp }} + {{ dbt_utils.current_timestamp }} + + If you had a project or package that called {{ dbt_utils.current_timestamp() }}, you should + continue to use this macro to guarantee identical behavior on those two databases. +#} */ + +{% macro current_timestamp_backcompat() %} + {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }} +{% endmacro %} + +{% macro default__current_timestamp_backcompat() %} + {{ return(adapter.dispatch('current_timestamp', 'dbt')()) }} +{% endmacro %} + +-- TODO move to dbt-postgres +{% macro postgres__current_timestamp_backcompat() %} + current_timestamp::{{ type_timestamp() }} +{% endmacro %} + +-- TODO move to dbt-snowflake +{% macro snowflake__current_timestamp_backcompat() %} + current_timestamp::{{ type_timestamp() }} +{% endmacro %} diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index 311a2249f7c..8c8e3d065e0 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -4,7 +4,7 @@ import json import warnings from datetime import datetime -from typing import List +from typing import Dict, List from contextlib import contextmanager from dbt.main import handle_and_check @@ -316,6 +316,15 @@ def check_relations_equal(adapter, relation_names, compare_snapshot_cols=False): ) +# Used to check that a particular relation has an expected schema +# expected_schema should look like {"column_name": "expected datatype"} +def check_relation_has_expected_schema(adapter, relation_name, expected_schema: Dict): + relation = relation_from_name(adapter, relation_name) + with get_connection(adapter): + actual_columns = {c.name: c.data_type for c in adapter.get_columns_in_relation(relation)} + assert actual_columns == expected_schema + + # This can be used when checking relations in different schemas, by supplying # a list of relations. Called by 'check_relations_equal'. # Uses: diff --git a/plugins/postgres/dbt/include/postgres/macros/adapters.sql b/plugins/postgres/dbt/include/postgres/macros/adapters.sql index 6b0ee3a7d0f..211365dee11 100644 --- a/plugins/postgres/dbt/include/postgres/macros/adapters.sql +++ b/plugins/postgres/dbt/include/postgres/macros/adapters.sql @@ -117,21 +117,6 @@ {{ return(load_result('check_schema_exists').table) }} {% endmacro %} - -{% macro postgres__current_timestamp() -%} - now() -{%- endmacro %} - -{% macro postgres__snapshot_string_as_time(timestamp) -%} - {%- set result = "'" ~ timestamp ~ "'::timestamp without time zone" -%} - {{ return(result) }} -{%- endmacro %} - - -{% macro postgres__snapshot_get_time() -%} - {{ current_timestamp() }}::timestamp without time zone -{%- endmacro %} - {# Postgres tables have a maximum length off 63 characters, anything longer is silently truncated. Temp and backup relations add a lot of extra characters to the end of table names to ensure uniqueness. diff --git a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql new file mode 100644 index 00000000000..b947fe0e70d --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql @@ -0,0 +1,12 @@ +{% macro postgres__current_timestamp() -%} + now() +{%- endmacro %} + +{% macro postgres__snapshot_string_as_time(timestamp) -%} + {%- set result = "'" ~ timestamp ~ "'::timestamp without time zone" -%} + {{ return(result) }} +{%- endmacro %} + +{% macro postgres__snapshot_get_time() -%} + {{ current_timestamp() }} :: TIMESTAMP without TIME ZONE +{%- endmacro %} diff --git a/tests/functional/timestamps/test_timestamps.py b/tests/functional/timestamps/test_timestamps.py new file mode 100644 index 00000000000..b395f827399 --- /dev/null +++ b/tests/functional/timestamps/test_timestamps.py @@ -0,0 +1,29 @@ +import pytest +from dbt.tests.util import check_relation_has_expected_schema, run_dbt + +_MODEL_CURRENT_TIMESTAMP = """ +SELECT {{current_timestamp()}} as current_timestamp, + {{current_timestamp_in_utc()}} as current_timestamp_in_utc, + {{now()}} as now_no_tz, + {{now(tz='EST')}} as now_w_tz +""" + + +class TestCurrentTimestamps: + @pytest.fixture(scope="class") + def models(self): + return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} + + def test_current_timestamps(self, project, models): + results = run_dbt(["run"]) + assert len(results) == 1 + check_relation_has_expected_schema( + project.adapter, + relation_name="get_current_timestamp", + expected_schema={ + "current_timestamp": "timestamp with time zone", + "current_timestamp_in_utc": "timestamp without time zone", + "now_w_tz": "timestamp with time zone", + "now_no_tz": "timestamp with time zone", + }, + ) From cd439ba4a2fbcc0a1338cb33da02431587a025a4 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 20 Sep 2022 10:43:52 -0700 Subject: [PATCH 05/35] fix whitespace --- tests/functional/timestamps/test_timestamps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/timestamps/test_timestamps.py b/tests/functional/timestamps/test_timestamps.py index b395f827399..bd5e5021e68 100644 --- a/tests/functional/timestamps/test_timestamps.py +++ b/tests/functional/timestamps/test_timestamps.py @@ -2,7 +2,7 @@ from dbt.tests.util import check_relation_has_expected_schema, run_dbt _MODEL_CURRENT_TIMESTAMP = """ -SELECT {{current_timestamp()}} as current_timestamp, +SELECT {{current_timestamp()}} as current_timestamp, {{current_timestamp_in_utc()}} as current_timestamp_in_utc, {{now()}} as now_no_tz, {{now(tz='EST')}} as now_w_tz From 44c18097054e28bfba71ed13e46725a677fca323 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 20 Sep 2022 16:15:27 -0700 Subject: [PATCH 06/35] remove now macro --- .../include/global_project/macros/adapters/timestamps.sql | 8 -------- tests/functional/timestamps/test_timestamps.py | 2 -- 2 files changed, 10 deletions(-) diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index baf396f65b1..9ba6e1d10cb 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -40,14 +40,6 @@ {%- endif -%} {%- endmacro -%} -{%- macro now(tz=None) -%} - {%- if not source_tz -%} - {{ current_timestamp() }} - {%- else -%} - {{ convert_timezone(current_timestamp_in_utc(), tz) }} - {%- endif -%} -{%- endmacro -%} - --------------------------------------------- diff --git a/tests/functional/timestamps/test_timestamps.py b/tests/functional/timestamps/test_timestamps.py index bd5e5021e68..1a2fffdb23a 100644 --- a/tests/functional/timestamps/test_timestamps.py +++ b/tests/functional/timestamps/test_timestamps.py @@ -23,7 +23,5 @@ def test_current_timestamps(self, project, models): expected_schema={ "current_timestamp": "timestamp with time zone", "current_timestamp_in_utc": "timestamp without time zone", - "now_w_tz": "timestamp with time zone", - "now_no_tz": "timestamp with time zone", }, ) From 3c47be6b6ec5f370dc91ffa09fd7b79d57190f47 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 20 Sep 2022 17:06:44 -0700 Subject: [PATCH 07/35] fix functional test --- tests/functional/timestamps/test_timestamps.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/functional/timestamps/test_timestamps.py b/tests/functional/timestamps/test_timestamps.py index 1a2fffdb23a..3c440cef1c1 100644 --- a/tests/functional/timestamps/test_timestamps.py +++ b/tests/functional/timestamps/test_timestamps.py @@ -3,9 +3,7 @@ _MODEL_CURRENT_TIMESTAMP = """ SELECT {{current_timestamp()}} as current_timestamp, - {{current_timestamp_in_utc()}} as current_timestamp_in_utc, - {{now()}} as now_no_tz, - {{now(tz='EST')}} as now_w_tz + {{current_timestamp_in_utc()}} as current_timestamp_in_utc """ From 9404bba195474b75a69b6051aa31b3b174b9019a Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 21 Sep 2022 14:36:25 -0700 Subject: [PATCH 08/35] remove local config --- core/dbt/dbt-oss.code-workspace | 35 --------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 core/dbt/dbt-oss.code-workspace diff --git a/core/dbt/dbt-oss.code-workspace b/core/dbt/dbt-oss.code-workspace deleted file mode 100644 index 8d05b957a6a..00000000000 --- a/core/dbt/dbt-oss.code-workspace +++ /dev/null @@ -1,35 +0,0 @@ -{ - "folders": [ - { - "path": "../.." - }, - { - "path": "../../../dbt-utils" - }, - { - "path": "../../../dbt-redshift" - }, - { - "path": "../../../dbt-bigquery" - }, - { - "path": "../../../dbt-snowflake" - }, - { - "path": "../../../dbt-spark" - } - ], - "settings": { - "makefile.extensionOutputFolder": "./.vscode", - "pytestersuits.pythonDefaultTestLibrary": "pytest", - "python.autoComplete.extraPaths": [], - "python.analysis.autoSearchPaths": true, - "python.analysis.extraPaths": [ - "core", - "dbt-core/core" - ], - "python.analysis.indexing": true, - "python.analysis.inlayHints.functionReturnTypes": true, - "python.analysis.inlayHints.variableTypes": true - } -} \ No newline at end of file From 471a63be5256d81d4721e20bd5b2ca23a34fcee3 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 22 Sep 2022 09:42:14 -0700 Subject: [PATCH 09/35] make snowflake backwards compat return utc --- core/dbt/include/global_project/macros/adapters/timestamps.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index 9ba6e1d10cb..b0fce033272 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -64,9 +64,10 @@ -- TODO move to dbt-postgres {% macro postgres__current_timestamp_backcompat() %} current_timestamp::{{ type_timestamp() }} + {% endmacro %} -- TODO move to dbt-snowflake {% macro snowflake__current_timestamp_backcompat() %} - current_timestamp::{{ type_timestamp() }} + convert_timezone('UTC', {{current_timestamp()}}) {% endmacro %} From d29dfb9a9761cfa6c726fd2799ec3478bf94d84e Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 22 Sep 2022 10:03:50 -0700 Subject: [PATCH 10/35] move timestamps to adaptor base tests --- .../tests/adapter/basic/test_timestamps.py | 25 ++++++++++++++++++ .../functional/timestamps/test_timestamps.py | 26 +++---------------- 2 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 tests/adapter/dbt/tests/adapter/basic/test_timestamps.py diff --git a/tests/adapter/dbt/tests/adapter/basic/test_timestamps.py b/tests/adapter/dbt/tests/adapter/basic/test_timestamps.py new file mode 100644 index 00000000000..3c440cef1c1 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/basic/test_timestamps.py @@ -0,0 +1,25 @@ +import pytest +from dbt.tests.util import check_relation_has_expected_schema, run_dbt + +_MODEL_CURRENT_TIMESTAMP = """ +SELECT {{current_timestamp()}} as current_timestamp, + {{current_timestamp_in_utc()}} as current_timestamp_in_utc +""" + + +class TestCurrentTimestamps: + @pytest.fixture(scope="class") + def models(self): + return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} + + def test_current_timestamps(self, project, models): + results = run_dbt(["run"]) + assert len(results) == 1 + check_relation_has_expected_schema( + project.adapter, + relation_name="get_current_timestamp", + expected_schema={ + "current_timestamp": "timestamp with time zone", + "current_timestamp_in_utc": "timestamp without time zone", + }, + ) diff --git a/tests/functional/timestamps/test_timestamps.py b/tests/functional/timestamps/test_timestamps.py index 3c440cef1c1..1ff07470de6 100644 --- a/tests/functional/timestamps/test_timestamps.py +++ b/tests/functional/timestamps/test_timestamps.py @@ -1,25 +1,5 @@ -import pytest -from dbt.tests.util import check_relation_has_expected_schema, run_dbt +from tests.adapter.dbt.tests.adapter.basic.test_timestamps import TestCurrentTimestamps -_MODEL_CURRENT_TIMESTAMP = """ -SELECT {{current_timestamp()}} as current_timestamp, - {{current_timestamp_in_utc()}} as current_timestamp_in_utc -""" - -class TestCurrentTimestamps: - @pytest.fixture(scope="class") - def models(self): - return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} - - def test_current_timestamps(self, project, models): - results = run_dbt(["run"]) - assert len(results) == 1 - check_relation_has_expected_schema( - project.adapter, - relation_name="get_current_timestamp", - expected_schema={ - "current_timestamp": "timestamp with time zone", - "current_timestamp_in_utc": "timestamp without time zone", - }, - ) +class TestCurrentTimestampsCore(TestCurrentTimestamps): + pass From c44b2aea633d94b268e868e4100c6db324a2e5c0 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 22 Sep 2022 10:14:42 -0700 Subject: [PATCH 11/35] move backcompat macros to respective adapters --- .../global_project/macros/adapters/timestamps.sql | 11 ----------- .../dbt/include/postgres/macros/timestamps.sql | 6 ++++++ 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index b0fce033272..1b30407b0e8 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -60,14 +60,3 @@ {% macro default__current_timestamp_backcompat() %} {{ return(adapter.dispatch('current_timestamp', 'dbt')()) }} {% endmacro %} - --- TODO move to dbt-postgres -{% macro postgres__current_timestamp_backcompat() %} - current_timestamp::{{ type_timestamp() }} - -{% endmacro %} - --- TODO move to dbt-snowflake -{% macro snowflake__current_timestamp_backcompat() %} - convert_timezone('UTC', {{current_timestamp()}}) -{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql index b947fe0e70d..aa9f331cf0a 100644 --- a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql +++ b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql @@ -10,3 +10,9 @@ {% macro postgres__snapshot_get_time() -%} {{ current_timestamp() }} :: TIMESTAMP without TIME ZONE {%- endmacro %} + +{% macro postgres__current_timestamp_backcompat() %} + current_timestamp::{{ type_timestamp() }} + +{% endmacro %} + From 2bcee965738b04c798f7382d20493ca0d587b46c Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 22 Sep 2022 10:46:54 -0700 Subject: [PATCH 12/35] change timestamp param to source_timestamp --- .../global_project/macros/adapters/timestamps.sql | 12 ++++++------ .../dbt/include/postgres/macros/timestamps.sql | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index 1b30407b0e8..524735fbb14 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -12,7 +12,7 @@ {% macro default__current_timestamp_in_utc() %} {{ convert_timezone(target_tz="UTC", - timestamp=current_timestamp())}} + source_timestamp=current_timestamp())}} {% endmacro %} {%- macro snapshot_get_time() -%} @@ -23,20 +23,20 @@ {{ current_timestamp() }} {% endmacro %} -{%- macro convert_timezone(source_tz, target_tz, timestamp) -%} +{%- macro convert_timezone(source_tz, target_tz, source_timestamp) -%} {%- if not target_tz is string -%} {{ exceptions.raise_compiler_error("'target_tz' must be a string") }} {%- else -%} - {{ adapter.dispatch('convert_timezone', 'dbt') (source_tz, target_tz, timestamp) }} + {{ adapter.dispatch('convert_timezone', 'dbt') (source_tz, target_tz, source_timestamp) }} {%- endif -%} {%- endmacro -%} -{%- macro default__convert_timezone(source_tz, target_tz, timestamp) -%} +{%- macro default__convert_timezone(source_tz, target_tz, source_timestamp) -%} {%- if not source_tz -%} - {{ timestamp }} at time zone '{{ target_tz }}' + {{ source_timestamp }} at time zone '{{ target_tz }}' {%- else -%} - {{ timestamp }} at time zone '{{ source_tz }}' at time zone '{{ target_tz }}' + {{ source_timestamp }} at time zone '{{ source_tz }}' at time zone '{{ target_tz }}' {%- endif -%} {%- endmacro -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql index aa9f331cf0a..67e6c41fbe5 100644 --- a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql +++ b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql @@ -13,6 +13,5 @@ {% macro postgres__current_timestamp_backcompat() %} current_timestamp::{{ type_timestamp() }} - -{% endmacro %} +{% endmacro %} From 33cd4493761494e265606da9b96d30780451c7f8 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 22 Sep 2022 12:20:13 -0700 Subject: [PATCH 13/35] move timestamps.py to utils --- .../dbt/tests/adapter/{basic => utils}/test_timestamps.py | 0 tests/functional/timestamps/test_timestamps.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/adapter/dbt/tests/adapter/{basic => utils}/test_timestamps.py (100%) diff --git a/tests/adapter/dbt/tests/adapter/basic/test_timestamps.py b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py similarity index 100% rename from tests/adapter/dbt/tests/adapter/basic/test_timestamps.py rename to tests/adapter/dbt/tests/adapter/utils/test_timestamps.py diff --git a/tests/functional/timestamps/test_timestamps.py b/tests/functional/timestamps/test_timestamps.py index 1ff07470de6..2dbcd5777b9 100644 --- a/tests/functional/timestamps/test_timestamps.py +++ b/tests/functional/timestamps/test_timestamps.py @@ -1,4 +1,4 @@ -from tests.adapter.dbt.tests.adapter.basic.test_timestamps import TestCurrentTimestamps +from tests.adapter.dbt.tests.adapter.utils.test_timestamps import TestCurrentTimestamps class TestCurrentTimestampsCore(TestCurrentTimestamps): From ad3b6456a764f072a4063ec5d94f12aea64f6cef Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 23 Sep 2022 09:58:57 -0700 Subject: [PATCH 14/35] update changie.yaml --- .changie.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.changie.yaml b/.changie.yaml index 8d2498ea8a9..b4a5f3d4ecd 100755 --- a/.changie.yaml +++ b/.changie.yaml @@ -5,15 +5,15 @@ versionHeaderPath: "" changelogPath: CHANGELOG.md versionExt: md versionFormat: '## dbt-core {{.Version}} - {{.Time.Format "January 02, 2006"}}' -kindFormat: '### {{.Kind}}' -changeFormat: '- {{.Body}} ([#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-core/issues/{{.Custom.Issue}}), [#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-core/pull/{{.Custom.PR}}))' +kindFormat: "### {{.Kind}}" +changeFormat: "- {{.Body}} ([#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-core/issues/{{.Custom.Issue}}), [#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-core/pull/{{.Custom.PR}}))" kinds: - label: Breaking Changes - label: Features - label: Fixes - label: Docs - changeFormat: '- {{.Body}} ([dbt-docs/#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-docs/issues/{{.Custom.Issue}}), [dbt-docs/#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-docs/pull/{{.Custom.PR}}))' + changeFormat: "- {{.Body}} ([dbt-docs/#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-docs/issues/{{.Custom.Issue}}), [dbt-docs/#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-docs/pull/{{.Custom.PR}}))" - label: Under the Hood - label: Dependencies changeFormat: '- {{.Body}} ({{if ne .Custom.Issue ""}}[#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-core/issues/{{.Custom.Issue}}), {{end}}[#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-core/pull/{{.Custom.PR}}))' @@ -28,23 +28,23 @@ newlines: endOfVersion: 1 custom: -- key: Author - label: GitHub Username(s) (separated by a single space if multiple) - type: string - minLength: 3 -- key: Issue - label: GitHub Issue Number - type: int - minInt: 1 -- key: PR - label: GitHub Pull Request Number - type: int - minInt: 1 + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: Issue + label: GitHub Issue Number + type: int + minInt: 1 + - key: PR + label: GitHub Pull Request Number + type: int + minInt: 1 footerFormat: | {{- $contributorDict := dict }} {{- /* any names added to this list should be all lowercase for later matching purposes */}} - {{- $core_team := list "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "dependabot[bot]" "snyk-bot" }} + {{- $core_team := list "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "dependabot[bot]" "snyk-bot" "colin-rogers-dbt" }} {{- range $change := .Changes }} {{- $authorList := splitList " " $change.Custom.Author }} {{- /* loop through all authors for a PR */}} From cfb445e4baf5efa3e0bb34ea8d8eaf185e0eca52 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 23 Sep 2022 15:02:34 -0700 Subject: [PATCH 15/35] make expected schema a fixture --- .../dbt/tests/adapter/utils/test_timestamps.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py index 3c440cef1c1..7bae1ae37ec 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py @@ -12,14 +12,18 @@ class TestCurrentTimestamps: def models(self): return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} - def test_current_timestamps(self, project, models): + @pytest.fixture(scope="class") + def expected_schema(self): + return { + "current_timestamp": "timestamp with time zone", + "current_timestamp_in_utc": "timestamp without time zone", + } + + def test_current_timestamps(self, project, models, expected_schema): results = run_dbt(["run"]) assert len(results) == 1 check_relation_has_expected_schema( project.adapter, relation_name="get_current_timestamp", - expected_schema={ - "current_timestamp": "timestamp with time zone", - "current_timestamp_in_utc": "timestamp without time zone", - }, + expected_schema=expected_schema, ) From 87b0aa5e7f007ea918ce70164fb52e4e9037d5e9 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 23 Sep 2022 17:13:06 -0700 Subject: [PATCH 16/35] formatting --- tests/adapter/dbt/tests/adapter/utils/test_timestamps.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py index 7bae1ae37ec..14eb32c5506 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py @@ -15,9 +15,9 @@ def models(self): @pytest.fixture(scope="class") def expected_schema(self): return { - "current_timestamp": "timestamp with time zone", - "current_timestamp_in_utc": "timestamp without time zone", - } + "current_timestamp": "timestamp with time zone", + "current_timestamp_in_utc": "timestamp without time zone", + } def test_current_timestamps(self, project, models, expected_schema): results = run_dbt(["run"]) From 60e95f03e8d123183eab5146d437444d8b96f05c Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 26 Sep 2022 14:27:08 -0700 Subject: [PATCH 17/35] add debug message to assert --- .changie.yaml | 30 +++++++++++++++--------------- core/dbt/tests/util.py | 4 +++- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.changie.yaml b/.changie.yaml index b4a5f3d4ecd..0fb72956dd2 100755 --- a/.changie.yaml +++ b/.changie.yaml @@ -5,15 +5,15 @@ versionHeaderPath: "" changelogPath: CHANGELOG.md versionExt: md versionFormat: '## dbt-core {{.Version}} - {{.Time.Format "January 02, 2006"}}' -kindFormat: "### {{.Kind}}" -changeFormat: "- {{.Body}} ([#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-core/issues/{{.Custom.Issue}}), [#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-core/pull/{{.Custom.PR}}))" +kindFormat: '### {{.Kind}}' +changeFormat: '- {{.Body}} ([#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-core/issues/{{.Custom.Issue}}), [#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-core/pull/{{.Custom.PR}}))' kinds: - label: Breaking Changes - label: Features - label: Fixes - label: Docs - changeFormat: "- {{.Body}} ([dbt-docs/#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-docs/issues/{{.Custom.Issue}}), [dbt-docs/#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-docs/pull/{{.Custom.PR}}))" + changeFormat: '- {{.Body}} ([dbt-docs/#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-docs/issues/{{.Custom.Issue}}), [dbt-docs/#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-docs/pull/{{.Custom.PR}}))' - label: Under the Hood - label: Dependencies changeFormat: '- {{.Body}} ({{if ne .Custom.Issue ""}}[#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-core/issues/{{.Custom.Issue}}), {{end}}[#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-core/pull/{{.Custom.PR}}))' @@ -28,18 +28,18 @@ newlines: endOfVersion: 1 custom: - - key: Author - label: GitHub Username(s) (separated by a single space if multiple) - type: string - minLength: 3 - - key: Issue - label: GitHub Issue Number - type: int - minInt: 1 - - key: PR - label: GitHub Pull Request Number - type: int - minInt: 1 +- key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 +- key: Issue + label: GitHub Issue Number + type: int + minInt: 1 +- key: PR + label: GitHub Pull Request Number + type: int + minInt: 1 footerFormat: | {{- $contributorDict := dict }} diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index b7b281e1eb3..d26204fa997 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -327,7 +327,9 @@ def check_relation_has_expected_schema(adapter, relation_name, expected_schema: relation = relation_from_name(adapter, relation_name) with get_connection(adapter): actual_columns = {c.name: c.data_type for c in adapter.get_columns_in_relation(relation)} - assert actual_columns == expected_schema + assert ( + actual_columns == expected_schema + ), f"Actual schema did not match expected, actual: {json.dumps(actual_columns)}" # This can be used when checking relations in different schemas, by supplying From a5db6a9914322090770f4c39d213d7ebb27d6052 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 26 Sep 2022 15:54:20 -0700 Subject: [PATCH 18/35] fix changie.yaml --- .changie.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changie.yaml b/.changie.yaml index 0fb72956dd2..9d938a9c519 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -44,7 +44,7 @@ custom: footerFormat: | {{- $contributorDict := dict }} {{- /* any names added to this list should be all lowercase for later matching purposes */}} - {{- $core_team := list "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "dependabot[bot]" "snyk-bot" "colin-rogers-dbt" }} + {{- $core_team := list "peterallenwebb" "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "dependabot[bot]" "snyk-bot" "colin-rogers-dbt" }} {{- range $change := .Changes }} {{- $authorList := splitList " " $change.Custom.Author }} {{- /* loop through all authors for a PR */}} From d91b0e3dfb5545d658926cdfa9bd175369c71aeb Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:58:38 -0700 Subject: [PATCH 19/35] Update tests/adapter/dbt/tests/adapter/utils/test_timestamps.py Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- tests/adapter/dbt/tests/adapter/utils/test_timestamps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py index 14eb32c5506..31472991ce4 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py @@ -2,8 +2,8 @@ from dbt.tests.util import check_relation_has_expected_schema, run_dbt _MODEL_CURRENT_TIMESTAMP = """ -SELECT {{current_timestamp()}} as current_timestamp, - {{current_timestamp_in_utc()}} as current_timestamp_in_utc +SELECT {{ current_timestamp() }} as current_timestamp, + {{ current_timestamp_in_utc() }} as current_timestamp_in_utc """ From 8c55ecd307b74b35ff878f8d6b52dc17cbabb137 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:58:44 -0700 Subject: [PATCH 20/35] Update plugins/postgres/dbt/include/postgres/macros/timestamps.sql Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- plugins/postgres/dbt/include/postgres/macros/timestamps.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql index 67e6c41fbe5..febe7c7edcd 100644 --- a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql +++ b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql @@ -8,7 +8,7 @@ {%- endmacro %} {% macro postgres__snapshot_get_time() -%} - {{ current_timestamp() }} :: TIMESTAMP without TIME ZONE + {{ current_timestamp() }}::timestamp without time zone {%- endmacro %} {% macro postgres__current_timestamp_backcompat() %} From ee97e7308be75903237b7f4b26e63121472b7039 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:44:35 -0700 Subject: [PATCH 21/35] Update .changie.yaml --- .changie.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changie.yaml b/.changie.yaml index 9d938a9c519..ab2570b0e2e 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -44,7 +44,7 @@ custom: footerFormat: | {{- $contributorDict := dict }} {{- /* any names added to this list should be all lowercase for later matching purposes */}} - {{- $core_team := list "peterallenwebb" "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "dependabot[bot]" "snyk-bot" "colin-rogers-dbt" }} + {{- $core_team := list "peterallenwebb" "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "dependabot[bot]" "snyk-bot" }} {{- range $change := .Changes }} {{- $authorList := splitList " " $change.Custom.Author }} {{- /* loop through all authors for a PR */}} From 7ce9b105f1e558deea7d4e1e700a7fc724369500 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 27 Sep 2022 10:17:19 -0700 Subject: [PATCH 22/35] add backcompat utc --- .../include/global_project/macros/adapters/timestamps.sql | 8 ++++++++ .../postgres/dbt/include/postgres/macros/timestamps.sql | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index 524735fbb14..e91e8bff954 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -60,3 +60,11 @@ {% macro default__current_timestamp_backcompat() %} {{ return(adapter.dispatch('current_timestamp', 'dbt')()) }} {% endmacro %} + +{% macro current_timestamp_in_utc_backcompat() %} + {{ return(adapter.dispatch('default__current_timestamp_in_utc_backcompat', 'dbt')()) }} +{% endmacro %} + +{% macro default__current_timestamp_in_utc_backcompat() %} + {{ return(adapter.dispatch('current_timestamp_in_utc', 'dbt')()) }} +{% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql index 67e6c41fbe5..29c44497abe 100644 --- a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql +++ b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql @@ -8,10 +8,13 @@ {%- endmacro %} {% macro postgres__snapshot_get_time() -%} - {{ current_timestamp() }} :: TIMESTAMP without TIME ZONE + {{ current_timestamp() }} :: timestamp without time zone {%- endmacro %} {% macro postgres__current_timestamp_backcompat() %} current_timestamp::{{ type_timestamp() }} +{% endmacro %} +{% macro postgres__current_timestamp_in_utc_backcompat() %} + (current_timestamp at time zone 'utc')::{{type_timestamp()}} {% endmacro %} From ad2124ccdc89c06bfe6464a8dec1d151f95b1444 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Sep 2022 14:29:36 -0700 Subject: [PATCH 23/35] remove current_timestamp_in_utc --- .../macros/adapters/timestamps.sql | 17 ++++++++--------- test/unit/test_macro_calls.py | 4 ++-- .../dbt/tests/adapter/utils/test_timestamps.py | 4 ++-- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index 524735fbb14..9f9fb1f9009 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -6,15 +6,6 @@ current_timestamp {% endmacro %} -{%- macro current_timestamp_in_utc() -%} - {{ adapter.dispatch('current_timestamp_in_utc', 'dbt')() }} -{%- endmacro -%} - -{% macro default__current_timestamp_in_utc() %} - {{ convert_timezone(target_tz="UTC", - source_timestamp=current_timestamp())}} -{% endmacro %} - {%- macro snapshot_get_time() -%} {{ adapter.dispatch('snapshot_get_time', 'dbt')() }} {%- endmacro -%} @@ -60,3 +51,11 @@ {% macro default__current_timestamp_backcompat() %} {{ return(adapter.dispatch('current_timestamp', 'dbt')()) }} {% endmacro %} + +{% macro current_timestamp_in_utc_backcompat() %} + {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }} +{% endmacro %} + +{% macro default__current_timestamp_in_utc_backcompat() %} + {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }} +{% endmacro %} diff --git a/test/unit/test_macro_calls.py b/test/unit/test_macro_calls.py index e5ffc4bd736..6fbbe786641 100644 --- a/test/unit/test_macro_calls.py +++ b/test/unit/test_macro_calls.py @@ -23,7 +23,7 @@ def setUp(self): * from {{ model }} ) {% endmacro %}""", - "{% macro test_my_test(model) %} select {{ dbt_utils.current_timestamp() }} {% endmacro %}", + "{% macro test_my_test(model) %} select {{ current_timestamp_backcompat() }} {% endmacro %}", "{% macro some_test(model) -%} {{ return(adapter.dispatch('test_some_kind4', 'foo_utils4')) }} {%- endmacro %}", "{% macro some_test(model) -%} {{ return(adapter.dispatch('test_some_kind5', macro_namespace = 'foo_utils5')) }} {%- endmacro %}", ] @@ -34,7 +34,7 @@ def setUp(self): ['get_snapshot_unique_id'], ['get_columns_in_query'], ['get_snapshot_unique_id'], - ['dbt_utils.current_timestamp'], + ['current_timestamp_backcompat()'], ['test_some_kind4', 'foo_utils4.test_some_kind4'], ['test_some_kind5', 'foo_utils5.test_some_kind5'], ] diff --git a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py index fbe547d07de..0f9c30ae9ed 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py @@ -3,7 +3,7 @@ _MODEL_CURRENT_TIMESTAMP = """ select {{ current_timestamp() }} as current_timestamp, - {{ current_timestamp_in_utc() }} as current_timestamp_in_utc, + {{ current_timestamp_in_utc_backcompat() }} as current_timestamp_in_utc_backcompat, {{ current_timestamp_backcompat() }} as current_timestamp_backcompat """ @@ -17,7 +17,7 @@ def models(self): def expected_schema(self): return { "current_timestamp": "timestamp with time zone", - "current_timestamp_in_utc": "timestamp without time zone", + "current_timestamp_in_utc_backcompat": "timestamp without time zone", "current_timestamp_backcompat": "timestamp without time zone", } From 935dcfc587111c4436faf6b21837e8b2d2581549 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Sep 2022 14:46:05 -0700 Subject: [PATCH 24/35] remove convert_timezone --- .../macros/adapters/timestamps.sql | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index 9f9fb1f9009..93b9a9b4bcd 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -14,24 +14,6 @@ {{ current_timestamp() }} {% endmacro %} -{%- macro convert_timezone(source_tz, target_tz, source_timestamp) -%} - {%- if not target_tz is string -%} - {{ exceptions.raise_compiler_error("'target_tz' must be a string") }} - {%- else -%} - {{ adapter.dispatch('convert_timezone', 'dbt') (source_tz, target_tz, source_timestamp) }} - {%- endif -%} - -{%- endmacro -%} - -{%- macro default__convert_timezone(source_tz, target_tz, source_timestamp) -%} - {%- if not source_tz -%} - {{ source_timestamp }} at time zone '{{ target_tz }}' - {%- else -%} - {{ source_timestamp }} at time zone '{{ source_tz }}' at time zone '{{ target_tz }}' - {%- endif -%} -{%- endmacro -%} - - --------------------------------------------- /* {# From 97fc332398d1cfc8de6012740fc2580f7d5b60d3 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Sep 2022 14:52:29 -0700 Subject: [PATCH 25/35] add _in_utc_backcompat --- plugins/postgres/dbt/include/postgres/macros/timestamps.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql index 8278a334ebf..7233571b677 100644 --- a/plugins/postgres/dbt/include/postgres/macros/timestamps.sql +++ b/plugins/postgres/dbt/include/postgres/macros/timestamps.sql @@ -14,3 +14,7 @@ {% macro postgres__current_timestamp_backcompat() %} current_timestamp::{{ type_timestamp() }} {% endmacro %} + +{% macro postgres__current_timestamp_in_utc_backcompat() %} + (current_timestamp at time zone 'utc')::{{ type_timestamp() }} +{% endmacro %} From 0aa04660faf36361bdd06c68737b941008a87edf Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Sep 2022 15:02:01 -0700 Subject: [PATCH 26/35] fix macro_calls typo --- test/unit/test_macro_calls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_macro_calls.py b/test/unit/test_macro_calls.py index 6fbbe786641..7c38e825371 100644 --- a/test/unit/test_macro_calls.py +++ b/test/unit/test_macro_calls.py @@ -34,7 +34,7 @@ def setUp(self): ['get_snapshot_unique_id'], ['get_columns_in_query'], ['get_snapshot_unique_id'], - ['current_timestamp_backcompat()'], + ['current_timestamp_backcompat'], ['test_some_kind4', 'foo_utils4.test_some_kind4'], ['test_some_kind5', 'foo_utils5.test_some_kind5'], ] From 9d5ab96ef0f16a9d8b27773e4da19c054ed96de0 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 29 Sep 2022 10:02:49 -0700 Subject: [PATCH 27/35] add expected sql validation to test_timestamps --- .../macros/adapters/timestamps.sql | 2 +- .../dbt/tests/adapter/utils/test_timestamps.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index 93b9a9b4bcd..fa19013b208 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -35,7 +35,7 @@ {% endmacro %} {% macro current_timestamp_in_utc_backcompat() %} - {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }} + {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }} {% endmacro %} {% macro default__current_timestamp_in_utc_backcompat() %} diff --git a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py index 0f9c30ae9ed..a2b58b05052 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py @@ -1,4 +1,5 @@ import pytest +import re from dbt.tests.util import check_relation_has_expected_schema, run_dbt _MODEL_CURRENT_TIMESTAMP = """ @@ -7,12 +8,22 @@ {{ current_timestamp_backcompat() }} as current_timestamp_backcompat """ +_MODEL_EXPECTED_SQL = """ +select now() as current_timestamp, + (current_timestamp at time zone 'utc')::TIMESTAMP as current_timestamp_in_utc_backcompat, + current_timestamp::TIMESTAMP as current_timestamp_backcompat +""" + class TestCurrentTimestamps: @pytest.fixture(scope="class") def models(self): return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} + @pytest.fixture(scope="class") + def expected_sql(self): + return _MODEL_EXPECTED_SQL + @pytest.fixture(scope="class") def expected_schema(self): return { @@ -21,7 +32,7 @@ def expected_schema(self): "current_timestamp_backcompat": "timestamp without time zone", } - def test_current_timestamps(self, project, models, expected_schema): + def test_current_timestamps(self, project, models, expected_schema, expected_sql): results = run_dbt(["run"]) assert len(results) == 1 check_relation_has_expected_schema( @@ -29,3 +40,8 @@ def test_current_timestamps(self, project, models, expected_schema): relation_name="get_current_timestamp", expected_schema=expected_schema, ) + + generated_sql = results.results[0].node.compiled_code + assert re.sub(r"\s+", "", generated_sql) == re.sub( + r"\s+", "", expected_sql + ), f"generated sql did not match expected: {generated_sql}" From e08cf71efa26e3a1a5d27e54e30d3a00e1019013 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 29 Sep 2022 10:11:42 -0700 Subject: [PATCH 28/35] make expected_sql optional --- .../adapter/dbt/tests/adapter/utils/test_timestamps.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py index a2b58b05052..56946868122 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py @@ -20,6 +20,7 @@ class TestCurrentTimestamps: def models(self): return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} + # any adapters that don't want to check can set expected schema to None @pytest.fixture(scope="class") def expected_sql(self): return _MODEL_EXPECTED_SQL @@ -41,7 +42,8 @@ def test_current_timestamps(self, project, models, expected_schema, expected_sql expected_schema=expected_schema, ) - generated_sql = results.results[0].node.compiled_code - assert re.sub(r"\s+", "", generated_sql) == re.sub( - r"\s+", "", expected_sql - ), f"generated sql did not match expected: {generated_sql}" + if expected_sql: + generated_sql = results.results[0].node.compiled_code + assert re.sub(r"\s+", "", generated_sql) == re.sub( + r"\s+", "", expected_sql + ), f"generated sql did not match expected: {generated_sql}" From 5cfd696fae9616400e5005b1f2ff128b9d7903aa Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 29 Sep 2022 10:39:55 -0700 Subject: [PATCH 29/35] improve sql check string comparison test --- tests/adapter/dbt/tests/adapter/utils/test_timestamps.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py index 56946868122..acc5765198b 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py @@ -44,6 +44,8 @@ def test_current_timestamps(self, project, models, expected_schema, expected_sql if expected_sql: generated_sql = results.results[0].node.compiled_code - assert re.sub(r"\s+", "", generated_sql) == re.sub( - r"\s+", "", expected_sql + generated_sql_check = re.sub(r"\s+", "", generated_sql).lower() + expected_sql_check = re.sub(r"\s+", "", expected_sql).lower() + assert ( + expected_sql_check == generated_sql_check ), f"generated sql did not match expected: {generated_sql}" From ea3203adccc584aacb06746e98d67b2be4a6fa0c Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 29 Sep 2022 10:54:39 -0700 Subject: [PATCH 30/35] remove extraneous test file --- core/dbt/clients/jinja_static.py | 2 +- tests/functional/timestamps/test_timestamps.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 tests/functional/timestamps/test_timestamps.py diff --git a/core/dbt/clients/jinja_static.py b/core/dbt/clients/jinja_static.py index ce3658f08db..337a25eadda 100644 --- a/core/dbt/clients/jinja_static.py +++ b/core/dbt/clients/jinja_static.py @@ -15,7 +15,7 @@ def statically_extract_macro_calls(string, ctx, db_wrapper=None): if hasattr(func_call, "node") and hasattr(func_call.node, "name"): func_name = func_call.node.name else: - # func_call for dbt_utils.current_timestamp macro + # func_call for dbt.current_timestamp macro # Call( # node=Getattr( # node=Name( diff --git a/tests/functional/timestamps/test_timestamps.py b/tests/functional/timestamps/test_timestamps.py deleted file mode 100644 index 2dbcd5777b9..00000000000 --- a/tests/functional/timestamps/test_timestamps.py +++ /dev/null @@ -1,5 +0,0 @@ -from tests.adapter.dbt.tests.adapter.utils.test_timestamps import TestCurrentTimestamps - - -class TestCurrentTimestampsCore(TestCurrentTimestamps): - pass From 345fea148277ea285a16642c8bc9803845158f39 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 29 Sep 2022 14:56:47 -0700 Subject: [PATCH 31/35] add timestamp casting back --- .../include/global_project/macros/adapters/timestamps.sql | 2 +- tests/adapter/dbt/tests/adapter/utils/test_timestamps.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index fa19013b208..2bde30cd405 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -31,7 +31,7 @@ {% endmacro %} {% macro default__current_timestamp_backcompat() %} - {{ return(adapter.dispatch('current_timestamp', 'dbt')()) }} + current_timestamp::timestamp {% endmacro %} {% macro current_timestamp_in_utc_backcompat() %} diff --git a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py index acc5765198b..b1338d6cfad 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py @@ -15,7 +15,7 @@ """ -class TestCurrentTimestamps: +class BaseCurrentTimestamps: @pytest.fixture(scope="class") def models(self): return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} @@ -49,3 +49,6 @@ def test_current_timestamps(self, project, models, expected_schema, expected_sql assert ( expected_sql_check == generated_sql_check ), f"generated sql did not match expected: {generated_sql}" + +class TestCurrentTimestamps(BaseCurrentTimestamps): + pass \ No newline at end of file From a8b5bfe25191954484dbb3fa0f7bc9b2f1917ea0 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Thu, 29 Sep 2022 14:59:30 -0700 Subject: [PATCH 32/35] Update plugins/postgres/dbt/include/postgres/macros/adapters.sql Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- plugins/postgres/dbt/include/postgres/macros/adapters.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/postgres/dbt/include/postgres/macros/adapters.sql b/plugins/postgres/dbt/include/postgres/macros/adapters.sql index 211365dee11..3aca43ab010 100644 --- a/plugins/postgres/dbt/include/postgres/macros/adapters.sql +++ b/plugins/postgres/dbt/include/postgres/macros/adapters.sql @@ -118,7 +118,7 @@ {% endmacro %} {# - Postgres tables have a maximum length off 63 characters, anything longer is silently truncated. + Postgres tables have a maximum length of 63 characters, anything longer is silently truncated. Temp and backup relations add a lot of extra characters to the end of table names to ensure uniqueness. To prevent this going over the character limit, the base_relation name is truncated to ensure that name + suffix + uniquestring is < 63 characters. From efc339edddbd7ef754d53f45b431f08b280aac75 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 09:18:23 -0700 Subject: [PATCH 33/35] add check_relation_has_expected_schema to comments --- core/dbt/tests/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index d26204fa997..b4fb24fdd50 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -35,6 +35,7 @@ # relation_from_name # check_relation_types (table/view) # check_relations_equal +# check_relation_has_expected_schema # check_relations_equal_with_relations # check_table_does_exist # check_table_does_not_exist From c97cead2f7d251b4a4ca3a9531d5da3a98c944a5 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 09:42:46 -0700 Subject: [PATCH 34/35] fix whitespace --- tests/adapter/dbt/tests/adapter/utils/test_timestamps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py index b1338d6cfad..3fb3b2cd13d 100644 --- a/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py +++ b/tests/adapter/dbt/tests/adapter/utils/test_timestamps.py @@ -50,5 +50,6 @@ def test_current_timestamps(self, project, models, expected_schema, expected_sql expected_sql_check == generated_sql_check ), f"generated sql did not match expected: {generated_sql}" + class TestCurrentTimestamps(BaseCurrentTimestamps): - pass \ No newline at end of file + pass From 2664dfca7b779dd53910f019658156a6c611c9c5 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 14:13:26 -0700 Subject: [PATCH 35/35] remove default impl of current_timestamp --- .../include/global_project/macros/adapters/timestamps.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/dbt/include/global_project/macros/adapters/timestamps.sql b/core/dbt/include/global_project/macros/adapters/timestamps.sql index 2bde30cd405..64b5fd3ddda 100644 --- a/core/dbt/include/global_project/macros/adapters/timestamps.sql +++ b/core/dbt/include/global_project/macros/adapters/timestamps.sql @@ -2,9 +2,10 @@ {{ adapter.dispatch('current_timestamp', 'dbt')() }} {%- endmacro -%} -{% macro default__current_timestamp() %} - current_timestamp -{% endmacro %} +{% macro default__current_timestamp() -%} + {{ exceptions.raise_not_implemented( + 'current_timestamp macro not implemented for adapter ' + adapter.type()) }} +{%- endmacro %} {%- macro snapshot_get_time() -%} {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}