From 7be3a27a7391f76741b619219a263189f4187210 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 15 Sep 2022 13:30:10 -0700 Subject: [PATCH 01/14] update retry settings --- dbt/adapters/bigquery/connections.py | 4 +--- tests/conftest.py | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index 50437622b..336db3fd1 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -184,10 +184,8 @@ def __pre_deserialize__(cls, d: Dict[Any, Any]) -> Dict[Any, Any]: class BigQueryConnectionManager(BaseConnectionManager): TYPE = "bigquery" - QUERY_TIMEOUT = 300 - RETRIES = 1 DEFAULT_INITIAL_DELAY = 1.0 # Seconds - DEFAULT_MAXIMUM_DELAY = 1.0 # Seconds + DEFAULT_MAXIMUM_DELAY = 3.0 # Seconds @classmethod def handle_error(cls, error, message): diff --git a/tests/conftest.py b/tests/conftest.py index e74fa424b..6ca033f8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,6 +29,7 @@ def oauth_target(): 'type': 'bigquery', 'method': 'oauth', 'threads': 1, + 'job_retries': 2, } @@ -40,6 +41,7 @@ def service_account_target(): 'type': 'bigquery', 'method': 'service-account-json', 'threads': 1, + 'job_retries': 2, 'project': project_id, 'keyfile_json': credentials, # following 3 for python model From ab1a7e29edce8db45badbd463eb3499d33366d6d Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 15 Sep 2022 14:52:32 -0700 Subject: [PATCH 02/14] changelog entry --- .changes/unreleased/Under the Hood-20220915-145212.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20220915-145212.yaml diff --git a/.changes/unreleased/Under the Hood-20220915-145212.yaml b/.changes/unreleased/Under the Hood-20220915-145212.yaml new file mode 100644 index 000000000..6ab035256 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20220915-145212.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Update BQ job and call retry settings +time: 2022-09-15T14:52:12.902965-07:00 +custom: + Author: colin-rogers-dbt + Issue: "311" + PR: "310" From d7cf1972c2d552f1ab79d5ab320e09d3ea4ab2a8 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 26 Sep 2022 10:48:52 -0700 Subject: [PATCH 03/14] consolidate timestamps --- dbt/include/bigquery/macros/adapters.sql | 11 ----------- dbt/include/bigquery/macros/utils/timestamps.sql | 12 ++++++++++++ dev-requirements.txt | 4 ++-- tests/functional/adapter/utils/test_timestamps.py | 11 +++++++++++ 4 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 dbt/include/bigquery/macros/utils/timestamps.sql create mode 100644 tests/functional/adapter/utils/test_timestamps.py diff --git a/dbt/include/bigquery/macros/adapters.sql b/dbt/include/bigquery/macros/adapters.sql index cbfba2627..07cf3c3e5 100644 --- a/dbt/include/bigquery/macros/adapters.sql +++ b/dbt/include/bigquery/macros/adapters.sql @@ -109,17 +109,6 @@ {%- endmacro %} -{% macro bigquery__current_timestamp() -%} - CURRENT_TIMESTAMP() -{%- endmacro %} - - -{% macro bigquery__snapshot_string_as_time(timestamp) -%} - {%- set result = 'TIMESTAMP("' ~ timestamp ~ '")' -%} - {{ return(result) }} -{%- endmacro %} - - {% macro bigquery__list_schemas(database) -%} {{ return(adapter.list_schemas(database)) }} {% endmacro %} diff --git a/dbt/include/bigquery/macros/utils/timestamps.sql b/dbt/include/bigquery/macros/utils/timestamps.sql new file mode 100644 index 000000000..fe8332875 --- /dev/null +++ b/dbt/include/bigquery/macros/utils/timestamps.sql @@ -0,0 +1,12 @@ +{% macro bigquery__current_timestamp() -%} + CURRENT_TIMESTAMP() +{%- endmacro %} + +{% macro bigquery__current_timestamp_utc() -%} + CURRENT_TIMESTAMP() +{%- endmacro %} + +{% macro bigquery__snapshot_string_as_time(timestamp) -%} + {%- set result = 'TIMESTAMP("' ~ timestamp ~ '")' -%} + {{ return(result) }} +{%- endmacro %} \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt index 129dbbe64..dcae2b9e3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@consolidate_timestamp_logic#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@consolidate_timestamp_logic#egg=dbt-tests-adapter&subdirectory=tests/adapter black==22.8.0 bumpversion diff --git a/tests/functional/adapter/utils/test_timestamps.py b/tests/functional/adapter/utils/test_timestamps.py new file mode 100644 index 000000000..18995c8c9 --- /dev/null +++ b/tests/functional/adapter/utils/test_timestamps.py @@ -0,0 +1,11 @@ +import pytest +from dbt.tests.adapter.utils import test_timestamps + + +class TestCurrentTimestampSnowflake(test_timestamps.TestCurrentTimestamps): + @pytest.fixture(scope="class") + def expected_schema(self): + return { + "CURRENT_TIMESTAMP": "TIMESTAMP", + "CURRENT_TIMESTAMP_IN_UTC": "TIMESTAMP" + } From efdc510027bd95e7984ef72bdd2fc243afd77b39 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 26 Sep 2022 10:57:10 -0700 Subject: [PATCH 04/14] add changie --- .changes/unreleased/Features-20220926-105700.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Features-20220926-105700.yaml diff --git a/.changes/unreleased/Features-20220926-105700.yaml b/.changes/unreleased/Features-20220926-105700.yaml new file mode 100644 index 000000000..543dd67ab --- /dev/null +++ b/.changes/unreleased/Features-20220926-105700.yaml @@ -0,0 +1,7 @@ +kind: Features +body: consolidate timestamp macros +time: 2022-09-26T10:57:00.942765-07:00 +custom: + Author: colin-rogers-dbt + Issue: "5521" + PR: "323" From 9483fba67fb90e1fef8f88adfd549e710acebbed Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 26 Sep 2022 10:58:21 -0700 Subject: [PATCH 05/14] whitespace fix --- dbt/include/bigquery/macros/utils/timestamps.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/bigquery/macros/utils/timestamps.sql b/dbt/include/bigquery/macros/utils/timestamps.sql index fe8332875..0e5d1887e 100644 --- a/dbt/include/bigquery/macros/utils/timestamps.sql +++ b/dbt/include/bigquery/macros/utils/timestamps.sql @@ -9,4 +9,4 @@ {% macro bigquery__snapshot_string_as_time(timestamp) -%} {%- set result = 'TIMESTAMP("' ~ timestamp ~ '")' -%} {{ return(result) }} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} From 570561b8f3304d0ea8b6463913f62ebd3b2fdf44 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 26 Sep 2022 11:39:10 -0700 Subject: [PATCH 06/14] fix macro name --- dbt/include/bigquery/macros/utils/timestamps.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/bigquery/macros/utils/timestamps.sql b/dbt/include/bigquery/macros/utils/timestamps.sql index 0e5d1887e..ee5a6fab8 100644 --- a/dbt/include/bigquery/macros/utils/timestamps.sql +++ b/dbt/include/bigquery/macros/utils/timestamps.sql @@ -2,7 +2,7 @@ CURRENT_TIMESTAMP() {%- endmacro %} -{% macro bigquery__current_timestamp_utc() -%} +{% macro bigquery__current_timestamp_in_utc() -%} CURRENT_TIMESTAMP() {%- endmacro %} From bbe14ae43aa770932b9cef6db1e5ef50af4bc0c8 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 26 Sep 2022 14:40:59 -0700 Subject: [PATCH 07/14] update expected test fixture --- tests/functional/adapter/utils/test_timestamps.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/functional/adapter/utils/test_timestamps.py b/tests/functional/adapter/utils/test_timestamps.py index 18995c8c9..40d0c72d4 100644 --- a/tests/functional/adapter/utils/test_timestamps.py +++ b/tests/functional/adapter/utils/test_timestamps.py @@ -5,7 +5,4 @@ class TestCurrentTimestampSnowflake(test_timestamps.TestCurrentTimestamps): @pytest.fixture(scope="class") def expected_schema(self): - return { - "CURRENT_TIMESTAMP": "TIMESTAMP", - "CURRENT_TIMESTAMP_IN_UTC": "TIMESTAMP" - } + return {"current_timestamp": "TIMESTAMP", "current_timestamp_in_utc": "TIMESTAMP"} From ba288cb9982e603c2c963e961964d2ce95b3ad9f Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:33:32 -0700 Subject: [PATCH 08/14] Update Features-20220926-105700.yaml --- .changes/unreleased/Features-20220926-105700.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Features-20220926-105700.yaml b/.changes/unreleased/Features-20220926-105700.yaml index 543dd67ab..6b4bc5f09 100644 --- a/.changes/unreleased/Features-20220926-105700.yaml +++ b/.changes/unreleased/Features-20220926-105700.yaml @@ -3,5 +3,5 @@ body: consolidate timestamp macros time: 2022-09-26T10:57:00.942765-07:00 custom: Author: colin-rogers-dbt - Issue: "5521" + Issue: "324" PR: "323" From 057ce0e222bad9ee243660e1865f79b971f08651 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 27 Sep 2022 11:10:07 -0700 Subject: [PATCH 09/14] add backcompat to test fixture --- tests/functional/adapter/utils/test_timestamps.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/utils/test_timestamps.py b/tests/functional/adapter/utils/test_timestamps.py index 40d0c72d4..1cb0aff1d 100644 --- a/tests/functional/adapter/utils/test_timestamps.py +++ b/tests/functional/adapter/utils/test_timestamps.py @@ -5,4 +5,8 @@ class TestCurrentTimestampSnowflake(test_timestamps.TestCurrentTimestamps): @pytest.fixture(scope="class") def expected_schema(self): - return {"current_timestamp": "TIMESTAMP", "current_timestamp_in_utc": "TIMESTAMP"} + return { + "current_timestamp": "TIMESTAMP", + "current_timestamp_in_utc": "TIMESTAMP", + "current_timestamp_backcompat": "TIMESTAMP", + } From 60afae9f738f5e8b0b6a1cbde1952db9a1bb7dc4 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 14:45:47 -0700 Subject: [PATCH 10/14] add backcompat --- dbt/include/bigquery/macros/utils/timestamps.sql | 8 ++++---- tests/functional/adapter/utils/test_timestamps.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/dbt/include/bigquery/macros/utils/timestamps.sql b/dbt/include/bigquery/macros/utils/timestamps.sql index ee5a6fab8..abc17b02f 100644 --- a/dbt/include/bigquery/macros/utils/timestamps.sql +++ b/dbt/include/bigquery/macros/utils/timestamps.sql @@ -2,11 +2,11 @@ CURRENT_TIMESTAMP() {%- endmacro %} -{% macro bigquery__current_timestamp_in_utc() -%} - CURRENT_TIMESTAMP() -{%- endmacro %} - {% macro bigquery__snapshot_string_as_time(timestamp) -%} {%- set result = 'TIMESTAMP("' ~ timestamp ~ '")' -%} {{ return(result) }} {%- endmacro %} + +{% macro bigquery__current_timestamp_backcompat() -%} + current_timestamp +{%- endmacro %} diff --git a/tests/functional/adapter/utils/test_timestamps.py b/tests/functional/adapter/utils/test_timestamps.py index 1cb0aff1d..96077f02e 100644 --- a/tests/functional/adapter/utils/test_timestamps.py +++ b/tests/functional/adapter/utils/test_timestamps.py @@ -1,12 +1,18 @@ import pytest -from dbt.tests.adapter.utils import test_timestamps +from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps -class TestCurrentTimestampSnowflake(test_timestamps.TestCurrentTimestamps): +class TestCurrentTimestampSnowflake(BaseCurrentTimestamps): @pytest.fixture(scope="class") def expected_schema(self): return { "current_timestamp": "TIMESTAMP", - "current_timestamp_in_utc": "TIMESTAMP", + "current_timestamp_in_utc_backcompat": "TIMESTAMP", "current_timestamp_backcompat": "TIMESTAMP", } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """select CURRENT_TIMESTAMP() as current_timestamp, + current_timestamp as current_timestamp_in_utc_backcompat, + current_timestamp as current_timestamp_backcompat""" \ No newline at end of file From 666724726f5679e81af8846bf490c15787a1db7e Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 15:34:15 -0700 Subject: [PATCH 11/14] update dev-requirements --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 715ad4f62..676703d3e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@consolidate_timestamp_logic#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@consolidate_timestamp_logic#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter black==22.8.0 bumpversion From 4bf2b33532ce220b10ef02522fb598b0f6a88507 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 15:40:40 -0700 Subject: [PATCH 12/14] Update change log body --- .changes/unreleased/Features-20220926-105700.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Features-20220926-105700.yaml b/.changes/unreleased/Features-20220926-105700.yaml index 6b4bc5f09..61e0ac741 100644 --- a/.changes/unreleased/Features-20220926-105700.yaml +++ b/.changes/unreleased/Features-20220926-105700.yaml @@ -1,5 +1,5 @@ kind: Features -body: consolidate timestamp macros +body: Migrate dbt-utils current_timestamp macros into core + adapters time: 2022-09-26T10:57:00.942765-07:00 custom: Author: colin-rogers-dbt From 6f1919f50a541f11176b0cf21eecc3f831f6d14f Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 15:51:01 -0700 Subject: [PATCH 13/14] update test class name --- tests/functional/adapter/utils/test_timestamps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/utils/test_timestamps.py b/tests/functional/adapter/utils/test_timestamps.py index 96077f02e..ab1306db8 100644 --- a/tests/functional/adapter/utils/test_timestamps.py +++ b/tests/functional/adapter/utils/test_timestamps.py @@ -2,7 +2,7 @@ from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps -class TestCurrentTimestampSnowflake(BaseCurrentTimestamps): +class TestCurrentTimestampBigQuery(BaseCurrentTimestamps): @pytest.fixture(scope="class") def expected_schema(self): return { From 867117b1bd7f6526b191d17e72c5bb6d0ed9f6e7 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 15:54:56 -0700 Subject: [PATCH 14/14] lowercase timestamps --- dbt/include/bigquery/macros/utils/timestamps.sql | 2 +- tests/functional/adapter/utils/test_timestamps.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/bigquery/macros/utils/timestamps.sql b/dbt/include/bigquery/macros/utils/timestamps.sql index abc17b02f..cdcbfd51e 100644 --- a/dbt/include/bigquery/macros/utils/timestamps.sql +++ b/dbt/include/bigquery/macros/utils/timestamps.sql @@ -1,5 +1,5 @@ {% macro bigquery__current_timestamp() -%} - CURRENT_TIMESTAMP() + current_timestamp() {%- endmacro %} {% macro bigquery__snapshot_string_as_time(timestamp) -%} diff --git a/tests/functional/adapter/utils/test_timestamps.py b/tests/functional/adapter/utils/test_timestamps.py index ab1306db8..2f35e40ee 100644 --- a/tests/functional/adapter/utils/test_timestamps.py +++ b/tests/functional/adapter/utils/test_timestamps.py @@ -13,6 +13,6 @@ def expected_schema(self): @pytest.fixture(scope="class") def expected_sql(self): - return """select CURRENT_TIMESTAMP() as current_timestamp, + return """select current_timestamp() as current_timestamp, current_timestamp as current_timestamp_in_utc_backcompat, current_timestamp as current_timestamp_backcompat""" \ No newline at end of file