diff --git a/CHANGELOG.md b/CHANGELOG.md index 33cb79c3a19..538353442bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,13 @@ - Save selectors dictionary to manifest, allow descriptions ([#2693](https://github.com/fishtown-analytics/dbt/issues/2693), [#2866](https://github.com/fishtown-analytics/dbt/pull/2866)) - Normalize cli-style-strings in manifest selectors dictionary ([#2879](https://github.com/fishtown-anaytics/dbt/issues/2879), [#2895](https://github.com/fishtown-analytics/dbt/pull/2895)) - Hourly, monthly and yearly partitions available in BigQuery ([#2476](https://github.com/fishtown-analytics/dbt/issues/2476), [#2903](https://github.com/fishtown-analytics/dbt/pull/2903)) +- Allow BigQuery to default to the environment's default project ([#2828](https://github.com/fishtown-analytics/dbt/pull/2828), [#2908](https://github.com/fishtown-analytics/dbt/pull/2908)) ### Fixes - Respect --project-dir in dbt clean command ([#2840](https://github.com/fishtown-analytics/dbt/issues/2840), [#2841](https://github.com/fishtown-analytics/dbt/pull/2841)) - Fix Redshift adapter `get_columns_in_relation` macro to push schema filter down to the `svv_external_columns` view ([#2855](https://github.com/fishtown-analytics/dbt/issues/2854)) - Add `unixodbc-dev` package to testing docker image ([#2859](https://github.com/fishtown-analytics/dbt/pull/2859)) - Increased the supported relation name length in postgres from 29 to 51 ([#2850](https://github.com/fishtown-analytics/dbt/pull/2850)) -- Widen supported Google Cloud libraries dependencies ([#2794](https://github.com/fishtown-analytics/dbt/pull/2794), [#2877](https://github.com/fishtown-analytics/dbt/pull/2877)). - dbt list command always return 0 as exit code ([#2886](https://github.com/fishtown-analytics/dbt/issues/2886), [#2892](https://github.com/fishtown-analytics/dbt/issues/2892)) - Set default `materialized` for test node configs to `test` ([#2806](https://github.com/fishtown-analytics/dbt/issues/2806), [#2902](https://github.com/fishtown-analytics/dbt/pull/2902)) @@ -21,6 +21,7 @@ - Bump hologram version to 0.0.11. Add scripts/dtr.py ([#2888](https://github.com/fishtown-analytics/dbt/issues/2840),[#2889](https://github.com/fishtown-analytics/dbt/pull/2889)) - Add event tracking for project parser/load times ([#2823](https://github.com/fishtown-analytics/dbt/issues/2823),[#2893](https://github.com/fishtown-analytics/dbt/pull/2893)) - Bump cryptography version to be >= 3.2 and bump snowflake connector to 2.3.6 ([#2896](https://github.com/fishtown-analytics/dbt/issues/2896)) +- Widen supported Google Cloud libraries dependencies ([#2794](https://github.com/fishtown-analytics/dbt/pull/2794), [#2877](https://github.com/fishtown-analytics/dbt/pull/2877)). Contributors: - [@feluelle](https://github.com/feluelle) ([#2841](https://github.com/fishtown-analytics/dbt/pull/2841)) @@ -29,7 +30,7 @@ Contributors: - [@brangisom](https://github.com/brangisom) [#2855](https://github.com/fishtown-analytics/dbt/pull/2855) - [@elexisvenator](https://github.com/elexisvenator) ([#2850](https://github.com/fishtown-analytics/dbt/pull/2850)) - [@franloza](https://github.com/franloza) ([#2837](https://github.com/fishtown-analytics/dbt/pull/2837)) -- [@max-sixty](https://github.com/max-sixty) ([#2877](https://github.com/fishtown-analytics/dbt/pull/2877)) +- [@max-sixty](https://github.com/max-sixty) ([#2877](https://github.com/fishtown-analytics/dbt/pull/2877), [#2908](https://github.com/fishtown-analytics/dbt/pull/2908)) - [@rsella](https://github.com/rsella) ([#2892](https://github.com/fishtown-analytics/dbt/issues/2892)) - [@plotneishestvo](https://github.com/plotneishestvo) ([#2896](https://github.com/fishtown-analytics/dbt/issues/2896)) - [@db-magnus](https://github.com/db-magnus) ([#2892](https://github.com/fishtown-analytics/dbt/issues/2892)) diff --git a/plugins/bigquery/dbt/adapters/bigquery/connections.py b/plugins/bigquery/dbt/adapters/bigquery/connections.py index 738935d3e23..743718869cd 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/connections.py +++ b/plugins/bigquery/dbt/adapters/bigquery/connections.py @@ -1,7 +1,8 @@ from contextlib import contextmanager from dataclasses import dataclass +from functools import lru_cache from requests.exceptions import ConnectionError -from typing import Optional, Any, Dict +from typing import Optional, Any, Dict, Tuple import google.auth import google.auth.exceptions @@ -45,6 +46,17 @@ ) +@lru_cache() +def get_bigquery_defaults() -> Tuple[Any, Optional[str]]: + """ + Returns (credentials, project_id) + + project_id is returned available from the environment; otherwise None + """ + # Cached, because the underlying implementation shells out, taking ~1s + return google.auth.default() + + class Priority(StrEnum): Interactive = 'interactive' Batch = 'batch' @@ -60,6 +72,9 @@ class BigQueryConnectionMethod(StrEnum): @dataclass class BigQueryCredentials(Credentials): method: BigQueryConnectionMethod + # BigQuery allows an empty database / project, where it defers to the + # environment for the project + database: Optional[str] timeout_seconds: Optional[int] = 300 location: Optional[str] = None priority: Optional[Priority] = None @@ -91,6 +106,16 @@ def _connection_keys(self): return ('method', 'database', 'schema', 'location', 'priority', 'timeout_seconds', 'maximum_bytes_billed') + def __post_init__(self): + # We need to inject the correct value of the database (aka project) at + # this stage, ref + # https://github.com/fishtown-analytics/dbt/pull/2908#discussion_r532927436. + + # `database` is an alias of `project` in BigQuery + if self.database is None: + _, database = get_bigquery_defaults() + self.database = database + class BigQueryConnectionManager(BaseConnectionManager): TYPE = 'bigquery' @@ -170,7 +195,7 @@ def get_bigquery_credentials(cls, profile_credentials): creds = GoogleServiceAccountCredentials.Credentials if method == BigQueryConnectionMethod.OAUTH: - credentials, project_id = google.auth.default(scopes=cls.SCOPE) + credentials, _ = get_bigquery_defaults() return credentials elif method == BigQueryConnectionMethod.SERVICE_ACCOUNT: diff --git a/test/unit/test_bigquery_adapter.py b/test/unit/test_bigquery_adapter.py index 1232ced021b..0380c7e095e 100644 --- a/test/unit/test_bigquery_adapter.py +++ b/test/unit/test_bigquery_adapter.py @@ -96,6 +96,13 @@ def setUp(self): 'priority': 'batch', 'maximum_bytes_billed': 0, }, + 'oauth--no-project': { + 'type': 'bigquery', + 'method': 'oauth', + 'schema': 'dummy_schema', + 'threads': 1, + 'location': 'Solar Station', + }, }, 'target': 'oauth', }