From 6e5c8750f61b48c4bc8efff102ea40a33e95ee8a Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 13 May 2020 10:16:53 -0600 Subject: [PATCH] Fix create_schema macro on bigquery --- CHANGELOG.md | 3 +- core/dbt/context/providers.py | 6 ++- .../bigquery/dbt/adapters/bigquery/impl.py | 1 + .../dbt/include/bigquery/macros/adapters.sql | 4 +- .../macros/wrapped_macros.sql | 43 +++++++++++++++++++ .../test_bigquery_adapter_functions.py | 40 +++++++++++++++++ 6 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 test/integration/022_bigquery_test/macros/wrapped_macros.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index c1961f735a5..14d13121d65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Fixes - When no columns are documented and persist_docs.columns is True, skip creating comments instead of failing with errors ([#2439](https://github.com/fishtown-analytics/dbt/issues/2439), [#2440](https://github.com/fishtown-analytics/dbt/pull/2440)) +- Fixed an argument issue with the `create_schema` macro on bigquery ([#2445](https://github.com/fishtown-analytics/dbt/issues/2445), [#2448](https://github.com/fishtown-analytics/dbt/pull/2448)) ## dbt 0.17.0rc1 (May 12, 2020) @@ -25,7 +26,7 @@ - Fix for extra spacing and parentheses when creating views in BigQuery ([#2421](https://github.com/fishtown-analytics/dbt/issues/2421), [#2422](https://github.com/fishtown-analytics/dbt/issues/2422)) ### Docs -- Do not render hidden models in the search bar ([docs#89](https://github.com/fishtown-analytics/dbt-docs/issues/89, [docs#90](https://github.com/fishtown-analytics/dbt-docs/pull/90)) +- Do not render hidden models in the search bar ([docs#89](https://github.com/fishtown-analytics/dbt-docs/issues/89), [docs#90](https://github.com/fishtown-analytics/dbt-docs/pull/90)) ### Under the hood - Track distinct project hashes in anonymous usage metrics for package downloads ([#2351](https://github.com/fishtown-analytics/dbt/issues/2351), [#2429](https://github.com/fishtown-analytics/dbt/pull/2429)) diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 820febbfc71..8b235feef62 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -234,7 +234,11 @@ def _validate(self, validator, value): validator(value) def _lookup(self, name, default=_MISSING): - result = self.model.config.get(name, default) + # if this is a macro, there might be no `model.config`. + if not hasattr(self.model, 'config'): + result = default + else: + result = self.model.config.get(name, default) if result is _MISSING: missing_config(self.model, name) return result diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index c1ea5aab78e..caafcb0fbfb 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -212,6 +212,7 @@ def expand_target_column_types( # This is a no-op on BigQuery pass + @available.parse_list def list_relations_without_caching( self, schema_relation: BigQueryRelation ) -> List[BigQueryRelation]: diff --git a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql index 5bf24ef2df3..d6365a27653 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql @@ -69,8 +69,8 @@ {% endmacro %} -{% macro bigquery__create_schema(database_name, schema_name) -%} - {{ adapter.create_schema(database_name, schema_name) }} +{% macro bigquery__create_schema(relation) -%} + {{ adapter.create_schema(relation) }} {% endmacro %} {% macro bigquery__drop_schema(relation) -%} diff --git a/test/integration/022_bigquery_test/macros/wrapped_macros.sql b/test/integration/022_bigquery_test/macros/wrapped_macros.sql new file mode 100644 index 00000000000..fc66f5a9588 --- /dev/null +++ b/test/integration/022_bigquery_test/macros/wrapped_macros.sql @@ -0,0 +1,43 @@ +{% macro my_create_schema(db_name, schema_name) %} + {% if not execute %} + {% do return(None) %} + {% endif %} + {% set relation = api.Relation.create(database=db_name, schema=schema_name).without_identifier() %} + {% do create_schema(relation) %} +{% endmacro %} + +{% macro my_drop_schema(db_name, schema_name) %} + {% if not execute %} + {% do return(None) %} + {% endif %} + {% set relation = api.Relation.create(database=db_name, schema=schema_name).without_identifier() %} + {% do drop_schema(relation) %} +{% endmacro %} + + +{% macro my_create_table_as(db_name, schema_name, table_name) %} + {% if not execute %} + {% do return(None) %} + {% endif %} + {% set relation = api.Relation.create(database=db_name, schema=schema_name, identifier=table_name) %} + {% do run_query(create_table_as(false, relation, 'select 1 as id')) %} +{% endmacro %} + + +{% macro ensure_one_relation_in(db_name, schema_name) %} + {% if not execute %} + {% do return(None) %} + {% endif %} + {% set relation = api.Relation.create(database=db_name, schema=schema_name).without_identifier() %} + {% set results = list_relations_without_caching(relation) %} + {% set rlen = (results | length) %} + {% if rlen != 1 %} + {% do exceptions.raise_compiler_error('Incorect number of results (expected 1): ' ~ rlen) %} + {% endif %} + {% set result = results[0] %} + {% set columns = get_columns_in_relation(result) %} + {% set clen = (columns | length) %} + {% if clen != 1 %} + {% do exceptions.raise_compiler_error('Incorrect number of columns (expected 1): ' ~ clen) %} + {% endif %} +{% endmacro %} diff --git a/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py b/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py index 4490bf7a9a4..35dddf44755 100644 --- a/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py +++ b/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py @@ -1,4 +1,5 @@ from test.integration.base import DBTIntegrationTest, FakeArgs, use_profile +import yaml class TestBigqueryAdapterFunctions(DBTIntegrationTest): @@ -29,3 +30,42 @@ def test__bigquery_adapter_functions(self): # status = # of failing rows self.assertEqual(result.status, 0) + +class TestBigqueryAdapterMacros(DBTIntegrationTest): + @property + def schema(self): + return "bigquery_test_022" + + @property + def models(self): + return "models" + + def _create_schema_named(self, database, schema): + # do not create the initial schema. We'll do this ourselves! + pass + + @use_profile('bigquery') + def test__bigquery_run_create_drop_schema(self): + schema_args = yaml.safe_dump({ + 'db_name': self.default_database, + 'schema_name': self.unique_schema(), + }) + self.run_dbt(['run-operation', 'my_create_schema', '--args', schema_args]) + relation_args = yaml.safe_dump({ + 'db_name': self.default_database, + 'schema_name': self.unique_schema(), + 'table_name': 'some_table', + }) + self.run_dbt(['run-operation', 'my_create_table_as', '--args', relation_args]) + # exercise list_relations_without_caching and get_columns_in_relation + self.run_dbt(['run-operation', 'ensure_one_relation_in', '--args', schema_args]) + # now to drop the schema + schema_relation = self.adapter.Relation.create(database=self.default_database, schema=self.unique_schema()).without_identifier() + with self.adapter.connection_named('test'): + results = self.adapter.list_relations_without_caching(schema_relation) + assert len(results) == 1 + + self.run_dbt(['run-operation', 'my_drop_schema', '--args', schema_args]) + with self.adapter.connection_named('test'): + results = self.adapter.list_relations_without_caching(schema_relation) + assert len(results) == 0