diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index d36da44365f..d3f2c27d7a9 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -613,3 +613,59 @@ def expand_target_column_types(cls, profile, project_cfg, temp_table, to_schema, to_table, model_name=None): # This is a no-op on BigQuery pass + + @classmethod + def _flat_columns_in_table(cls, profile, project_cfg, schema_name, + table_name): + """An iterator over the flattened columns for a given schema and table. + Resolves child columns as having the name "parent.child". + """ + cols = cls.get_columns_in_table(profile, project_cfg, + schema_name, table_name) + for col in cols: + flattened = col.flatten() + for subcol in flattened: + yield subcol + + @classmethod + def get_catalog(cls, profile, project_cfg, manifest): + schemas = { + node.to_dict()['schema'] + for node in manifest.nodes.values() + } + + column_names = ( + 'table_schema', + 'table_name', + 'table_type', + 'table_comment', + 'column_name', + 'column_index', + 'column_type', + 'column_comment', + ) + columns = [] + + for schema_name in schemas: + relations = cls.list_relations(profile, project_cfg, schema_name) + for relation in relations: + flattened = cls._flat_columns_in_table( + profile, + project_cfg, + schema_name, + relation.name + ) + for index, column in enumerate(flattened, start=1): + column_data = ( + relation.schema, + relation.name, + relation.type, + None, + column.name, + index, + column.data_type, + None, + ) + columns.append(dict(zip(column_names, column_data))) + + return dbt.clients.agate_helper.table_from_data(columns, column_names) diff --git a/test/integration/029_docs_generate_tests/bq_models/model.sql b/test/integration/029_docs_generate_tests/bq_models/model.sql new file mode 100644 index 00000000000..e831275ddc1 --- /dev/null +++ b/test/integration/029_docs_generate_tests/bq_models/model.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized='view' + ) +}} + +select * from {{ ref('seed') }} diff --git a/test/integration/029_docs_generate_tests/bq_models/seed.sql b/test/integration/029_docs_generate_tests/bq_models/seed.sql new file mode 100644 index 00000000000..012951927a6 --- /dev/null +++ b/test/integration/029_docs_generate_tests/bq_models/seed.sql @@ -0,0 +1,9 @@ +select + 1 as field_1, + 2 as field_2, + 3 as field_3, + + struct( + 4 as field_4, + 5 as field_5 + ) as nested_field diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 9e43c8a5bfe..6d5be9f108d 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1,8 +1,7 @@ import json import os -from nose.plugins.attrib import attr -from test.integration.base import DBTIntegrationTest +from test.integration.base import DBTIntegrationTest, use_profile class TestDocsGenerate(DBTIntegrationTest): @@ -12,7 +11,7 @@ def setUp(self): @property def schema(self): - return 'simple_dependency_029' + return 'docs_generate_029' @staticmethod def dir(path): @@ -86,25 +85,11 @@ def verify_manifest_macros(self, manifest): } ) - def verify_manifest(self): - self.assertTrue(os.path.exists('./target/manifest.json')) - - with open('./target/manifest.json') as fp: - manifest = json.load(fp) - - self.assertEqual( - set(manifest), - {'nodes', 'macros', 'parent_map', 'child_map'} - ) - - self.verify_manifest_macros(manifest) - manifest_without_macros = { - k: v for k, v in manifest.items() if k != 'macros' - } + def expected_seeded_manifest(self): # the manifest should be consistent across DBs for this test model_sql_path = self.dir('models/model.sql') my_schema_name = self.unique_schema() - expected_manifest = { + return { 'nodes': { 'model.test.model': { 'name': 'model', @@ -164,12 +149,26 @@ def verify_manifest(self): 'seed.test.seed': ['model.test.model'], }, } - self.assertEqual(manifest_without_macros, expected_manifest) + def verify_manifest(self, expected_manifest): + self.assertTrue(os.path.exists('./target/manifest.json')) + + with open('./target/manifest.json') as fp: + manifest = json.load(fp) + + self.assertEqual( + set(manifest), + {'nodes', 'macros', 'parent_map', 'child_map'} + ) - @attr(type='postgres') + self.verify_manifest_macros(manifest) + manifest_without_macros = { + k: v for k, v in manifest.items() if k != 'macros' + } + self.assertEqual(manifest_without_macros, expected_manifest) + + @use_profile('postgres') def test__postgres__run_and_generate(self): - self.use_profile('postgres') self.run_and_generate() my_schema_name = self.unique_schema() expected_cols = [ @@ -225,11 +224,10 @@ def test__postgres__run_and_generate(self): }, } self.verify_catalog(expected_catalog) - self.verify_manifest() + self.verify_manifest(self.expected_seeded_manifest()) - @attr(type='snowflake') + @use_profile('snowflake') def test__snowflake__run_and_generate(self): - self.use_profile('snowflake') self.run_and_generate() my_schema_name = self.unique_schema() expected_cols = [ @@ -286,4 +284,198 @@ def test__snowflake__run_and_generate(self): } self.verify_catalog(expected_catalog) - self.verify_manifest() + self.verify_manifest(self.expected_seeded_manifest()) + + @use_profile('bigquery') + def test__bigquery__run_and_generate(self): + self.run_and_generate() + my_schema_name = self.unique_schema() + expected_cols = [ + { + 'name': 'id', + 'index': 1, + 'type': 'INTEGER', + 'comment': None, + }, + { + 'name': 'first_name', + 'index': 2, + 'type': 'STRING', + 'comment': None, + }, + { + 'name': 'email', + 'index': 3, + 'type': 'STRING', + 'comment': None, + }, + { + 'name': 'ip_address', + 'index': 4, + 'type': 'STRING', + 'comment': None, + }, + { + 'name': 'updated_at', + 'index': 5, + 'type': 'DATETIME', + 'comment': None, + }, + ] + expected_catalog = { + 'model': { + 'metadata': { + 'schema': my_schema_name, + 'name': 'model', + 'type': 'view', + 'comment': None, + }, + 'columns': expected_cols, + }, + 'seed': { + 'metadata': { + 'schema': my_schema_name, + 'name': 'seed', + 'type': 'table', + 'comment': None, + }, + 'columns': expected_cols, + }, + } + self.verify_catalog(expected_catalog) + self.verify_manifest(self.expected_seeded_manifest()) + + @use_profile('bigquery') + def test__bigquery__nested_models(self): + self.use_default_project({'source-paths': [self.dir('bq_models')]}) + + self.assertEqual(len(self.run_dbt()), 2) + self.run_dbt(['docs', 'generate']) + + my_schema_name = self.unique_schema() + expected_cols = [ + { + "name": "field_1", + "index": 1, + "type": "INTEGER", + "comment": None + }, + { + "name": "field_2", + "index": 2, + "type": "INTEGER", + "comment": None + }, + { + "name": "field_3", + "index": 3, + "type": "INTEGER", + "comment": None + }, + { + "name": "nested_field.field_4", + "index": 4, + "type": "INTEGER", + "comment": None + }, + { + "name": "nested_field.field_5", + "index": 5, + "type": "INTEGER", + "comment": None + } + ] + catalog = { + "model": { + "metadata": { + "schema": my_schema_name, + "name": "model", + "type": "view", + "comment": None + }, + "columns": expected_cols + }, + "seed": { + "metadata": { + "schema": my_schema_name, + "name": "seed", + "type": "view", + "comment": None + }, + "columns": expected_cols + } + } + self.verify_catalog(catalog) + model_sql_path = self.dir('bq_models/model.sql') + seed_sql_path = self.dir('bq_models/seed.sql') + expected_manifest = { + 'nodes': { + 'model.test.model': { + 'alias': 'model', + 'config': { + 'column_types': {}, + 'enabled': True, + 'materialized': 'view', + 'post-hook': [], + 'pre-hook': [], + 'quoting': {}, + 'vars': {} + }, + 'depends_on': { + 'macros': [], + 'nodes': ['model.test.seed'] + }, + 'empty': False, + 'fqn': ['test', 'model'], + 'name': 'model', + 'original_file_path': model_sql_path, + 'package_name': 'test', + 'path': 'model.sql', + 'raw_sql': open(model_sql_path).read().rstrip('\n'), + 'refs': [['seed']], + 'resource_type': 'model', + 'root_path': os.getcwd(), + 'schema': my_schema_name, + 'tags': [], + 'unique_id': 'model.test.model' + }, + 'model.test.seed': { + 'alias': 'seed', + 'config': { + 'column_types': {}, + 'enabled': True, + 'materialized': 'view', + 'post-hook': [], + 'pre-hook': [], + 'quoting': {}, + 'vars': {} + }, + 'depends_on': { + 'macros': [], + 'nodes': [] + }, + 'empty': False, + 'fqn': ['test', 'seed'], + 'name': 'seed', + 'original_file_path': seed_sql_path, + 'package_name': 'test', + 'path': 'seed.sql', + 'raw_sql': open(seed_sql_path).read().rstrip('\n'), + 'refs': [], + 'resource_type': 'model', + 'root_path': os.getcwd(), + 'schema': my_schema_name, + 'tags': [], + 'unique_id': 'model.test.seed' + } + }, + 'child_map': { + 'model.test.model': [], + 'model.test.seed': ['model.test.model'] + }, + 'parent_map': { + 'model.test.model': ['model.test.seed'], + 'model.test.seed': [] + }, + } + self.verify_manifest(expected_manifest) diff --git a/test/integration/base.py b/test/integration/base.py index 188aeb04802..c71e8f076b9 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -5,6 +5,9 @@ import random import time import json +from functools import wraps + +from nose.plugins.attrib import attr from dbt.adapters.factory import get_adapter from dbt.project import Project @@ -528,3 +531,28 @@ def assertTableColumnsEqual(self, table_a, table_b, table_a_schema=None, table_b def assertEquals(self, *args, **kwargs): # assertEquals is deprecated. This makes the warnings less chatty self.assertEqual(*args, **kwargs) + + +def use_profile(profile_name): + """A decorator to declare a test method as using a particular profile. + Handles both setting the nose attr and calling self.use_profile. + + Use like this: + + class TestSomething(DBIntegrationTest): + @use_profile('postgres') + def test_postgres_thing(self): + self.assertEqual(self.adapter_type, 'postgres') + + @use_profile('snowflake') + def test_snowflake_thing(self): + self.assertEqual(self.adapter_type, 'snowflake') + """ + def outer(wrapped): + @attr(type=profile_name) + @wraps(wrapped) + def func(self, *args, **kwargs): + self.use_profile(profile_name) + return wrapped(self, *args, **kwargs) + return func + return outer