Bigquery catalog generation (#830)

dbt/adapters/bigquery/impl.py

@@ -613,3 +613,59 @@ def expand_target_column_types(cls, profile, project_cfg, temp_table,
                                    to_schema, to_table, model_name=None):
         # This is a no-op on BigQuery
         pass
+
+    @classmethod
+    def _flat_columns_in_table(cls, profile, project_cfg, schema_name,
+                               table_name):
+        """An iterator over the flattened columns for a given schema and table.
+        Resolves child columns as having the name "parent.child".
+        """
+        cols = cls.get_columns_in_table(profile, project_cfg,
+                                        schema_name, table_name)
+        for col in cols:
+            flattened = col.flatten()
+            for subcol in flattened:
+                yield subcol
+
+    @classmethod
+    def get_catalog(cls, profile, project_cfg, manifest):
+        schemas = {
+            node.to_dict()['schema']
+            for node in manifest.nodes.values()
+        }
+
+        column_names = (
+            'table_schema',
+            'table_name',
+            'table_type',
+            'table_comment',
+            'column_name',
+            'column_index',
+            'column_type',
+            'column_comment',
+        )
+        columns = []
+
+        for schema_name in schemas:
+            relations = cls.list_relations(profile, project_cfg, schema_name)
+            for relation in relations:
+                flattened = cls._flat_columns_in_table(
+                    profile,
+                    project_cfg,
+                    schema_name,
+                    relation.name
+                )
+                for index, column in enumerate(flattened, start=1):
+                    column_data = (
+                        relation.schema,
+                        relation.name,
+                        relation.type,
+                        None,
+                        column.name,
+                        index,
+                        column.data_type,
+                        None,
+                    )
+                    columns.append(dict(zip(column_names, column_data)))
+
+        return dbt.clients.agate_helper.table_from_data(columns, column_names)

test/integration/029_docs_generate_tests/bq_models/model.sql

@@ -0,0 +1,7 @@
+{{
+    config(
+        materialized='view'
+    )
+}}
+
+select * from {{ ref('seed') }}

test/integration/029_docs_generate_tests/bq_models/seed.sql

@@ -0,0 +1,9 @@
+select
+    1 as field_1,
+    2 as field_2,
+    3 as field_3,
+
+    struct(
+        4 as field_4,
+        5 as field_5
+    ) as nested_field

test/integration/029_docs_generate_tests/test_docs_generate.py

@@ -1,8 +1,7 @@
 import json
 import os
 
-from nose.plugins.attrib import attr
-from test.integration.base import DBTIntegrationTest
+from test.integration.base import DBTIntegrationTest, use_profile
 
 
 class TestDocsGenerate(DBTIntegrationTest):
@@ -12,7 +11,7 @@ def setUp(self):
 
     @property
     def schema(self):
-        return 'simple_dependency_029'
+        return 'docs_generate_029'
 
     @staticmethod
     def dir(path):
@@ -86,25 +85,11 @@ def verify_manifest_macros(self, manifest):
             }
         )
 
-    def verify_manifest(self):
-        self.assertTrue(os.path.exists('./target/manifest.json'))
-
-        with open('./target/manifest.json') as fp:
-            manifest = json.load(fp)
-
-        self.assertEqual(
-            set(manifest),
-            {'nodes', 'macros', 'parent_map', 'child_map'}
-        )
-
-        self.verify_manifest_macros(manifest)
-        manifest_without_macros = {
-            k: v for k, v in manifest.items() if k != 'macros'
-        }
+    def expected_seeded_manifest(self):
         # the manifest should be consistent across DBs for this test
         model_sql_path = self.dir('models/model.sql')
         my_schema_name = self.unique_schema()
-        expected_manifest = {
+        return {
             'nodes': {
                 'model.test.model': {
                     'name': 'model',
@@ -164,12 +149,26 @@ def verify_manifest(self):
                 'seed.test.seed': ['model.test.model'],
             },
         }
-        self.assertEqual(manifest_without_macros, expected_manifest)
 
+    def verify_manifest(self, expected_manifest):
+        self.assertTrue(os.path.exists('./target/manifest.json'))
+
+        with open('./target/manifest.json') as fp:
+            manifest = json.load(fp)
+
+        self.assertEqual(
+            set(manifest),
+            {'nodes', 'macros', 'parent_map', 'child_map'}
+        )
 
-    @attr(type='postgres')
+        self.verify_manifest_macros(manifest)
+        manifest_without_macros = {
+            k: v for k, v in manifest.items() if k != 'macros'
+        }
+        self.assertEqual(manifest_without_macros, expected_manifest)
+
+    @use_profile('postgres')
     def test__postgres__run_and_generate(self):
-        self.use_profile('postgres')
         self.run_and_generate()
         my_schema_name = self.unique_schema()
         expected_cols = [
@@ -225,11 +224,10 @@ def test__postgres__run_and_generate(self):
             },
         }
         self.verify_catalog(expected_catalog)
-        self.verify_manifest()
+        self.verify_manifest(self.expected_seeded_manifest())
 
-    @attr(type='snowflake')
+    @use_profile('snowflake')
     def test__snowflake__run_and_generate(self):
-        self.use_profile('snowflake')
         self.run_and_generate()
         my_schema_name = self.unique_schema()
         expected_cols = [
@@ -286,4 +284,198 @@ def test__snowflake__run_and_generate(self):
         }
 
         self.verify_catalog(expected_catalog)
-        self.verify_manifest()
+        self.verify_manifest(self.expected_seeded_manifest())
+
+    @use_profile('bigquery')
+    def test__bigquery__run_and_generate(self):
+        self.run_and_generate()
+        my_schema_name = self.unique_schema()
+        expected_cols = [
+            {
+                'name': 'id',
+                'index': 1,
+                'type': 'INTEGER',
+                'comment': None,
+            },
+            {
+                'name': 'first_name',
+                'index': 2,
+                'type': 'STRING',
+                'comment': None,
+            },
+            {
+                'name': 'email',
+                'index': 3,
+                'type': 'STRING',
+                'comment': None,
+            },
+            {
+                'name': 'ip_address',
+                'index': 4,
+                'type': 'STRING',
+                'comment': None,
+            },
+            {
+                'name': 'updated_at',
+                'index': 5,
+                'type': 'DATETIME',
+                'comment': None,
+            },
+        ]
+        expected_catalog = {
+            'model': {
+                'metadata': {
+                    'schema': my_schema_name,
+                    'name': 'model',
+                    'type': 'view',
+                    'comment': None,
+                },
+                'columns': expected_cols,
+            },
+            'seed': {
+                'metadata': {
+                    'schema': my_schema_name,
+                    'name': 'seed',
+                    'type': 'table',
+                    'comment': None,
+                },
+                'columns': expected_cols,
+            },
+        }
+        self.verify_catalog(expected_catalog)
+        self.verify_manifest(self.expected_seeded_manifest())
+
+    @use_profile('bigquery')
+    def test__bigquery__nested_models(self):
+        self.use_default_project({'source-paths': [self.dir('bq_models')]})
+
+        self.assertEqual(len(self.run_dbt()), 2)
+        self.run_dbt(['docs', 'generate'])
+
+        my_schema_name = self.unique_schema()
+        expected_cols = [
+            {
+                "name": "field_1",
+                "index": 1,
+                "type": "INTEGER",
+                "comment": None
+            },
+            {
+                "name": "field_2",
+                "index": 2,
+                "type": "INTEGER",
+                "comment": None
+            },
+            {
+                "name": "field_3",
+                "index": 3,
+                "type": "INTEGER",
+                "comment": None
+            },
+            {
+                "name": "nested_field.field_4",
+                "index": 4,
+                "type": "INTEGER",
+                "comment": None
+            },
+            {
+                "name": "nested_field.field_5",
+                "index": 5,
+                "type": "INTEGER",
+                "comment": None
+            }
+        ]
+        catalog = {
+            "model": {
+                "metadata": {
+                    "schema": my_schema_name,
+                    "name": "model",
+                    "type": "view",
+                    "comment": None
+                },
+                "columns": expected_cols
+            },
+            "seed": {
+                "metadata": {
+                "schema": my_schema_name,
+                "name": "seed",
+                "type": "view",
+                "comment": None
+                },
+                "columns": expected_cols
+            }
+        }
+        self.verify_catalog(catalog)
+        model_sql_path = self.dir('bq_models/model.sql')
+        seed_sql_path = self.dir('bq_models/seed.sql')
+        expected_manifest = {
+            'nodes': {
+               'model.test.model': {
+                    'alias': 'model',
+                    'config': {
+                        'column_types': {},
+                        'enabled': True,
+                        'materialized': 'view',
+                        'post-hook': [],
+                        'pre-hook': [],
+                        'quoting': {},
+                        'vars': {}
+                    },
+                    'depends_on': {
+                        'macros': [],
+                        'nodes': ['model.test.seed']
+                    },
+                    'empty': False,
+                    'fqn': ['test', 'model'],
+                    'name': 'model',
+                    'original_file_path': model_sql_path,
+                    'package_name': 'test',
+                    'path': 'model.sql',
+                    'raw_sql': open(model_sql_path).read().rstrip('\n'),
+                    'refs': [['seed']],
+                    'resource_type': 'model',
+                    'root_path': os.getcwd(),
+                    'schema': my_schema_name,
+                    'tags': [],
+                    'unique_id': 'model.test.model'
+                },
+                'model.test.seed': {
+                    'alias': 'seed',
+                    'config': {
+                        'column_types': {},
+                        'enabled': True,
+                        'materialized': 'view',
+                        'post-hook': [],
+                        'pre-hook': [],
+                        'quoting': {},
+                        'vars': {}
+                    },
+                    'depends_on': {
+                        'macros': [],
+                        'nodes': []
+                    },
+                    'empty': False,
+                    'fqn': ['test', 'seed'],
+                    'name': 'seed',
+                    'original_file_path': seed_sql_path,
+                    'package_name': 'test',
+                    'path': 'seed.sql',
+                    'raw_sql': open(seed_sql_path).read().rstrip('\n'),
+                    'refs': [],
+                    'resource_type': 'model',
+                    'root_path': os.getcwd(),
+                    'schema': my_schema_name,
+                    'tags': [],
+                    'unique_id': 'model.test.seed'
+                }
+            },
+            'child_map': {
+               'model.test.model': [],
+                'model.test.seed': ['model.test.model']
+            },
+            'parent_map': {
+                'model.test.model': ['model.test.seed'],
+                'model.test.seed': []
+            },
+        }
+        self.verify_manifest(expected_manifest)