From 4cef1af8cbdea6ce434f3fa2dd60ad5a995e9377 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 29 Jun 2020 14:57:05 -0600 Subject: [PATCH] On bigquery, persist docs for seeds as well as tables/views --- CHANGELOG.md | 1 + .../bigquery/dbt/adapters/bigquery/impl.py | 15 ++++++++ .../bigquery/macros/materializations/seed.sql | 3 ++ .../060_persist_docs_tests/data/seed.csv | 3 ++ .../060_persist_docs_tests/models/schema.yml | 25 +++++++++++++ .../test_persist_docs.py | 37 ++++++++++++++++++- 6 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 test/integration/060_persist_docs_tests/data/seed.csv diff --git a/CHANGELOG.md b/CHANGELOG.md index 91a5c7833b7..c48140088f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Fixes - Hash name of local packages ([#2600](https://github.com/fishtown-analytics/dbt/pull/2600)) +- On bigquery, also persist docs for seeds ([#2598](https://github.com/fishtown-analytics/dbt/issues/2598), [#2601](https://github.com/fishtown-analytics/dbt/pull/2601)) ## dbt 0.17.1rc2 (June 25, 2020) diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index 0b07eb728bf..5800b2d6b4f 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -615,6 +615,21 @@ def update_column_descriptions(self, relation, columns): new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema) conn.handle.update_table(new_table, ['schema']) + @available.parse_none + def update_table_description(self, database, schema, identifier, description): + conn = self.connections.get_thread_connection() + client = conn.handle + + table_ref = self.connections.table_ref( + database, + schema, + identifier, + conn + ) + table = client.get_table(table_ref) + table.description = description + client.update_table(table, ['description']) + @available.parse_none def alter_table_add_columns(self, relation, columns): diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql index d628673747b..d95cc4e1b10 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql @@ -12,5 +12,8 @@ {%- set column_override = model['config'].get('column_types', {}) -%} {{ adapter.load_dataframe(model['database'], model['schema'], model['alias'], agate_table, column_override) }} + {% if config.persist_relation_docs() and 'description' in model %} + {{ adapter.update_table_description(model['database'], model['schema'], model['alias'], model['description']) }} + {% endif %} {% endmacro %} diff --git a/test/integration/060_persist_docs_tests/data/seed.csv b/test/integration/060_persist_docs_tests/data/seed.csv new file mode 100644 index 00000000000..1a728c8ab74 --- /dev/null +++ b/test/integration/060_persist_docs_tests/data/seed.csv @@ -0,0 +1,3 @@ +id,name +1,Alice +2,Bob diff --git a/test/integration/060_persist_docs_tests/models/schema.yml b/test/integration/060_persist_docs_tests/models/schema.yml index 74dfa0a3378..5a909162456 100644 --- a/test/integration/060_persist_docs_tests/models/schema.yml +++ b/test/integration/060_persist_docs_tests/models/schema.yml @@ -43,3 +43,28 @@ models: -- /* comment */ Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + +seeds: + - name: seed + description: | + Seed model description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + columns: + - name: id + description: | + id Column description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + - name: name + description: | + Some stuff here and then a call to + {{ doc('my_fun_doc')}} diff --git a/test/integration/060_persist_docs_tests/test_persist_docs.py b/test/integration/060_persist_docs_tests/test_persist_docs.py index 44e651701b6..deea4b2323e 100644 --- a/test/integration/060_persist_docs_tests/test_persist_docs.py +++ b/test/integration/060_persist_docs_tests/test_persist_docs.py @@ -148,7 +148,15 @@ def project_config(self): "columns": True, }, } - } + }, + 'seeds': { + 'test': { + '+persist_docs': { + "relation": True, + "columns": True, + }, + } + }, } @use_profile('snowflake') @@ -157,7 +165,31 @@ def test_snowflake_persist_docs(self): @use_profile('bigquery') def test_bigquery_persist_docs(self): + self.run_dbt(['seed']) self.run_dbt() + desc_map = { + 'seed': 'Seed model description', + 'table_model': 'Table model description', + 'view_model': 'View model description', + } + for node_id in ['seed', 'table_model', 'view_model']: + with self.adapter.connection_named('_test'): + client = self.adapter.connections \ + .get_thread_connection().handle + + table_id = "{}.{}.{}".format( + self.default_database, + self.unique_schema(), + node_id + ) + bq_table = client.get_table(table_id) + + bq_schema = bq_table.schema + + assert bq_table.description.startswith(desc_map[node_id]) + assert bq_schema[0].description.startswith('id Column description ') + if not node_id.startswith('view'): + assert bq_schema[1].description.startswith('Some stuff here and then a call to') class TestPersistDocsNested(BasePersistDocsTest): @@ -187,13 +219,14 @@ def test_bigquery_persist_docs(self): Next, generate the catalog and check if the comments are also included. """ + self.run_dbt(['seed']) self.run_dbt() self.run_dbt(['docs', 'generate']) with open('target/catalog.json') as fp: catalog_data = json.load(fp) assert 'nodes' in catalog_data - assert len(catalog_data['nodes']) == 2 # table and view model + assert len(catalog_data['nodes']) == 3 # seed, table, and view model for node_id in ['table_model_nested', 'view_model_nested']: # check the descriptions using the api