From 4bea303c5f846678390a131dadb79180202e5de2 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 21 Apr 2020 14:21:52 -0400 Subject: [PATCH 1/3] (#2336) Fix for non-json-serializable values in BigQuery nested columns --- CHANGELOG.md | 1 + core/dbt/clients/agate_helper.py | 3 ++- .../test_bigquery_repeated_records.py | 12 ++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 758f59aee8a..6d8390465d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Fix skipped node count in stdout at the end of a run ([#2095](https://github.com/fishtown-analytics/dbt/issues/2095), [#2310](https://github.com/fishtown-analytics/dbt/pull/2310)) - Fix an issue where BigQuery incorrectly used a relation's quote policy as the basis for the information schema's include policy, instead of the relation's include policy. ([#2188](https://github.com/fishtown-analytics/dbt/issues/2188), [#2325](https://github.com/fishtown-analytics/dbt/pull/2325)) - Fix "dbt deps" command so it respects the "--project-dir" arg if specified. ([#2338](https://github.com/fishtown-analytics/dbt/issues/2338), [#2339](https://github.com/fishtown-analytics/dbt/issues/2339)) +- Fix Object of type Decimal is not JSON serializable" error when BigQuery queries returned numeric types in nested data structures ([#2336](https://github.com/fishtown-analytics/dbt/issues/2336), [#2348](https://github.com/fishtown-analytics/dbt/pull/2348)) ### Under the hood - Added more tests for source inheritance ([#2264](https://github.com/fishtown-analytics/dbt/issues/2264), [#2291](https://github.com/fishtown-analytics/dbt/pull/2291)) diff --git a/core/dbt/clients/agate_helper.py b/core/dbt/clients/agate_helper.py index 9e26dc88a71..29f285edac3 100644 --- a/core/dbt/clients/agate_helper.py +++ b/core/dbt/clients/agate_helper.py @@ -4,6 +4,7 @@ import datetime import isodate import json +import dbt.utils from typing import Iterable, List, Dict, Union, Optional, Any from dbt.exceptions import RuntimeException @@ -92,7 +93,7 @@ def table_from_data_flat(data, column_names: Iterable[str]) -> agate.Table: row = [] for value in list(_row.values()): if isinstance(value, (dict, list, tuple)): - row.append(json.dumps(value)) + row.append(json.dumps(value, cls=dbt.utils.JSONEncoder)) else: row.append(value) rows.append(row) diff --git a/test/integration/022_bigquery_test/test_bigquery_repeated_records.py b/test/integration/022_bigquery_test/test_bigquery_repeated_records.py index 17c529291b0..06b22fceec0 100644 --- a/test/integration/022_bigquery_test/test_bigquery_repeated_records.py +++ b/test/integration/022_bigquery_test/test_bigquery_repeated_records.py @@ -26,8 +26,8 @@ def test__bigquery_fetch_nested_records(self): cast('Stonebreaker' as string) as lname ) as user, [ - struct(1 as val_1, 2 as val_2), - struct(3 as val_1, 4 as val_2) + struct(1 as val_1, cast(2.12 as numeric) as val_2), + struct(3 as val_1, cast(4.83 as numeric) as val_2) ] as val union all @@ -38,8 +38,8 @@ def test__bigquery_fetch_nested_records(self): cast('Brickmaker' as string) as lname ) as user, [ - struct(7 as val_1, 8 as val_2), - struct(9 as val_1, 0 as val_2) + struct(7 as val_1, cast(8 as numeric) as val_2), + struct(9 as val_1, cast(null as numeric) as val_2) ] as val """ @@ -54,8 +54,8 @@ def test__bigquery_fetch_nested_records(self): '{"fname": "Johnny", "lname": "Brickmaker"}' ], "val": [ - '[{"val_1": 1, "val_2": 2}, {"val_1": 3, "val_2": 4}]', - '[{"val_1": 7, "val_2": 8}, {"val_1": 9, "val_2": 0}]' + '[{"val_1": 1, "val_2": 2.12}, {"val_1": 3, "val_2": 4.83}]', + '[{"val_1": 7, "val_2": 8}, {"val_1": 9, "val_2": null}]' ] } From 044f8fce4eebdae5751352de53dc17478c9435a6 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 21 Apr 2020 16:19:38 -0400 Subject: [PATCH 2/3] Update CHANGELOG.md Co-Authored-By: Jacob Beck --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d8390465d5..e60e6000fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ - Fix skipped node count in stdout at the end of a run ([#2095](https://github.com/fishtown-analytics/dbt/issues/2095), [#2310](https://github.com/fishtown-analytics/dbt/pull/2310)) - Fix an issue where BigQuery incorrectly used a relation's quote policy as the basis for the information schema's include policy, instead of the relation's include policy. ([#2188](https://github.com/fishtown-analytics/dbt/issues/2188), [#2325](https://github.com/fishtown-analytics/dbt/pull/2325)) - Fix "dbt deps" command so it respects the "--project-dir" arg if specified. ([#2338](https://github.com/fishtown-analytics/dbt/issues/2338), [#2339](https://github.com/fishtown-analytics/dbt/issues/2339)) -- Fix Object of type Decimal is not JSON serializable" error when BigQuery queries returned numeric types in nested data structures ([#2336](https://github.com/fishtown-analytics/dbt/issues/2336), [#2348](https://github.com/fishtown-analytics/dbt/pull/2348)) +- Fix "Object of type Decimal is not JSON serializable" error when BigQuery queries returned numeric types in nested data structures ([#2336](https://github.com/fishtown-analytics/dbt/issues/2336), [#2348](https://github.com/fishtown-analytics/dbt/pull/2348)) ### Under the hood - Added more tests for source inheritance ([#2264](https://github.com/fishtown-analytics/dbt/issues/2264), [#2291](https://github.com/fishtown-analytics/dbt/pull/2291)) From ab886cde16935899a8921f221c30b1fc17b601b4 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 27 Apr 2020 10:47:20 -0600 Subject: [PATCH 3/3] added --no-browser argument to docs serve --- CHANGELOG.md | 3 ++- core/dbt/main.py | 5 +++++ core/dbt/task/serve.py | 9 +++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 758f59aee8a..943daad7cbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ - Users can supply paths as arguments to `--models` and `--select`, either explicitily by prefixing with `path:` or implicitly with no prefix. ([#454](https://github.com/fishtown-analytics/dbt/issues/454), [#2258](https://github.com/fishtown-analytics/dbt/pull/2258)) - dbt now builds the relation cache for "dbt compile" and "dbt ls" as well as "dbt run" ([#1705](https://github.com/fishtown-analytics/dbt/issues/1705), [#2319](https://github.com/fishtown-analytics/dbt/pull/2319)) - Snowflake now uses "show terse objects" to build the relations cache instead of selecting from the information schema ([#2174](https://github.com/fishtown-analytics/dbt/issues/2174), [#2322](https://github.com/fishtown-analytics/dbt/pull/2322)) -- Snowflake now uses "describe table" to get the columns in a relation ([#2260](https://github.com/fishtown-analytics/dbt/issues/2260), [#2324](https://github.com/fishtown-analytics/dbt/pull/2324)) +- Snowflake now uses "describe table" to get the columns in a relation ([#2260](https://github.com/fishtown-analytics/dbt/issues/2260), [#2324](https://github.com/fishtown-analytics/dbt/pull/2324)) +- Added a '--no-browser' argument to "dbt docs serve" so you can serve docs in an environment that only has a CLI browser which would otherwise deadlock dbt ([#2004](https://github.com/fishtown-analytics/dbt/issues/2004), [#2364](https://github.com/fishtown-analytics/dbt/pull/2364)) ### Fixes - When a jinja value is undefined, give a helpful error instead of failing with cryptic "cannot pickle ParserMacroCapture" errors ([#2110](https://github.com/fishtown-analytics/dbt/issues/2110), [#2184](https://github.com/fishtown-analytics/dbt/pull/2184)) diff --git a/core/dbt/main.py b/core/dbt/main.py index 0da30df2695..f1ca3be0d2f 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -536,6 +536,11 @@ def _build_docs_serve_subparser(subparsers, base_subparser): Specify the port number for the docs server. ''' ) + serve_sub.add_argument( + '--no-browser', + dest='open_browser', + action='store_false', + ) serve_sub.set_defaults(cls=serve_task.ServeTask, which='serve', rpc_method=None) return serve_sub diff --git a/core/dbt/task/serve.py b/core/dbt/task/serve.py index 42258eb83c2..4d0e2473498 100644 --- a/core/dbt/task/serve.py +++ b/core/dbt/task/serve.py @@ -31,10 +31,11 @@ def run(self): SimpleHTTPRequestHandler # type: ignore ) # type: ignore - try: - webbrowser.open_new_tab('http://127.0.0.1:{}'.format(port)) - except webbrowser.Error: - pass + if self.args.open_browser: + try: + webbrowser.open_new_tab(f'http://127.0.0.1:{port}') + except webbrowser.Error: + pass try: httpd.serve_forever() # blocks