From ee17999414ba64392de8527b565dd8ea8db02140 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 1 Aug 2019 12:40:00 -0600 Subject: [PATCH] for agate, use the "Urb" mode on python 2, handle BOM fiddling --- core/dbt/clients/agate_helper.py | 12 +++--------- core/dbt/compat.py | 16 +++++++++------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/core/dbt/clients/agate_helper.py b/core/dbt/clients/agate_helper.py index f810f2ff176..9397695de01 100644 --- a/core/dbt/clients/agate_helper.py +++ b/core/dbt/clients/agate_helper.py @@ -1,11 +1,7 @@ -from codecs import BOM_UTF8 - import dbt.compat import agate -BOM = BOM_UTF8.decode('utf-8') # '\ufeff' - DEFAULT_TYPE_TESTER = agate.TypeTester(types=[ agate.data_types.Number(null_values=('null', '')), agate.data_types.TimeDelta(null_values=('null', '')), @@ -46,9 +42,7 @@ def as_matrix(table): def from_csv(abspath): - with dbt.compat.open_file(abspath) as fp: - if fp.read(1) != BOM: + with dbt.compat.open_seed_file(abspath) as fp: + if fp.read(len(dbt.compat.BOM_UTF8)) != dbt.compat.BOM_UTF8: fp.seek(0) - - file_buf = dbt.compat.read_into_buffer(fp) - return agate.Table.from_csv(file_buf, column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_csv(fp, column_types=DEFAULT_TYPE_TESTER) diff --git a/core/dbt/compat.py b/core/dbt/compat.py index a56b9fbae98..6eba90f2ee1 100644 --- a/core/dbt/compat.py +++ b/core/dbt/compat.py @@ -100,16 +100,18 @@ def open_file(path): return open(path, encoding='utf-8') -def read_into_buffer(fp): - buf = StringIO() +if WHICH_PYTHON == 2: + BOM_UTF8 = codecs.BOM_UTF8 +else: + BOM_UTF8 = codecs.BOM_UTF8.decode('utf-8') + +def open_seed_file(path): if WHICH_PYTHON == 2: - buf.write(fp.read().encode('utf-8')) + fp = open(path, 'Urb') else: - buf.write(fp.read()) - - buf.seek(0) - return buf + fp = open(path, encoding='utf-8') + return fp if WHICH_PYTHON == 2: