Skip to content

Commit

Permalink
for agate, use the "Urb" mode on python 2, handle BOM fiddling
Browse files Browse the repository at this point in the history
  • Loading branch information
Jacob Beck committed Aug 1, 2019
1 parent 7001afb commit ee17999
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 16 deletions.
12 changes: 3 additions & 9 deletions core/dbt/clients/agate_helper.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
from codecs import BOM_UTF8

import dbt.compat

import agate

BOM = BOM_UTF8.decode('utf-8') # '\ufeff'

DEFAULT_TYPE_TESTER = agate.TypeTester(types=[
agate.data_types.Number(null_values=('null', '')),
agate.data_types.TimeDelta(null_values=('null', '')),
Expand Down Expand Up @@ -46,9 +42,7 @@ def as_matrix(table):


def from_csv(abspath):
with dbt.compat.open_file(abspath) as fp:
if fp.read(1) != BOM:
with dbt.compat.open_seed_file(abspath) as fp:
if fp.read(len(dbt.compat.BOM_UTF8)) != dbt.compat.BOM_UTF8:
fp.seek(0)

file_buf = dbt.compat.read_into_buffer(fp)
return agate.Table.from_csv(file_buf, column_types=DEFAULT_TYPE_TESTER)
return agate.Table.from_csv(fp, column_types=DEFAULT_TYPE_TESTER)
16 changes: 9 additions & 7 deletions core/dbt/compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,16 +100,18 @@ def open_file(path):
return open(path, encoding='utf-8')


def read_into_buffer(fp):
buf = StringIO()
if WHICH_PYTHON == 2:
BOM_UTF8 = codecs.BOM_UTF8
else:
BOM_UTF8 = codecs.BOM_UTF8.decode('utf-8')


def open_seed_file(path):
if WHICH_PYTHON == 2:
buf.write(fp.read().encode('utf-8'))
fp = open(path, 'Urb')
else:
buf.write(fp.read())

buf.seek(0)
return buf
fp = open(path, encoding='utf-8')
return fp


if WHICH_PYTHON == 2:
Expand Down

0 comments on commit ee17999

Please sign in to comment.