Skip to content

Commit

Permalink
fix bq seed (#735)
Browse files Browse the repository at this point in the history
* fix bq seed

* revert makefile
  • Loading branch information
drewbanin authored Apr 20, 2018
1 parent d966ec2 commit c81417d
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 10 deletions.
12 changes: 9 additions & 3 deletions dbt/adapters/bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -514,12 +514,18 @@ def _agate_to_schema(cls, agate_table):
def load_csv_rows(cls, profile, schema, table_name, agate_table):
bq_schema = cls._agate_to_schema(agate_table)
dataset = cls.get_dataset(profile, schema, None)
table = dataset.table(table_name, schema=bq_schema)
table = dataset.table(table_name)
conn = cls.get_connection(profile, None)
client = conn.get('handle')

load_config = google.cloud.bigquery.LoadJobConfig()
load_config.skip_leading_rows = 1
load_config.schema = bq_schema

with open(agate_table.original_abspath, "rb") as f:
job = table.upload_from_file(f, "CSV", rewind=True,
client=client, skip_leading_rows=1)
job = client.load_table_from_file(f, table, rewind=True,
job_config=load_config)

with cls.exception_handler(profile, "LOAD TABLE"):
cls.poll_until_job_completes(job, cls.get_timeout(conn))

Expand Down
5 changes: 5 additions & 0 deletions test/integration/022_bigquery_test/data/data_seed.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
id,dupe
1,a
2,a
3,a
4,a
14 changes: 7 additions & 7 deletions test/integration/022_bigquery_test/models/view.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
)
}}

select 1 as id, current_date as updated_at, 'a' as dupe
union all
select 2 as id, current_date as updated_at, 'a' as dupe
union all
select 3 as id, current_date as updated_at, 'a' as dupe
union all
select 4 as id, current_date as updated_at, 'a' as dupe

select
id,
current_date as updated_at,
dupe

from {{ ref('data_seed') }}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ def models(self):
@property
def project_config(self):
return {
'data-paths': ['test/integration/022_bigquery_test/data'],
'macro-paths': ['test/integration/022_bigquery_test/macros'],
}

Expand All @@ -26,6 +27,7 @@ def profile_config(self):
def test__bigquery_simple_run(self):
self.use_profile('bigquery')
self.use_default_project()
self.run_dbt(['seed'])
self.run_dbt()

# The 'dupe' model should fail, but all others should pass
Expand Down

0 comments on commit c81417d

Please sign in to comment.