Skip to content

Commit

Permalink
Fix bigquery bulk complete failing (#2441)
Browse files Browse the repository at this point in the history
* bugfix bigquery bulk complete failing when argument is a generator
  • Loading branch information
drj42 authored and Tarrasch committed Jun 16, 2018
1 parent bda236d commit 1a4802d
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 3 deletions.
5 changes: 2 additions & 3 deletions luigi/contrib/bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -420,11 +420,10 @@ class MixinBigQueryBulkComplete(object):

@classmethod
def bulk_complete(cls, parameter_tuples):
if len(parameter_tuples) < 1:
return

# Instantiate the tasks to inspect them
tasks_with_params = [(cls(p), p) for p in parameter_tuples]
if not tasks_with_params:
return

# Grab the set of BigQuery datasets we are interested in
datasets = {t.output().table.dataset for t, p in tasks_with_params}
Expand Down
8 changes: 8 additions & 0 deletions test/contrib/bigquery_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,14 @@ def test_bulk_complete(self):
complete = list(TestRunQueryTask.bulk_complete(parameters))
self.assertEqual(complete, ['table2'])

# Test that bulk_complete accepts lazy sequences in addition to lists
def parameters_gen():
yield 'table1'
yield 'table2'

complete = list(TestRunQueryTask.bulk_complete(parameters_gen()))
self.assertEqual(complete, ['table2'])

def test_dataset_doesnt_exist(self):
client = MagicMock()
client.dataset_exists.return_value = False
Expand Down

0 comments on commit 1a4802d

Please sign in to comment.