Skip to content

Commit

Permalink
fix(pyarrow): avoid catching ValueError and hiding legitimate failures
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud committed Nov 20, 2023
1 parent c83494f commit b7f650c
Showing 1 changed file with 11 additions and 16 deletions.
27 changes: 11 additions & 16 deletions ibis/backends/base/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -327,23 +327,18 @@ def to_pyarrow(
"""
pa = self._import_pyarrow()
self._run_pre_execute_hooks(expr)

table_expr = expr.as_table()
arrow_schema = table_expr.schema().to_pyarrow()
try:
with self.to_pyarrow_batches(
table_expr, params=params, limit=limit, **kwargs
) as reader:
table = (
pa.Table.from_batches(reader)
.rename_columns(table_expr.columns)
.cast(arrow_schema)
)
except pa.lib.ArrowInvalid:
raise
except ValueError:
table = arrow_schema.empty_table()

return expr.__pyarrow_result__(table)
schema = table_expr.schema()
arrow_schema = schema.to_pyarrow()
with self.to_pyarrow_batches(
table_expr, params=params, limit=limit, **kwargs
) as reader:
table = pa.Table.from_batches(reader, schema=arrow_schema)

return expr.__pyarrow_result__(
table.rename_columns(table_expr.columns).cast(arrow_schema)
)

@util.experimental
def to_pyarrow_batches(
Expand Down

0 comments on commit b7f650c

Please sign in to comment.