feat(polars): add streaming kwarg to to_pandas

ibis-project · Dec 12, 2023 · 703507f · 703507f
1 parent 36e1db5
commit 703507f
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 5 deletions.
diff --git a/docs/posts/pydata-performance-part2/polars_ibis.py b/docs/posts/pydata-performance-part2/polars_ibis.py
@@ -30,4 +30,4 @@
     .dropna("ext")
     .order_by([_.month.desc(), _.project_count.desc()])
 )
-df = expr.to_pandas()
+df = expr.to_pandas(streaming=True)
diff --git a/ibis/backends/polars/__init__.py b/ibis/backends/polars/__init__.py
@@ -404,15 +404,16 @@ def execute(
         expr: ir.Expr,
         params: Mapping[ir.Expr, object] | None = None,
         limit: int | None = None,
+        streaming: bool = False,
         **kwargs: Any,
     ):
         lf = self.compile(expr, params=params, **kwargs)
         if limit == "default":
             limit = ibis.options.sql.default_limit
         if limit is not None:
-            df = lf.fetch(limit, streaming=True)
+            df = lf.fetch(limit, streaming=streaming)
         else:
-            df = lf.collect(streaming=True)
+            df = lf.collect(streaming=streaming)
 
         if isinstance(expr, (ir.Table, ir.Scalar)):
             return expr.__pandas_result__(df.to_pandas())
@@ -429,13 +430,14 @@ def _to_pyarrow_table(
         expr: ir.Expr,
         params: Mapping[ir.Expr, object] | None = None,
         limit: int | None = None,
+        streaming: bool = False,
         **kwargs: Any,
     ):
         lf = self.compile(expr, params=params, **kwargs)
         if limit is not None:
-            df = lf.fetch(limit, streaming=True)
+            df = lf.fetch(limit, streaming=streaming)
         else:
-            df = lf.collect(streaming=True)
+            df = lf.collect(streaming=streaming)
 
         table = df.to_arrow()
         if isinstance(expr, (ir.Table, ir.Value)):