fix(deps): update dependency datafusion to v38 (#9278)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com>

ibis/backends/datafusion/__init__.py

@@ -503,8 +503,8 @@ def read_delta(
             )
 
         delta_table = DeltaTable(source_table, **kwargs)
-
-        return self.register(delta_table.to_pyarrow_dataset(), table_name=table_name)
+        self.con.register_dataset(table_name, delta_table.to_pyarrow_dataset())
+        return self.table(table_name)
 
     def to_pyarrow_batches(
         self,

ibis/backends/datafusion/tests/test_udf.py

@@ -2,13 +2,14 @@
 
 import pandas.testing as tm
 import pytest
+from packaging.version import parse as vparse
 
 import ibis.expr.datatypes as dt
 import ibis.expr.types as ir
 from ibis import udf
 from ibis.legacy.udf.vectorized import elementwise, reduction
 
-pytest.importorskip("datafusion")
+datafusion = pytest.importorskip("datafusion")
 pc = pytest.importorskip("pyarrow.compute")
 
 with pytest.warns(FutureWarning, match="v9.0"):
@@ -68,6 +69,10 @@ def median(a: float) -> float:
     assert result == con.tables.batting.G.execute().median()
 
 
+@pytest.mark.xfail(
+    condition=vparse(datafusion.__version__) == vparse("38.0.1"),
+    reason="internal error about MEDIAN(G) naming",
+)
 def test_builtin_agg_udf_filtered(con):
     @udf.agg.builtin
     def median(a: float, where: bool = True) -> float:

ibis/backends/tests/test_aggregation.py

@@ -892,6 +892,11 @@ def test_quantile(
                     raises=PsycoPg2InternalError,
                     reason="function covar_pop(integer, integer) does not exist",
                 ),
+                pytest.mark.xfail_version(
+                    datafusion=["datafusion==38.0.1"],
+                    reason="datafusion FILTER syntax seems broken",
+                    strict=False,  # passes with no filter condition
+                ),
             ],
         ),
         param(

ibis/backends/tests/test_generic.py

@@ -1282,6 +1282,9 @@ def test_pivot_longer(backend):
     assert len(res.execute()) == len(expected)
 
 
+@pytest.mark.xfail_version(
+    datafusion=["datafusion==38.0.1"], reason="internal error about MEDIAN(G) naming"
+)
 def test_pivot_wider(backend):
     diamonds = backend.diamonds
     expr = (

pyproject.toml

@@ -60,7 +60,7 @@ dask = { version = ">=2022.9.1,<2024.3.0", optional = true, extras = [
   "array",
   "dataframe",
 ] }
-datafusion = { version = ">=0.6,<38", optional = true }
+datafusion = { version = ">=0.6,<39", optional = true }
 db-dtypes = { version = ">=0.3,<2", optional = true }
 deltalake = { version = ">=0.9.0,<1", optional = true }
 duckdb = { version = ">=0.8.1,<1", optional = true }

requirements-dev.txt

@@ -48,7 +48,7 @@ crashtest==0.4.1 ; python_version >= "3.10" and python_version < "4.0"
 cryptography==42.0.7 ; python_version >= "3.10" and python_version < "4.0"
 cycler==0.12.1 ; python_version >= "3.10" and python_version < "3.13"
 dask[array,dataframe]==2024.2.1 ; python_version >= "3.10" and python_version < "4.0"
-datafusion==37.1.0 ; python_version >= "3.10" and python_version < "4.0"
+datafusion==38.0.1 ; python_version >= "3.10" and python_version < "4.0"
 db-dtypes==1.2.0 ; python_version >= "3.10" and python_version < "4.0"
 debugpy==1.8.1 ; python_version >= "3.10" and python_version < "3.13"
 decorator==5.1.1 ; python_version >= "3.10" and python_version < "4.0"