fix(deps): update dependency datafusion to v39 (#9506)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com>

ibis/backends/datafusion/tests/test_udf.py

@@ -70,7 +70,7 @@ def median(a: float) -> float:
 
 
 @pytest.mark.xfail(
-    condition=vparse(datafusion.__version__) == vparse("38.0.1"),
+    condition=vparse(datafusion.__version__) >= vparse("38.0.1"),
     reason="internal error about MEDIAN(G) naming",
 )
 def test_builtin_agg_udf_filtered(con):

ibis/backends/tests/test_generic.py

@@ -1482,7 +1482,7 @@ def test_pivot_longer(backend):
 
 
 @pytest.mark.xfail_version(
-    datafusion=["datafusion==38.0.1"], reason="internal error about MEDIAN(G) naming"
+    datafusion=["datafusion>=38.0.1"], reason="internal error about MEDIAN(G) naming"
 )
 def test_pivot_wider(backend):
     diamonds = backend.diamonds

ibis/backends/tests/tpc/ds/test_queries.py

@@ -281,10 +281,6 @@ def test_07(store_sales, customer_demographics, date_dim, item, promotion):
 
 
 @tpc_test("ds")
-@pytest.mark.broken(
-    ["datafusion"],
-    reason='The left or right side of the join does not have all columns on "on"',
-)
 def test_08(store_sales, date_dim, store, customer_address, customer):
     zip_codes = (
         "24128",

pyproject.toml

@@ -59,7 +59,7 @@ dask = { version = ">=2022.9.1,<2024.3.0", optional = true, extras = [
   "array",
   "dataframe",
 ] }
-datafusion = { version = ">=0.6,<39", optional = true }
+datafusion = { version = ">=0.6,<40", optional = true }
 db-dtypes = { version = ">=0.3,<2", optional = true }
 deltalake = { version = ">=0.9.0,<1", optional = true }
 duckdb = { version = ">=0.8.1,<2", optional = true }

requirements-dev.txt

@@ -42,7 +42,7 @@ crashtest==0.4.1 ; python_version >= "3.10" and python_version < "4.0"
 cryptography==42.0.8 ; python_version >= "3.10" and python_version < "4.0"
 cycler==0.12.1 ; python_version >= "3.10" and python_version < "3.13"
 dask[array,dataframe]==2024.2.1 ; python_version >= "3.10" and python_version < "4.0"
-datafusion==38.0.1 ; python_version >= "3.10" and python_version < "4.0"
+datafusion==39.0.0 ; python_version >= "3.10" and python_version < "4.0"
 db-dtypes==1.2.0 ; python_version >= "3.10" and python_version < "4.0"
 debugpy==1.8.2 ; python_version >= "3.10" and python_version < "3.13"
 decorator==5.1.1 ; python_version >= "3.10" and python_version < "4.0"