fix(pyspark): set lower bound of pyspark to 3.3.3 to avoid maintenanc…

…e burden of pytest collection hook (#9606)

Increase the lower bound of pyspark to avoid a big hack in test
collection. Also reduces hacking around numpy imports when making those
optional in #9564.

.github/workflows/ibis-backends.yml

@@ -680,7 +680,7 @@ jobs:
       matrix:
         include:
           - python-version: "3.10"
-            pyspark-version: "3.3"
+            pyspark-version: "3.3.3"
             deps:
               - "'pandas@<2'"
               - "'numpy@<1.24'"

ibis/backends/conftest.py

@@ -10,7 +10,6 @@
 from typing import TYPE_CHECKING, Any
 
 import _pytest
-import numpy as np
 import pandas as pd
 import pytest
 from packaging.requirements import Requirement
@@ -237,11 +236,6 @@ def pytest_collection_modifyitems(session, config, items):
     all_backends = _get_backend_names()
     additional_markers = []
 
-    try:
-        import pyspark
-    except ImportError:
-        pyspark = None
-
     unrecognized_backends = set()
     for item in items:
         # Yell loudly if unrecognized backend in notimpl, notyet or never
@@ -271,23 +265,6 @@ def pytest_collection_modifyitems(session, config, items):
             if not any(item.iter_markers(name="benchmark")):
                 item.add_marker(pytest.mark.core)
 
-        # skip or xfail pyspark tests that run afoul of our non-ancient stack
-        for _ in item.iter_markers(name="pyspark"):
-            if not isinstance(item, pytest.DoctestItem):
-                additional_markers.append(
-                    (
-                        item,
-                        [
-                            pytest.mark.skipif(
-                                pyspark is not None
-                                and vparse(pyspark.__version__) < vparse("3.3.3")
-                                and vparse(np.__version__) >= vparse("1.24"),
-                                reason="PySpark doesn't support numpy >= 1.24",
-                            ),
-                        ],
-                    )
-                )
-
     if unrecognized_backends:
         raise pytest.PytestCollectionWarning("\n" + "\n".join(unrecognized_backends))
 

pyproject.toml

@@ -81,7 +81,7 @@ pydruid = { version = ">=0.6.7,<1", optional = true }
 pyexasol = { version = ">=0.25.2,<1", optional = true, extras = ["pandas"] }
 pymysql = { version = ">=1,<2", optional = true }
 pyodbc = { version = ">=4.0.39,<6", optional = true }
-pyspark = { version = ">=3,<4", optional = true }
+pyspark = { version = ">=3.3.3,<4", optional = true }
 # used to support posix regexen in the pandas, dask and sqlite backends
 regex = { version = ">=2021.7.6", optional = true }
 shapely = { version = ">=2,<3", optional = true }