feat: drop Python 3.9 and test on Python 3.10/3.12 (#9213)

## Description of changes
According to NEP-29, we could have dropped Python 3.9 last month.

* Test on 3.10/3.12 in CI, except for a few exceptions (e.g. Flink).
* Apply `pyupgrade` rules to remove code for Python<3.10.
* Remove workaround for DuckDB geospatial tests.

.github/workflows/ibis-backends-cloud.yml

@@ -40,21 +40,21 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-          - "3.9"
-          - "3.11"
+          - "3.10"
+          - "3.12"
         backend:
           - name: snowflake
             title: Snowflake
             extras:
               - snowflake
         include:
-          - python-version: "3.9"
+          - python-version: "3.10"
             backend:
               name: bigquery
               title: BigQuery
               extras:
                 - bigquery
-          - python-version: "3.11"
+          - python-version: "3.12"
             backend:
               name: bigquery
               title: BigQuery

.github/workflows/ibis-backends.yml

@@ -52,8 +52,8 @@ jobs:
           - ubuntu-latest
           - windows-latest
         python-version:
-          - "3.9"
-          - "3.11"
+          - "3.10"
+          - "3.12"
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -95,27 +95,21 @@ jobs:
           - ubuntu-latest
           - windows-latest
         python-version:
-          - "3.9"
-          - "3.11"
+          - "3.10"
+          - "3.12"
         backend:
           - name: duckdb
             title: DuckDB
             serial: true
             extras:
               - duckdb
               - deltalake
+              - geospatial
               - examples
               - decompiler
               - polars
             additional_deps:
               - torch
-          # TODO: remove this duckdb job once the next duckdb_spatial is released
-          - name: duckdb
-            title: DuckDB + Geospatial
-            extras:
-              - geospatial
-            additional_deps:
-              - "duckdb==0.9.2"
           - name: clickhouse
             title: ClickHouse
             services:
@@ -243,12 +237,32 @@ jobs:
               extras:
                 - dask
           - os: ubuntu-latest
-            python-version: "3.9"
+            python-version: "3.11"
             backend:
-              name: dask
-              title: Dask
+              name: flink
+              title: Flink
+              serial: true
               extras:
-                - dask
+                - flink
+              additional_deps:
+                - "'apache-flink < 1.20.0'"
+                - "'pandas < 2.2'"
+              services:
+                - flink
+          - os: ubuntu-latest
+            python-version: "3.11"
+            backend:
+              name: impala
+              title: Impala
+              serial: true
+              extras:
+                - impala
+              services:
+                - impala
+                - kudu
+              sys-deps:
+                - cmake
+                - ninja-build
         exclude:
           - os: windows-latest
             backend:
@@ -304,6 +318,21 @@ jobs:
                 - postgres
               sys-deps:
                 - libgeos-dev
+          # TODO(deepyaman): Test whether this works upon releasing https://github.com/cloudera/impyla/commit/bf1f94c3c4106ded6267d2485c1e939775a6a87f
+          - os: ubuntu-latest
+            python-version: "3.12"
+            backend:
+              name: impala
+              title: Impala
+              serial: true
+              extras:
+                - impala
+              services:
+                - impala
+                - kudu
+              sys-deps:
+                - cmake
+                - ninja-build
           - os: windows-latest
             backend:
               name: impala
@@ -354,6 +383,19 @@ jobs:
                 - oracle
               services:
                 - oracle
+          - os: ubuntu-latest
+            python-version: "3.12"
+            backend:
+              name: flink
+              title: Flink
+              serial: true
+              extras:
+                - flink
+              additional_deps:
+                - "'apache-flink < 1.20.0'"
+                - "'pandas < 2.2'"
+              services:
+                - flink
           - os: windows-latest
             backend:
               name: flink
@@ -468,8 +510,8 @@ jobs:
           - ubuntu-latest
           - windows-latest
         python-version:
-          - "3.9"
-          - "3.11"
+          - "3.10"
+          - "3.12"
         backend:
           - name: dask
             title: Dask
@@ -515,7 +557,7 @@ jobs:
               extras:
                 - postgres
                 - geospatial
-          - python-version: "3.11"
+          - python-version: "3.12"
             backend:
               name: postgres
               title: PostgreSQL
@@ -532,7 +574,7 @@ jobs:
               extras:
                 - postgres
                 - geospatial
-          - python-version: "3.11"
+          - python-version: "3.12"
             backend:
               name: dask
               title: Dask
@@ -623,7 +665,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python-version: "3.9"
+          - python-version: "3.10"
             pyspark-version: "3.3"
             deps:
               - "'pandas@<2'"

.github/workflows/ibis-main.yml

@@ -46,7 +46,6 @@ jobs:
           - ubuntu-latest
           - windows-latest
         python-version:
-          - "3.9"
           - "3.10"
           - "3.11"
           - "3.12"

.github/workflows/nix-skip-helper.yml

@@ -33,7 +33,6 @@ jobs:
         os:
           - ubuntu-latest
         python-version:
-          - "3.9"
           - "3.10"
           - "3.11"
           - "3.12"

.github/workflows/nix.yml

@@ -34,7 +34,6 @@ jobs:
         os:
           - ubuntu-latest
         python-version:
-          - "3.9"
           - "3.10"
           - "3.11"
           - "3.12"

ibis/backends/__init__.py

@@ -6,10 +6,9 @@
 import importlib.metadata
 import keyword
 import re
-import sys
 import urllib.parse
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
 from urllib.parse import parse_qs, urlparse
 
 import ibis
@@ -21,7 +20,7 @@
 from ibis.common.caching import RefCountedCache
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Iterator, Mapping, MutableMapping
+    from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping
 
     import pandas as pd
     import polars as pl
@@ -1296,10 +1295,7 @@ def _get_backend_names(*, exclude: tuple[str] = ()) -> frozenset[str]:
 
     """
 
-    if sys.version_info < (3, 10):
-        entrypoints = importlib.metadata.entry_points()["ibis.backends"]
-    else:
-        entrypoints = importlib.metadata.entry_points(group="ibis.backends")
+    entrypoints = importlib.metadata.entry_points(group="ibis.backends")
     return frozenset(ep.name for ep in entrypoints).difference(exclude)
 
 

ibis/backends/bigquery/__init__.py

@@ -7,7 +7,7 @@
 import glob
 import os
 import re
-from typing import TYPE_CHECKING, Any, Callable, Optional
+from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import parse_qs, urlparse
 
 import google.auth.credentials
@@ -39,7 +39,7 @@
 from ibis.backends.sql.datatypes import BigQueryType
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Mapping
+    from collections.abc import Callable, Iterable, Mapping
     from pathlib import Path
 
     import pyarrow as pa

ibis/backends/bigquery/udf/core.py

@@ -8,11 +8,14 @@
 import inspect
 import textwrap
 from collections import ChainMap
-from typing import Callable
+from typing import TYPE_CHECKING
 
 from ibis.backends.bigquery.udf.find import find_names
 from ibis.backends.bigquery.udf.rewrite import rewrite
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 
 class SymbolTable(ChainMap):
     """ChainMap subclass implementing scope for the translator.

ibis/backends/bigquery/udf/rewrite.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
 import ast
-from typing import Callable
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
 
 
 def matches(value: ast.AST, pattern: ast.AST) -> bool:

ibis/backends/clickhouse/tests/conftest.py

@@ -2,7 +2,7 @@
 
 import contextlib
 import os
-from typing import TYPE_CHECKING, Any, Callable
+from typing import TYPE_CHECKING, Any
 
 import pytest
 
@@ -12,7 +12,7 @@
 from ibis.backends.tests.base import ServiceBackendTest
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable
+    from collections.abc import Callable, Iterable
     from pathlib import Path
 
 CLICKHOUSE_HOST = os.environ.get("IBIS_TEST_CLICKHOUSE_HOST", "localhost")

ibis/backends/dask/helpers.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Callable
+from typing import TYPE_CHECKING
 
 import dask.array as da
 import dask.dataframe as dd
@@ -9,6 +9,9 @@
 
 from ibis.backends.pandas.helpers import PandasUtils
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 
 class DaskUtils(PandasUtils):
     @classmethod

ibis/backends/pandas/helpers.py

@@ -2,13 +2,16 @@
 
 import itertools
 import math
-from typing import Callable
+from typing import TYPE_CHECKING
 
 import numpy as np
 import pandas as pd
 
 from ibis.util import gen_name
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 
 def isnull(obj):
     return obj is None or obj is pd.NA or (isinstance(obj, float) and math.isnan(obj))

ibis/backends/postgres/__init__.py

@@ -8,7 +8,7 @@
 from functools import partial
 from itertools import repeat, takewhile
 from operator import itemgetter
-from typing import TYPE_CHECKING, Any, Callable
+from typing import TYPE_CHECKING, Any
 from urllib.parse import parse_qs, urlparse
 
 import numpy as np
@@ -32,6 +32,8 @@
 from ibis.common.exceptions import InvalidDecoratorError
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
     import pandas as pd
     import pyarrow as pa
 

ibis/backends/postgres/tests/test_udf.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import functools
-import sys
 
 import pytest
 
@@ -115,11 +114,6 @@ def mult_a_b(a: int, b: int) -> int:
     assert result["mult_result"].iat[0] == 8
 
 
-@pytest.mark.xfail(
-    condition=sys.version_info[:2] < (3, 9),
-    raises=TypeError,
-    reason="no straightforward way to use new (Python 3.9) annotations syntax",
-)
 def test_array_type(test_database, table):
     """Test that usage of Array types work Other scalar types can be
     represented either by the class or an instance, but Array types work

ibis/backends/sql/compiler.py

@@ -7,7 +7,7 @@
 import operator
 import string
 from functools import partial, reduce
-from typing import TYPE_CHECKING, Any, Callable, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
 
 import sqlglot as sg
 import sqlglot.expressions as sge
@@ -34,7 +34,7 @@
 from ibis.expr.rewrites import lower_stringslice
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Mapping
+    from collections.abc import Callable, Iterable, Mapping
 
     import ibis.expr.schema as sch
     import ibis.expr.types as ir