feat(sqlalchemy): support unknown types

ibis/backends/base/sql/alchemy/datatypes.py

@@ -115,6 +115,10 @@ def compiles_uuid(element, compiler, **kw):
         return "UUID"
 
 
+class Unknown(sa.Text):
+    pass
+
+
 # TODO(cleanup)
 ibis_type_to_sqla = {
     dt.Null: sat.NullType,
@@ -141,6 +145,7 @@ def compiles_uuid(element, compiler, **kw):
     dt.UInt64: UInt64,
     dt.JSON: sa.JSON,
     dt.Interval: sa.Interval,
+    dt.Unknown: Unknown,
 }
 
 
@@ -262,6 +267,11 @@ def sa_json(_, satype, nullable=True):
     return dt.JSON(nullable=nullable)
 
 
+@dt.dtype.register(Dialect, Unknown)
+def sa_unknown(_, satype, nullable=True):
+    return dt.Unknown(nullable=nullable)
+
+
 if geospatial_supported:
 
     @dt.dtype.register(Dialect, (ga.Geometry, ga.types._GISType))

ibis/backends/postgres/datatypes.py

@@ -49,11 +49,8 @@ def _get_type(typestr: str) -> dt.DataType:
     try:
         return _parse_numeric(typestr)
     except parsy.ParseError:
-        # postgres can have arbitrary types unknown to ibis, so we just
-        # consider them null since we can't know what to do with them without
-        # explicit support, return null to effectively give no public API to
-        # such columns
-        return dt.null
+        # postgres can have arbitrary types unknown to ibis
+        return dt.unknown
 
 
 _type_mapping = {
@@ -181,4 +178,4 @@ def sa_pg_array(dialect, satype, nullable=True):
 
 @dt.dtype.register(PGDialect, postgresql.TSVECTOR)
 def sa_postgres_tsvector(_, satype, nullable=True):
-    return dt.String(nullable=nullable)
+    return dt.Unknown(nullable=nullable)

ibis/backends/postgres/tests/test_client.py

@@ -209,9 +209,7 @@ def test_get_schema_from_query(con, pg_type, expected_type):
     assert result_schema == expected_schema
 
 
-@pytest.mark.parametrize(
-    ("col", "expected_type"), [("search", dt.string), ("simvec", dt.null)]
-)
-def test_unknown_column_type(con, col, expected_type):
+@pytest.mark.parametrize("col", ["search", "simvec"])
+def test_unknown_column_type(con, col):
     awards_players = con.table("awards_players")
-    assert awards_players[col].type() == expected_type
+    assert awards_players[col].type().is_unknown()

ibis/backends/pyarrow/datatypes.py

@@ -28,6 +28,8 @@
     dt.Date: pa.date64(),
     dt.JSON: pa.string(),
     dt.Null: pa.null(),
+    # assume unknown types can be converted into strings
+    dt.Unknown: pa.string(),
 }
 
 

ibis/expr/datatypes/core.py

@@ -269,6 +269,9 @@ def is_uint64(self) -> bool:
     def is_uint8(self) -> bool:
         return isinstance(self, UInt8)
 
+    def is_unknown(self) -> bool:
+        return isinstance(self, Unknown)
+
     def is_unsigned_integer(self) -> bool:
         return isinstance(self, UnsignedInteger)
 
@@ -284,6 +287,14 @@ def from_ibis_dtype(value: DataType) -> DataType:
     return value
 
 
+@public
+class Unknown(DataType, Singleton):
+    """An unknown type."""
+
+    scalar = ir.UnknownScalar
+    column = ir.UnknownColumn
+
+
 @public
 class Primitive(DataType, Singleton):
     """Values with known size."""
@@ -922,6 +933,7 @@ class INET(String):
 macaddr = MACADDR()
 inet = INET()
 decimal = Decimal()
+unknown = Unknown()
 
 Enum = String
 
@@ -1023,6 +1035,7 @@ def from_numpy_dtype(value):
     macaddr=macaddr,
     inet=inet,
     decimal=decimal,
+    unknown=unknown,
     Enum=Enum,
     Geography=GeoSpatial,
     Geometry=GeoSpatial,

ibis/expr/types/generic.py

@@ -5,10 +5,10 @@
 from public import public
 
 import ibis
-from ibis import util
 import ibis.common.exceptions as com
 import ibis.expr.datatypes as dt
 import ibis.expr.operations as ops
+from ibis import util
 from ibis.common.grounds import Singleton
 from ibis.expr.types.core import Expr, _binop, _FixedTextJupyterMixin
 
@@ -1113,6 +1113,21 @@ def nth(self, n: int | ir.IntegerValue) -> Column:
         return ops.NthValue(self, n).to_expr()
 
 
+@public
+class UnknownValue(Value):
+    pass
+
+
+@public
+class UnknownScalar(Scalar):
+    pass
+
+
+@public
+class UnknownColumn(Column):
+    pass
+
+
 @public
 class NullValue(Value):
     pass