Support ibis kernel computations (#38)

.github/workflows/pr_checks.yml

@@ -2,7 +2,7 @@ name: PR Checks
 
 on:
   pull_request:
-    types: [opened, synchronize]
+    types: [opened, synchronize, draft]
   push:
     branches: [main]
 

.gitignore

@@ -165,6 +165,9 @@ cython_debug/
 #.idea/
 script.*
 tmp_graphic_walker.json
-tmp.db
 tmp.ibis.db
+tmp.ibis.db.wal
+tmp.db
 tmp.db.wal
+tmp.ibis.duckdb.db
+tmp.ibis.sqlite.db

README.md

@@ -189,6 +189,7 @@ Our dream is that this package is super simple to use and supports your use case
 | Pandas | ✅ | ✅ | |
 | Polars | ✅ | ✅ | |
 | DuckDB Relation | ✅ | ✅ | |
+| Ibis Table | ✅ | ✅ | Too good to be True. Please report feedback. |
 | Dask | ✅ | ❌ | [Not supported by Pygwalker](https://github.com/Kanaries/pygwalker/issues/658) |
 | Pygwalker Database Connector | ❌ | ❌ | [Not supported by Narwhals](https://github.com/narwhals-dev/narwhals/issues/1289) |
 

examples/reference/backends.py

@@ -1,5 +1,6 @@
 import dask.dataframe as dd
 import duckdb
+import ibis
 import pandas as pd
 import panel as pn
 import polars as pl
@@ -8,8 +9,8 @@
 
 pn.extension()
 
-
 DATA = "https://datasets.holoviz.org/significant_earthquakes/v1/significant_earthquakes.parquet"
+
 df_pandas = pd.read_parquet(DATA)
 duckdb_simple = duckdb.sql("SELECT * FROM df_pandas")
 
@@ -19,10 +20,22 @@
 con_persistent = duckdb.connect("tmp.db")
 duckdb_persistent = con_persistent.sql("SELECT * FROM df_pandas")
 
+con = ibis.connect("duckdb://tmp.ibis.duckdb.db")
+if not "my_table" in con.list_tables():
+    con.read_parquet(DATA, "my_table")
+ibis_duckdb_table = con.table("my_table")
+
+con = ibis.connect("sqlite://tmp.ibis.sqlite.db")
+if not "my_table" in con.list_tables():
+    con.create_table("my_table", df_pandas)
+ibis_sqlite_table = con.table("my_table")
+
 DATAFRAMES = {
     "pandas": df_pandas,
     "polars": pl.read_parquet(DATA),
     "dask": dd.read_parquet(DATA, npartitions=1),
+    "ibis-duckdb": ibis_duckdb_table,
+    "ibis-sqlite": ibis_sqlite_table,
     "duckdb-simple": duckdb_simple,
     "duckdb in-memory": duckdb_in_memory,
     "duckdb persistent": duckdb_persistent,
@@ -31,6 +44,7 @@
 select = pn.widgets.Select(options=list(DATAFRAMES), name="Data Source")
 kernel_computation = pn.widgets.Checkbox(name="Kernel Computation", value=False)
 
+
 if pn.state.location:
     pn.state.location.sync(select, {"value": "backend"})
     pn.state.location.sync(kernel_computation, {"value": "kernel_computation"})

pyproject.toml

@@ -30,6 +30,7 @@ tests = [
   "duckdb",
   "fastparquet",
   "gw-dsl-parser",
+  "ibis-framework[duckdb,sqlite]",
   "polars",
   "pygwalker",
   "pytest-asyncio",
@@ -41,6 +42,7 @@ examples = [
   "duckdb",
   "fastparquet",
   "gw-dsl-parser",
+  "ibis-framework[duckdb,sqlite]",
   "polars",
   "pygwalker",
   "requests",

src/panel_gwalker/_pygwalker.py

@@ -1,6 +1,7 @@
-import sys
 from typing import TYPE_CHECKING, Any, Dict, List, Protocol, runtime_checkable
 
+from narwhals.dependencies import is_duckdb_relation, is_ibis_table
+
 if TYPE_CHECKING:
     try:
         from pygwalker.data_parsers.base import BaseDataParser
@@ -33,6 +34,27 @@ def _convert_to_field_spec(spec: dict) -> dict:
     }
 
 
+def get_ibis_dataframe_parser():
+    from pygwalker.data_parsers.pandas_parser import PandasDataFrameDataParser
+    from pygwalker.services.fname_encodings import rename_columns
+
+    class IbisDataFrameParser(PandasDataFrameDataParser):
+        def _rename_dataframe(self, df):
+            df = df.rename(
+                {
+                    old_col: new_col
+                    for old_col, new_col in zip(df.columns, rename_columns(df.columns))
+                }
+            )
+            return df
+
+    @property
+    def dataset_type(self) -> str:
+        return "ibis_dataframe"
+
+    return IbisDataFrameParser
+
+
 @runtime_checkable
 class ConnectorP(Protocol):
     def query_datas(self, sql: str) -> List[Dict[str, Any]]: ...
@@ -92,23 +114,22 @@ def get_data_parser(
             "Server dependencies are not installed. Please: pip install panel-graphic-walker[kernel]."
         ) from exc
 
-    custom_connector = None
-    if "duckdb" in sys.modules:
-        from duckdb.duckdb import DuckDBPyRelation
+    object_type = type(object)
 
-        if isinstance(object, DuckDBPyRelation):
-            custom_connector = DuckDBPyRelationConnector(object)
+    if is_ibis_table(object):
+        IbisDataFrameParser = get_ibis_dataframe_parser()
+        __classname2method[object_type] = (IbisDataFrameParser, "ibis")
 
-    if custom_connector:
-        object = custom_connector
-        __classname2method[DatabaseDataParser] = (DatabaseDataParser, "connector")
-        parser, name = __classname2method[DatabaseDataParser]
-    else:
-        try:
-            parser, name = _get_data_parser_pygwalker(object)
-        except TypeError as exc:
-            msg = f"Data type {type(object)} is currently not supported"
-            raise NotImplementedError(msg) from exc
+    if is_duckdb_relation(object):
+        object = DuckDBPyRelationConnector(object)
+        object_type = type(object)
+        __classname2method[object_type] = (DatabaseDataParser, "duckdb")
+
+    try:
+        parser, name = _get_data_parser_pygwalker(object)
+    except TypeError as exc:
+        msg = f"Data type {type(object)} is currently not supported"
+        raise NotImplementedError(msg) from exc
 
     _field_specs = [FieldSpec(**_convert_to_field_spec(spec)) for spec in field_specs]
     return parser(

src/panel_gwalker/_tabular_data.py

@@ -8,6 +8,7 @@
 from narwhals.dependencies import (
     is_dask_dataframe,
     is_duckdb_relation,
+    is_ibis_table,
     is_into_dataframe,
     is_polars_lazyframe,
 )
@@ -25,6 +26,7 @@ def _validate(val: Any):
         or is_dask_dataframe(val)
         or is_polars_lazyframe(val)
         or is_duckdb_relation(val)
+        or is_ibis_table(val)
     ):
         return
 

tests/conftest.py

@@ -1,5 +1,6 @@
 import dask.dataframe as dd
 import duckdb
+import ibis
 import pandas as pd
 import polars as pl
 import pytest
@@ -30,6 +31,8 @@ def persistent_conn(tmp_path):
         "pandas",
         "polars",
         "dask",
+        "ibis-duckdb-persistent",
+        "ibis-sqlite",
         "duckdb-simple",
         "duckdb-in-memory",
         "duckdb-persistent",
@@ -42,6 +45,18 @@ def data(request, tmp_path, memory_conn, persistent_conn):
         return pl.DataFrame({"a": [1, 2, 3]})
     if request.param == "dask":
         return dd.from_pandas(pd.DataFrame({"a": [1, 2, 3]}), npartitions=1)
+    if request.param == "ibis-duckdb-persistent":
+        path = (tmp_path / "tmp.ibis.db").as_posix()
+        con = ibis.duckdb.connect(path)
+        table = con.create_table("my_table", schema=ibis.schema(dict(a="int64")))
+        con.insert("my_table", obj=[(1,), (2,), (3,)])
+        return table
+    if request.param == "ibis-sqlite":
+        path = (tmp_path / "tmp.ibis.db").as_posix()
+        con = ibis.sqlite.connect(path)
+        table = con.create_table("my_table", schema=ibis.schema(dict(a="int64")))
+        con.insert("my_table", obj=[(1,), (2,), (3,)])
+        return table
     if request.param == "duckdb-simple":
         df_pandas = pd.DataFrame({"a": [1, 2, 3]})
         return duckdb.sql("SELECT * FROM df_pandas")

tests/test_pygwalker.py

@@ -1,9 +1,11 @@
+import dask.dataframe as dd
 import pytest
 
 from panel_gwalker._gwalker import get_data_parser
 
 
 def test_get_data_parser(data):
-    if str(type(data)) == "<class 'dask_expr._collection.DataFrame'>":
-        pytest.xfail("Dask DataFrame is not supported yet")
+    if isinstance(data, dd.DataFrame):
+        pytest.xfail(f"Unsupported data type: {type(data)}")
+
     assert get_data_parser(data, [], False, False, {})