Support more data sources (#22)

.github/workflows/pr_checks.yml

@@ -22,10 +22,13 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade uv
+          uv venv
           uv pip install -e .[dev,tests]
 
       - name: Run pre-commit
         uses: pre-commit/action@v3.0.1
 
       - name: Run pytest
-        run: pytest
+        run: |
+          source .venv/bin/activate
+          pytest tests

README.md

@@ -182,6 +182,20 @@ Our dream is that this package is super simple to use and supports your use case
 - Supports persisting and reusing Graphic Walker specifications.
 - Scales to even the largest datasets, only limited by your server, cluster, or database.
 
+## Supported Backends
+
+| Name | `kernel_computation=False` | `kernel_computation=True` | Comment |
+| ---- | - | - | - |
+| Pandas | ✅ | ✅ | |
+| Polars | ✅ | ✅ | |
+| Dask | ✅ | ❌ | [Not supported by Pygwalker](https://github.com/Kanaries/pygwalker/issues/658) |
+| DuckDB Relation | ✅ | ❌ | [Not supported by Pygwalker](https://github.com/Kanaries/pygwalker/issues/658) |
+| Pygwalker Database Connector | ❌ | ❌ | [Not supported by Narwhals](https://github.com/narwhals-dev/narwhals/issues/1289) |
+
+Other backends might be supported if they are supported by both [Narwhals](https://github.com/narwhals-dev/narwhals) and [PygWalker](https://github.com/Kanaries/pygwalker).
+
+Via the [backends](examples/reference/backends.py) example its possible to explore backends. In the [`data` test fixture](tests/conftest.py) you can see which backends we currently test.
+
 ## ❤️ Contributions
 
 Contributions and co-maintainers are very welcome! Please submit issues or pull requests to the [GitHub repository](https://github.com/panel-extensions/panel-graphic-walker). Check out the [DEVELOPER_GUIDE](DEVELOPER_GUIDE.md) for more information.

examples/bikesharing_dashboard/bikesharing_dashboard.py

@@ -2,6 +2,7 @@
 
 import pandas as pd
 import panel as pn
+import requests
 
 from panel_gwalker import GraphicWalker
 
@@ -10,7 +11,7 @@
 ROOT = Path(__file__).parent
 # Source: https://kanaries-app.s3.ap-northeast-1.amazonaws.com/public-datasets/bike_sharing_dc.csv
 DATASET = "https://datasets.holoviz.org/bikesharing_dc/v1/bikesharing_dc.parquet"
-SPEC_PATH = ROOT / "bikesharing_dashboard.json"
+SPEC = "https://cdn.jsdelivr.net/gh/panel-extensions/panel-graphic-walker@main/examples/bikesharing_dashboard/bikesharing_dashboard.json"
 ACCENT = "#ff4a4a"
 
 if pn.config.theme == "dark":
@@ -34,16 +35,18 @@
 }
 """
 
+
 @pn.cache
 def get_data():
     return pd.read_parquet(DATASET)
 
+
 data = get_data()
 
 walker = GraphicWalker(
     data,
     theme_key="streamlit",
-    spec=SPEC_PATH,
+    spec=SPEC,
     sizing_mode="stretch_both",
     kernel_computation=True,
 )

examples/earthquake_dashboard/earthquake_dashboard.py

@@ -2,6 +2,7 @@
 
 import pandas as pd
 import panel as pn
+import requests
 
 from panel_gwalker import GraphicWalker
 
@@ -29,8 +30,7 @@
 }
 """
 DATASET = "https://datasets.holoviz.org/significant_earthquakes/v1/significant_earthquakes.parquet"
-# https://cdn.jsdelivr.net/gh/panel-extensions/panel-graphic-walker@main/examples/earthquake_dashboard/earthquake_dashboard.json
-SPEC = ROOT / "earthquake_dashboard.json"
+SPEC = "https://cdn.jsdelivr.net/gh/panel-extensions/panel-graphic-walker@main/examples/earthquake_dashboard/earthquake_dashboard.json"
 
 
 @pn.cache

examples/reference/backends.py

@@ -0,0 +1,47 @@
+import dask.dataframe as dd
+import duckdb
+import pandas as pd
+import panel as pn
+import polars as pl
+
+from panel_gwalker import GraphicWalker
+
+pn.extension()
+
+
+DATA = "https://datasets.holoviz.org/significant_earthquakes/v1/significant_earthquakes.parquet"
+df_pandas = pd.read_parquet(DATA)
+duckdb_relation = duckdb.sql("SELECT * FROM df_pandas")
+
+DATAFRAMES = {
+    "pandas": df_pandas,
+    "polars": pl.read_parquet(DATA),
+    "dask": dd.read_parquet(DATA, npartitions=1),
+    "duckdb": duckdb_relation,
+}
+
+select = pn.widgets.Select(options=list(DATAFRAMES), name="Data Source")
+kernel_computation = pn.widgets.Checkbox(name="Kernel Computation", value=False)
+
+
+@pn.depends(select, kernel_computation)
+def get_data(value, kernel_computation):
+    data = DATAFRAMES[value]
+    if not kernel_computation:
+        try:
+            data = data.head(10)
+        except:
+            data = data.df().head(10)
+    try:
+        return GraphicWalker(
+            data,
+            kernel_computation=kernel_computation,
+            sizing_mode="stretch_width",
+            tab="data",
+        )
+    except Exception as ex:
+        msg = f"Combination of {value=} and {kernel_computation=} is currently not supported."
+        return pn.pane.Alert(msg, alert_type="danger")
+
+
+pn.Column(select, kernel_computation, get_data).servable()

examples/reference_app/reference_app.py

@@ -3,6 +3,7 @@
 
 import pandas as pd
 import panel as pn
+import requests
 
 from panel_gwalker import GraphicWalker
 
@@ -13,10 +14,8 @@
 GW_LOGO = "https://kanaries.net/_next/static/media/kanaries-logo.0a9eb041.png"
 GW_API = "https://github.com/Kanaries/graphic-walker"
 GW_GUIDE_URL = "https://docs.kanaries.net/graphic-walker/data-viz/create-data-viz"
-# https://cdn.jsdelivr.net/gh/panel-extensions/panel-graphic-walker@main/examples/reference_app/spec_simple.json
-SPEC_CAPACITY_STATE = ROOT / "spec_capacity_state.json"
-# https://cdn.jsdelivr.net/gh/panel-extensions/panel-graphic-walker@main/examples/reference_app/spec_capacity_state.json
-SPEC_SIMPLE = ROOT / "spec_simple.json"
+SPEC_CAPACITY_STATE = "https://cdn.jsdelivr.net/gh/panel-extensions/panel-graphic-walker@main/examples/reference_app/spec_simple.json"
+SPEC_SIMPLE = "https://cdn.jsdelivr.net/gh/panel-extensions/panel-graphic-walker@main/examples/reference_app/spec_capacity_state.json"
 ACCENT = "#5B8FF9"
 
 

pyproject.toml

@@ -1,13 +1,13 @@
 [project]
 name = "panel-graphic-walker"
-version = "0.4.0"
+version = "0.5.0"
 description = "A project providing a Graphic Walker Pane for use with HoloViz Panel."
 readme = "README.md"
 authors = [
     { name = "Philipp Rudiger", email = "philipp.jfr@gmail.com" }
 ]
 requires-python = ">=3.9"
-dependencies = ["panel>=1.5.2"]
+dependencies = ["panel>=1.5.2", "narwhals"]
 
 [build-system]
 requires = ["hatchling"]
@@ -18,34 +18,42 @@ dev = [
     "jedi-language-server;sys_platform == 'linux'",
     "mypy",
     "pandas-stubs",
+    "types-requests",
     "pre-commit",
     "pytest",
     "ruff",
     "watchfiles",
 ]
 tests = [
+  "aiohttp",
+  "dask[dataframe]",
   "duckdb",
   "fastparquet",
   "gw-dsl-parser",
+  "polars",
   "pygwalker",
   "pytest-asyncio",
   "pytest",
+  "requests",
 ]
 examples = [
+  "dask[dataframe]",
   "duckdb",
   "fastparquet",
   "gw-dsl-parser",
+  "polars",
   "pygwalker",
+  "requests",
 ]
 kernel = [
   "duckdb ; platform_system != 'Emscripten'",
   "gw-dsl-parser ; platform_system != 'Emscripten'",
-  "pygwalker ; platform_system != 'Emscripten'"
+  "pygwalker ; platform_system != 'Emscripten'",
 ]
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/panel_gwalker"]
 
 [[tool.mypy.overrides]]
-module = "param.*,pygwalker.*,gw_dsl_parser.*"
+module = "param.*,pygwalker.*,gw_dsl_parser.*,requests.*"
 ignore_missing_imports = true

src/panel_gwalker/_gwalker.js

@@ -22,25 +22,13 @@ function cleanToDict(value){
     return value
 }
 
-function fetchSpec(url) {
-  return fetch(url)
-    .then(response => response.json())
-    .catch(err => {
-      console.error('Error fetching spec from URL', err);
-    });
-}
-
 function transformSpec(spec) {
   /* The spec must be an null or array of objects */
   if (spec === null) {
     return null;
   }
   if (typeof spec === 'string') {
-    if (spec.startsWith('http://') || spec.startsWith('https://')) {
-      spec = fetchSpec(spec);
-    } else {
-      spec = JSON.parse(spec);
-    }
+    spec = JSON.parse(spec);
   }
 
   if (!Array.isArray(spec)) {

src/panel_gwalker/_gwalker.py

@@ -29,6 +29,7 @@
 from panel.widgets import Button, IntInput, RadioButtonGroup, TextInput
 
 from panel_gwalker._pygwalker import get_data_parser, get_sql_from_payload
+from panel_gwalker._tabular_data import TabularData, TabularDataType
 from panel_gwalker._utils import (
     SPECTYPES,
     SpecType,
@@ -233,7 +234,7 @@ class GraphicWalker(ReactComponent):
     ```
     """
 
-    object: pd.DataFrame = param.DataFrame(
+    object: TabularDataType = TabularData(
         doc="""The data to explore.
         Please note that if you update the `object`, then the existing charts will not be deleted."""
     )
@@ -304,6 +305,15 @@ class GraphicWalker(ReactComponent):
         }
     }
 
+    _rename = {
+        "export": None,
+        "export_mode": None,
+        "export_scope": None,
+        "export_timeout": None,
+        "save": None,
+        "save_path": None,
+    }
+
     _esm = "_gwalker.js"
 
     _THEME_CONFIG = {
@@ -394,11 +404,13 @@ def _handle_msg(self, msg: Any) -> None:
         if action == "export" and event_id in self._exports:
             self._exports[event_id] = msg["data"]
         elif action == "compute":
-            self._send_msg({
-                "action": "compute",
-                "id": event_id,
-                "result": self._compute(msg["payload"]),
-            })
+            self._send_msg(
+                {
+                    "action": "compute",
+                    "id": event_id,
+                    "result": self._compute(msg["payload"]),
+                }
+            )
 
     async def export_chart(
         self,
@@ -423,6 +435,10 @@ async def export_chart(
         -------
         Dictionary containing the exported chart(s).
         """
+        mode = mode or self.export_mode
+        scope = scope or self.export_scope
+        timeout = timeout or self.export_timeout
+
         event_id = uuid.uuid4().hex
         self._send_msg(
             {"action": "export", "id": event_id, "scope": f"{scope}", "mode": mode}

src/panel_gwalker/_pygwalker.py

@@ -1,6 +1,9 @@
-from typing import TYPE_CHECKING, Any, Dict, List
+import json
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol
 
 import pandas as pd
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine
 
 if TYPE_CHECKING:
     try:
@@ -44,20 +47,22 @@ def get_data_parser(
     try:
         from pygwalker import data_parsers
         from pygwalker.data_parsers.base import FieldSpec
+        from pygwalker.services.data_parsers import _get_data_parser
     except ImportError as exc:
         raise ImportError(
             "Server dependencies are not installed. Please: pip install panel-graphic-walker[kernel]."
         ) from exc
 
     _field_specs = [FieldSpec(**_convert_to_field_spec(spec)) for spec in field_specs]
-
-    if isinstance(object, pd.DataFrame):
-        return data_parsers.pandas_parser.PandasDataFrameDataParser(
+    try:
+        parser, name = _get_data_parser(object)
+        return parser(
             object,
             _field_specs,
             infer_string_to_date,
             infer_number_to_dimension,
             other_params,
         )
-    msg = f"Data type {type(object)} is currently not supported"
-    raise NotImplementedError(msg)
+    except TypeError as exc:
+        msg = f"Data type {type(object)} is currently not supported"
+        raise NotImplementedError(msg) from exc