Improve serialization of Pandas DataFrames to ipyvega

.github/workflows/test.yml

@@ -14,7 +14,7 @@ jobs:
 
     strategy:
       matrix:
-        python: ["3.6", "3.7", "3.8", "3.9"]
+        python: ["3.7", "3.8", "3.9", "3.10"]
 
     name: Python ${{ matrix.python }}
 
@@ -29,7 +29,7 @@ jobs:
       - name: Setup poetry
         uses: abatilo/actions-poetry@v2.2.0
         with:
-          poetry-version: 1.1.12
+          poetry-version: 1.3.1
 
       - name: Configure poetry
         run: poetry config virtualenvs.in-project true

.github/workflows/ui.yml

@@ -0,0 +1,68 @@
+name: UI Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python: ["3.7", "3.8", "3.9", "3.10"]
+
+    name: Python ${{ matrix.python }}
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v2.3.1
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Setup poetry
+        uses: abatilo/actions-poetry@v2.1.4
+        with:
+          poetry-version: 1.3.1
+
+      - name: Configure poetry
+        run: poetry config virtualenvs.create false
+
+      - name: Install Python dependencies
+        run: |
+          poetry install
+          python -m pip install jupyterlab-widgets==1.0.2
+          python -m pip install jupyterlab==3.0.11
+
+      - name: Setup Node
+        uses: actions/setup-node@v2.5.1
+
+      - name: Install Node dependencies
+        run: yarn --frozen-lockfile
+
+      - name: Build
+        run: yarn run build
+
+      - name: Run tests
+        run: |
+          jupyter labextension install .
+          cd ui-tests/
+          yarn
+          yarn run start-jlab:detached
+          sleep 10
+          yarn run test
+      - name: Upload output file
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: cell-artifact
+          path: ui-tests/test-output/test/screenshots/vega_cell_*.png

README.md

@@ -19,7 +19,8 @@ To install `vega` and its dependencies from the Python Package Index using
 ```sh
 pip install jupyter pandas vega
 pip install --upgrade notebook  # need jupyter_client >= 4.2 for sys-prefix below
-jupyter nbextension install --sys-prefix --py vega  # not needed in notebook >= 5.3
+jupyter nbextension install --sys-prefix --py vega
+jupyter nbextension enable --py --sys-prefix vega
 ```
 
 ### Conda Forge
@@ -51,7 +52,8 @@ Then activate the virtual environment with `poetry shell`.
 Symlink files instead of copying files:
 
 ```sh
-jupyter nbextension install --py --symlink vega
+jupyter nbextension install --py --symlink --sys-prefix vega
+jupyter nbextension enable --py --sys-prefix vega
 ```
 
 Run kernel with `jupyter notebook`. Run the tests with `pytest vega`.

notebooks/AllCompressions.ipynb

@@ -0,0 +1,214 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "247bdd49",
+   "metadata": {},
+   "source": [
+    "# Compression methods\n",
+    "Two compression methods are provided to send data: zlib and lz4. The latter is enabled by default since it is both fast and efficient for numerical columns. You may experience better results for particular column types so feel free to tune the compression method for your particular data.\n",
+    "\n",
+    "With a `VegaWidget`, the field `.compression` specifies the method used to send data.\n",
+    "The following code shows how to do it and some time results. Keep in mind that the time returned by Python are not exactly related to the performance of the compression because the data transmission is asynchronous. The numbers are still informative.\n",
+    "The content of the `.compression` field can be either a string with the name of the compression or an instance of the compressor, to specify the compression level."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "688a1886",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d23699478a5e48f98027c11ce7792fb7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VegaWidget()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 6 µs, sys: 7 µs, total: 13 µs\n",
+      "Wall time: 15.3 µs\n"
+     ]
+    }
+   ],
+   "source": [
+    "spec_no_data = {\n",
+    "  \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.json\",\n",
+    "  \"data\": {\"name\": \"data\"},\n",
+    "  \"mark\": \"bar\",\n",
+    "  \"encoding\": {\n",
+    "    \"x\": {\"aggregate\": \"sum\", \"field\": \"yield\"},\n",
+    "    \"y\": {\"field\": \"variety\"},\n",
+    "    \"color\": {\"field\": \"site\"}\n",
+    "  }\n",
+    "}\n",
+    "from vega.widget import VegaWidget\n",
+    "import requests\n",
+    "import json\n",
+    "req = requests.get(\"https://cdn.jsdelivr.net/npm/vega-datasets@2.2.0/data/barley.json\")\n",
+    "values = json.loads(req.text)\n",
+    "#data\n",
+    "widget = VegaWidget(spec=spec_no_data)\n",
+    "display(widget)\n",
+    "%time widget.update('data', insert=values)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "7d0f4bf0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5864bd22268b4b03bfecb71af6206a23",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VegaWidget()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 7.41 ms, sys: 0 ns, total: 7.41 ms\n",
+      "Wall time: 7.1 ms\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "URL = \"https://forge.scilab.org/index.php/p/rdataset/source/file/368b19abcb4292c56e4f21079f750eb76b325907/csv/lattice/barley.csv\"\n",
+    "df = pd.read_csv(URL)\n",
+    "widget = VegaWidget(spec=spec_no_data)\n",
+    "display(widget)\n",
+    "%time widget.update(\"data\", insert=df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a13d013c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "112e7a4059d64fa08cab2f0583de3074",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VegaWidget()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 9.44 ms, sys: 0 ns, total: 9.44 ms\n",
+      "Wall time: 8.62 ms\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "URL = \"https://forge.scilab.org/index.php/p/rdataset/source/file/368b19abcb4292c56e4f21079f750eb76b325907/csv/lattice/barley.csv\"\n",
+    "df = pd.read_csv(URL)\n",
+    "widget = VegaWidget(spec=spec_no_data)\n",
+    "widget.compression = 'zlib'\n",
+    "display(widget)\n",
+    "%time widget.update('data', insert=df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "6792044d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ab67b74c7db74b1896e859031a0deb72",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VegaWidget()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 9.84 ms, sys: 1.57 ms, total: 11.4 ms\n",
+      "Wall time: 10.7 ms\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from ipytablewidgets import LZ4Compressor\n",
+    "URL = \"https://forge.scilab.org/index.php/p/rdataset/source/file/368b19abcb4292c56e4f21079f750eb76b325907/csv/lattice/barley.csv\"\n",
+    "df = pd.read_csv(URL)\n",
+    "widget = VegaWidget(spec=spec_no_data)\n",
+    "widget.compression = LZ4Compressor(2)\n",
+    "display(widget)\n",
+    "%time widget.update('data', insert=df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11890337",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}