Fix binary serialization from JS -> Pyodide (#6490)

panel/_templates/pyodide_handler.js

@@ -2,8 +2,10 @@ const pyodideWorker = new Worker("./{{ name }}.js");
 pyodideWorker.busy = false
 pyodideWorker.queue = []
 
+let patching = 0
+
 function send_change(jsdoc, event) {
-  if (event.setter_id != null && event.setter_id == 'py') {
+  if ((event.setter_id != null && event.setter_id == 'py') || (patching > 0)) {
     return
   } else if (pyodideWorker.busy && event.model && event.attr) {
     let events = []
@@ -82,6 +84,11 @@ pyodideWorker.onmessage = async (event) => {
     pyodideWorker.postMessage({'type': 'rendered'})
     pyodideWorker.postMessage({'type': 'location', location: JSON.stringify(window.location)})
   } else if (msg.type === 'patch') {
-    pyodideWorker.jsdoc.apply_json_patch(msg.patch, msg.buffers, setter_id='py')
+    try {
+      patching += 1
+      pyodideWorker.jsdoc.apply_json_patch(msg.patch, msg.buffers)
+    } finally {
+      patching -= 1
+    }
   }
 };

panel/_templates/pyodide_worker.js

@@ -74,7 +74,8 @@ self.onmessage = async (event) => {
   } else if (msg.type === 'patch') {
     self.pyodide.globals.set('patch', msg.patch)
     self.pyodide.runPythonAsync(`
-    state.curdoc.apply_json_patch(patch.to_py(), setter='js')
+    from panel.io.pyodide import _convert_json_patch
+    state.curdoc.apply_json_patch(_convert_json_patch(patch), setter='js')
     `)
     self.postMessage({type: 'idle'})
   } else if (msg.type === 'location') {

panel/command/convert.py

@@ -82,7 +82,7 @@ class Convert(Subcommand):
         )),
     )
 
-    _targets = ('pyscript', 'pyodide', 'pyodide-worker')
+    _targets = ('pyscript', 'pyodide', 'pyodide-worker', 'pyscript-worker')
 
     def invoke(self, args: argparse.Namespace) -> None:
         runtime = args.to.lower()

panel/io/convert.py

@@ -2,6 +2,7 @@
 
 import concurrent.futures
 import dataclasses
+import json
 import os
 import pathlib
 import uuid
@@ -41,17 +42,17 @@
 PANEL_ROOT = pathlib.Path(__file__).parent.parent
 BOKEH_VERSION = base_version(bokeh.__version__)
 PY_VERSION = base_version(__version__)
-PYODIDE_VERSION = 'v0.24.1'
-PYSCRIPT_VERSION = '2023.05.1'
+PYODIDE_VERSION = 'v0.25.0'
+PYSCRIPT_VERSION = '2024.2.1'
 PANEL_LOCAL_WHL = DIST_DIR / 'wheels' / f'panel-{__version__.replace("-dirty", "")}-py3-none-any.whl'
 BOKEH_LOCAL_WHL = DIST_DIR / 'wheels' / f'bokeh-{BOKEH_VERSION}-py3-none-any.whl'
 PANEL_CDN_WHL = f'{CDN_DIST}wheels/panel-{PY_VERSION}-py3-none-any.whl'
 BOKEH_CDN_WHL = f'{CDN_ROOT}wheels/bokeh-{BOKEH_VERSION}-py3-none-any.whl'
 PYODIDE_URL = f'https://cdn.jsdelivr.net/pyodide/{PYODIDE_VERSION}/full/pyodide.js'
 PYODIDE_PYC_URL = f'https://cdn.jsdelivr.net/pyodide/{PYODIDE_VERSION}/pyc/pyodide.js'
-PYSCRIPT_CSS = f'<link rel="stylesheet" href="https://pyscript.net/releases/{PYSCRIPT_VERSION}/pyscript.css" />'
+PYSCRIPT_CSS = f'<link rel="stylesheet" href="https://pyscript.net/releases/{PYSCRIPT_VERSION}/core.css" />'
 PYSCRIPT_CSS_OVERRIDES = f'<link rel="stylsheet" href="{CDN_DIST}css/pyscript.css" />'
-PYSCRIPT_JS = f'<script defer src="https://pyscript.net/releases/{PYSCRIPT_VERSION}/pyscript.js"></script>'
+PYSCRIPT_JS = f'<script type="module" src="https://pyscript.net/releases/{PYSCRIPT_VERSION}/core.js"></script>'
 PYODIDE_JS = f'<script src="{PYODIDE_URL}"></script>'
 PYODIDE_PYC_JS = f'<script src="{PYODIDE_PYC_URL}"></script>'
 
@@ -68,7 +69,7 @@
     ICON_DIR / 'index_background.png'
 ]
 
-Runtimes = Literal['pyodide', 'pyscript', 'pyodide-worker']
+Runtimes = Literal['pyodide', 'pyscript', 'pyodide-worker', 'pyscript-worker']
 
 PRE = """
 import asyncio
@@ -276,15 +277,19 @@ def script_to_html(
     web_worker = None
     if css_resources is None:
         css_resources = []
-    if runtime == 'pyscript':
+    if runtime.startswith('pyscript'):
         if js_resources == 'auto':
             js_resources = [PYSCRIPT_JS]
         if css_resources == 'auto':
             css_resources = [PYSCRIPT_CSS, PYSCRIPT_CSS_OVERRIDES]
         elif not css_resources:
             css_resources = []
-        pyenv = ','.join([repr(req) for req in reqs])
-        plot_script = f'<py-config>\npackages = [{pyenv}]\n</py-config>\n<py-script>{code}</py-script>'
+        pyconfig = json.dumps({'packages': reqs, 'plugins': ["!error"]})
+        if 'worker' in runtime:
+            plot_script = f'<script type="py" async worker config=\'{pyconfig}\' src="{app_name}.py"></script>'
+            web_worker = code
+        else:
+            plot_script = f'<py-script config=\'{pyconfig}\'>{code}</py-script>'
     else:
         if css_resources == 'auto':
             css_resources = []
@@ -316,8 +321,8 @@ def script_to_html(
         json_id = make_id()
         docs_json, render_items = standalone_docs_json_and_render_items(document)
         render_item = render_items[0]
-        json = escape(serialize_json(docs_json), quote=False)
-        plot_script += wrap_in_script_tag(json, "application/json", json_id)
+        escaped_json = escape(serialize_json(docs_json), quote=False)
+        plot_script += wrap_in_script_tag(escaped_json, "application/json", json_id)
         plot_script += wrap_in_script_tag(script_for_render_items(json_id, render_items))
     else:
         render_item = RenderItem(
@@ -333,7 +338,7 @@ def script_to_html(
         config.loading_spinner, config.loading_color, config.loading_max_height
     )
     css_resources.append(
-        f'<style type="text/css">\n{spinner_css}\n</style>'
+        f'<style type="text/css">\n{spinner_css}\n.py-error {{ display: none; }}</style>'
     )
     with set_curdoc(document):
         bokeh_js, bokeh_css = bundle_resources(document.roots, resources)
@@ -368,6 +373,13 @@ def script_to_html(
     html = (html
         .replace('<body>', f'<body class="{LOADING_INDICATOR_CSS_CLASS} pn-{config.loading_spinner}">')
     )
+    if runtime == 'pyscript-worker':
+        # pyscript-worker apps must have strict cross-origin policies
+        html = (html
+            .replace('<script type="text/javascript"', '<script type="text/javascript" crossorigin="anonymous"')
+            .replace('<link rel="stylesheet"', '<link rel="stylesheet" crossorigin="anonymous"')
+            .replace('<link rel="icon"', '<link rel="icon" crossorigin="anonymous"')
+        )
     return html, web_worker
 
 
@@ -391,7 +403,7 @@ def convert_app(
 
     try:
         with set_resource_mode('inline' if inline else 'cdn'):
-            html, js_worker = script_to_html(
+            html, worker = script_to_html(
                 app, requirements=requirements, runtime=runtime,
                 prerender=prerender, manifest=manifest,
                 panel_version=panel_version, http_patch=http_patch,
@@ -407,9 +419,12 @@ def convert_app(
 
     with open(dest_path / filename, 'w', encoding="utf-8") as out:
         out.write(html)
-    if runtime == 'pyodide-worker':
+    if runtime == 'pyscript-worker':
+        with open(dest_path / f'{name}.py', 'w', encoding="utf-8") as out:
+            out.write(worker)
+    elif runtime == 'pyodide-worker':
         with open(dest_path / f'{name}.js', 'w', encoding="utf-8") as out:
-            out.write(js_worker)
+            out.write(worker)
     if verbose:
         print(f'Successfully converted {app} to {runtime} target and wrote output to {filename}.')
     return (name.replace('_', ' '), filename)

panel/io/pyodide.py

@@ -8,6 +8,7 @@
 import os
 import pathlib
 import sys
+import uuid
 
 from typing import (
     Any, Callable, List, Tuple,
@@ -20,7 +21,7 @@
 import pyodide # isort: split
 
 from bokeh import __version__
-from bokeh.core.serialization import Buffer, Serializer
+from bokeh.core.serialization import Buffer, Serialized, Serializer
 from bokeh.document import Document
 from bokeh.document.json import PatchJson
 from bokeh.embed.elements import script_for_render_items
@@ -48,16 +49,24 @@
 
 try:
     from js import document as js_document  # noqa
-    _IN_WORKER = False
-    _IN_PYSCRIPT_WORKER = False
+    try:
+        # PyScript Next Worker now patches js.document by default
+        from pyscript import RUNNING_IN_WORKER as _IN_PYSCRIPT_WORKER
+        if _IN_PYSCRIPT_WORKER:
+            from pyscript import window
+            js.window = window
+        _IN_WORKER = True
+    except Exception:
+        _IN_PYSCRIPT_WORKER = False
+        _IN_WORKER = False
 except Exception:
     try:
-        # PyScript Next Worker support
+        # Initial version of PyScript Next Worker support did not patch js.document
         from pyscript import RUNNING_IN_WORKER as _IN_PYSCRIPT_WORKER
         if _IN_PYSCRIPT_WORKER:
             from pyscript import document, window
             js.document = document
-            js.Bokeh = window.Bokeh
+            js.window = window
     except Exception:
         _IN_PYSCRIPT_WORKER = False
     _IN_WORKER = True
@@ -253,6 +262,27 @@ def _process_document_events(doc: Document, events: List[Any]):
 })
 """)
 
+_current_buffers = []
+_patching = False
+
+def _bytes_converter(value, converter, other):
+    if not hasattr(value, 'buffer'):
+        return value
+    value = dict(value.object_entries())
+    uid = uuid.uuid4().hex
+    _current_buffers.append(
+        Buffer(id=uid, data=value['buffer'].to_bytes())
+    )
+    return {'id': uid}
+
+def _convert_json_patch(json_patch):
+    try:
+        patch = json_patch.to_py(default_converter=_bytes_converter)
+        serialized = Serialized(content=patch, buffers=list(_current_buffers))
+    finally:
+        _current_buffers.clear()
+    return serialized
+
 def _link_docs(pydoc: Document, jsdoc: Any) -> None:
     """
     Links Python and JS documents in Pyodide ensuring that messages
@@ -265,23 +295,30 @@ def _link_docs(pydoc: Document, jsdoc: Any) -> None:
     jsdoc: Javascript Document
         The Javascript Bokeh Document instance to sync.
     """
+
     def jssync(event):
         setter_id = getattr(event, 'setter_id', None)
-        if (setter_id is not None and setter_id == 'python'):
+        if (setter_id is not None and setter_id == 'python') or _patching:
             return
         json_patch = jsdoc.create_json_patch(pyodide.ffi.to_js([event]))
-        pydoc.apply_json_patch(json_patch.to_py(), setter='js')
+        patch = _convert_json_patch(json_patch)
+        pydoc.apply_json_patch(patch, setter='js')
 
     jsdoc.on_change(pyodide.ffi.create_proxy(jssync), pyodide.ffi.to_js(False))
 
     def pysync(event):
+        global _patching
         setter = getattr(event, 'setter', None)
         if setter is not None and setter == 'js':
             return
         json_patch, buffer_map = _process_document_events(pydoc, [event])
         json_patch = pyodide.ffi.to_js(json_patch, dict_converter=_dict_converter)
         buffer_map = pyodide.ffi.to_js(buffer_map)
-        jsdoc.apply_json_patch(json_patch, buffer_map)
+        _patching = True
+        try:
+            jsdoc.apply_json_patch(json_patch, buffer_map)
+        finally:
+            _patching = False
 
     pydoc.on_change(pysync)
 
@@ -331,12 +368,12 @@ async def _link_model(ref: str, doc: Document) -> None:
     doc: bokeh.document.Document
         The bokeh Document to sync the rendered Model with.
     """
-    rendered = js.Bokeh.index.object_keys()
+    rendered = js.window.Bokeh.index.object_keys()
     if ref not in rendered:
         await asyncio.sleep(0.1)
         await _link_model(ref, doc)
         return
-    views = js.Bokeh.index.object_values()
+    views = js.window.Bokeh.index.object_values()
     view = views[rendered.indexOf(ref)]
     _link_docs(doc, view.model.document)
 
@@ -466,7 +503,7 @@ async def write(target: str, obj: Any) -> None:
 
     obj = as_panel(obj)
     pydoc, model_json = _model_json(obj, target)
-    views = await js.Bokeh.embed.embed_item(JSON.parse(model_json))
+    views = await js.window.Bokeh.embed.embed_item(JSON.parse(model_json))
     jsdoc = list(views.roots)[0].model.document
     _link_docs(pydoc, jsdoc)
     pydoc.unhold()
@@ -486,8 +523,10 @@ def sync_location():
     """
     if not state.location:
         return
-    from js import window
-    loc_string = JSON.stringify(window.location)
+    try:
+        loc_string = JSON.stringify(js.window.location)
+    except Exception:
+        return
     loc_data = json.loads(loc_string)
     with edit_readonly(state.location):
         state.location.param.update({
@@ -531,7 +570,7 @@ async def write_doc(doc: Document | None = None) -> Tuple[str, str, str]:
 
     # If we have DOM access render and sync the document
     if root_els is not None:
-        views = await js.Bokeh.embed.embed_items(JSON.parse(docs_json), JSON.parse(render_items))
+        views = await js.window.Bokeh.embed.embed_items(JSON.parse(docs_json), JSON.parse(render_items))
         jsdoc = list(views[0].roots)[0].model.document
         _link_docs(pydoc, jsdoc)
         sync_location()