Load Pyodide runtime through HTTP

samples/pyodide-fastapi/config.capnp

@@ -13,6 +13,10 @@ const config :Workerd.Config = (
       service = "main"
     ),
   ],
+
+  autogates = [
+    "workerd-autogate-pyodide-load-external",
+  ]
 );
 
 const mainWorker :Workerd.Worker = (

samples/pyodide-langchain/config.capnp

@@ -13,6 +13,10 @@ const config :Workerd.Config = (
       service = "main"
     ),
   ],
+
+  autogates = [
+    "workerd-autogate-pyodide-load-external",
+  ]
 );
 
 const mainWorker :Workerd.Worker = (

samples/pyodide-secret/config.capnp

@@ -17,6 +17,7 @@ const config :Workerd.Config = (
     # Pyodide is included as a builtin wasm module so it requires the
     # corresponding autogate flag.
     "workerd-autogate-builtin-wasm-modules",
+    "workerd-autogate-pyodide-load-external",
   ]
 );
 

samples/pyodide/config.capnp

@@ -13,6 +13,10 @@ const config :Workerd.Config = (
       service = "main"
     ),
   ],
+
+  autogates = [
+    "workerd-autogate-pyodide-load-external",
+  ]
 );
 
 const mainWorker :Workerd.Worker = (

samples/repl-server-python/config.capnp

@@ -13,6 +13,10 @@ const config :Workerd.Config = (
       service = "main"
     ),
   ],
+
+  autogates = [
+    "workerd-autogate-pyodide-load-external",
+  ]
 );
 
 const mainWorker :Workerd.Worker = (

src/workerd/api/pyodide/pyodide.h

@@ -23,7 +23,8 @@ jsg::Bundle::Reader getPyodideBundle(kj::StringPtr version);
 
 
 struct PythonConfig {
-  kj::Maybe<kj::Own<const kj::Directory>> diskCacheRoot;
+  kj::Maybe<kj::Own<const kj::Directory>> packageDiskCacheRoot;
+  kj::Maybe<kj::Own<const kj::Directory>> pyodideDiskCacheRoot;
   bool createSnapshot;
   bool createBaselineSnapshot;
 };

src/workerd/io/compatibility-date.c++

@@ -296,4 +296,64 @@ kj::Maybe<kj::String> normalizeCompatDate(kj::StringPtr date) {
   return CompatDate::parse(date).map([](auto v) { return v.toString(); });
 }
 
+struct PythonSnapshotParsedField {
+  PythonSnapshotRelease::Reader pythonSnapshotRelease;
+  capnp::StructSchema::Field field;
+};
+
+kj::Array<const PythonSnapshotParsedField> makePythonSnapshotFieldTable(
+    capnp::StructSchema::FieldList fields) {
+  kj::Vector<PythonSnapshotParsedField> table(fields.size());
+
+  for (auto field: fields) {
+    kj::Maybe<PythonSnapshotRelease::Reader> maybePythonSnapshotRelease;
+
+    for (auto annotation: field.getProto().getAnnotations()) {
+      if (annotation.getId() == PYTHON_SNAPSHOT_RELEASE_ANNOTATION_ID) {
+        maybePythonSnapshotRelease =
+            annotation.getValue().getStruct().getAs<workerd::PythonSnapshotRelease>();
+      }
+    }
+
+    KJ_IF_SOME(pythonSnapshotRelease, maybePythonSnapshotRelease) {
+      table.add(PythonSnapshotParsedField{
+        .pythonSnapshotRelease = pythonSnapshotRelease,
+        .field = field,
+      });
+    }
+  }
+
+  return table.releaseAsArray();
+}
+
+kj::Maybe<PythonSnapshotRelease::Reader> getPythonSnapshotRelease(
+    CompatibilityFlags::Reader featureFlags) {
+  uint latestFieldOrdinal = 0;
+  kj::Maybe<PythonSnapshotRelease::Reader> result;
+
+  static auto fieldTable =
+      makePythonSnapshotFieldTable(capnp::Schema::from<CompatibilityFlags>().getFields());
+
+  for (auto field: fieldTable) {
+    bool isEnabled = capnp::toDynamic(featureFlags).get(field.field).as<bool>();
+    if (!isEnabled) {
+      continue;
+    }
+
+    // We pick the flag with the highest ordinal value that is enabled and has a
+    // pythonSnapshotRelease annotation.
+    //
+    // The fieldTable is probably ordered by the ordinal anyway, but doesn't hurt to be explicit
+    // here.
+    //
+    // TODO(later): make sure this is well tested once we have more than one compat flag.
+    if (latestFieldOrdinal < field.field.getIndex()) {
+      latestFieldOrdinal = field.field.getIndex();
+      result = field.pythonSnapshotRelease;
+    }
+  }
+
+  return result;
+}
+
 }  // namespace workerd

src/workerd/io/compatibility-date.h

@@ -50,6 +50,9 @@ kj::Maybe<kj::String> normalizeCompatDate(kj::StringPtr date);
 // Returns the current date as a string formatted by CompatDate.
 kj::String currentDateStr();
 
+kj::Maybe<PythonSnapshotRelease::Reader> getPythonSnapshotRelease(
+    CompatibilityFlags::Reader featureFlags);
+
 // These values come from src/workerd/io/compatibility-date.capnp
 static constexpr uint64_t COMPAT_ENABLE_FLAG_ANNOTATION_ID = 0xb6dabbc87cd1b03eull;
 static constexpr uint64_t COMPAT_DISABLE_FLAG_ANNOTATION_ID = 0xd145cf1adc42577cull;

src/workerd/server/server.c++

@@ -33,6 +33,7 @@
 #include <workerd/util/use-perfetto-categories.h>
 #include <workerd/api/worker-rpc.h>
 #include "workerd-api.h"
+#include "workerd/api/pyodide/pyodide.h"
 #include "workerd/io/hibernation-manager.h"
 #include <stdlib.h>
 

src/workerd/server/server.h

@@ -61,8 +61,11 @@ class Server final: private kj::TaskSet::ErrorHandler {
   void enableControl(uint fd) {
     controlOverride = kj::heap<kj::FdOutputStream>(fd);
   }
-  void setPythonDiskCacheRoot(kj::Maybe<kj::Own<const kj::Directory>> &&dkr) {
-    pythonConfig.diskCacheRoot = kj::mv(dkr);
+  void setPackageDiskCacheRoot(kj::Maybe<kj::Own<const kj::Directory>> &&dkr) {
+    pythonConfig.packageDiskCacheRoot = kj::mv(dkr);
+  }
+  void setPyodideDiskCacheRoot(kj::Maybe<kj::Own<const kj::Directory>> &&dkr) {
+    pythonConfig.pyodideDiskCacheRoot = kj::mv(dkr);
   }
   void setPythonCreateSnapshot() {
     pythonConfig.createSnapshot = true;
@@ -103,7 +106,8 @@ class Server final: private kj::TaskSet::ErrorHandler {
   kj::EntropySource& entropySource;
   kj::Function<void(kj::String)> reportConfigError;
   PythonConfig pythonConfig = PythonConfig {
-    .diskCacheRoot = kj::none,
+    .packageDiskCacheRoot = kj::none,
+    .pyodideDiskCacheRoot = kj::none,
     .createSnapshot = false,
     .createBaselineSnapshot = false
   };

src/workerd/server/tests/python/BUILD.bazel

@@ -1,6 +1,20 @@
 load("//:build/wd_test.bzl", "wd_test")
 
-wd_test(
+load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test")
+
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+
+# pyodide-dev.capnp.bin represents a custom pyodide version "dev" that is generated
+# at build time using the latest contents of the src/pyodide directory.
+# This is used to run tests to ensure that they are always run against the latest build of
+# the Pyodide bundle.
+copy_file(
+    name = "pyodide-dev.capnp.bin@rule",
+    src = "//src/pyodide:pyodide.capnp.bin",
+    out = "pyodide-bundle-cache/pyodide-dev.capnp.bin"
+)
+
+py_wd_test(
     src = "hello/hello.wd-test",
     args = ["--experimental"],
     data = glob(
@@ -11,7 +25,7 @@ wd_test(
     ),
 )
 
-wd_test(
+py_wd_test(
     src = "env-param/env.wd-test",
     args = ["--experimental"],
     data = glob(
@@ -22,7 +36,7 @@ wd_test(
     ),
 )
 
-wd_test(
+py_wd_test(
     src = "random/random.wd-test",
     args = ["--experimental"],
     data = glob(
@@ -33,19 +47,7 @@ wd_test(
     ),
 )
 
-# Disabled because it tests the same thing as the import test defined in import_tests.bzl
-# wd_test(
-#     src = "langchain/langchain.wd-test",
-#     args = ["--experimental", "--disk-cache-dir", "../all_pyodide_wheels"],
-#     data = glob(
-#         [
-#             "langchain/*",
-#         ],
-#         exclude = ["**/*.wd-test"],
-#     ) + ["@all_pyodide_wheels//:whls"],
-# )
-
-wd_test(
+py_wd_test(
     src = "subdirectory/subdirectory.wd-test",
     args = ["--experimental"],
     data = glob(

src/workerd/server/tests/python/env-param/env.wd-test

@@ -18,4 +18,8 @@ const unitTests :Workerd.Config = (
       )
     ),
   ],
+
+  autogates = [
+    "workerd-autogate-pyodide-load-external",
+  ]
 );

src/workerd/server/tests/python/hello/hello.wd-test

@@ -12,4 +12,8 @@ const unitTests :Workerd.Config = (
       )
     ),
   ],
+
+  autogates = [
+    "workerd-autogate-pyodide-load-external",
+  ]
 );

src/workerd/server/tests/python/import_tests.bzl

@@ -1,5 +1,6 @@
 load("@bazel_skylib//rules:write_file.bzl", "write_file")
-load("//:build/wd_test.bzl", "wd_test")
+
+load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test")
 
 def generate_import_py_file(imports):
   res = ""
@@ -26,6 +27,10 @@ const unitTests :Workerd.Config = (
       )
     ),
   ],
+
+  autogates = [
+    "workerd-autogate-pyodide-load-external",
+  ]
 );"""
 
 def generate_wd_test_file(requirement):
@@ -49,9 +54,9 @@ def gen_import_tests(to_test):
       tags = ["slow"],
     )
 
-    wd_test(
+    py_wd_test(
       src = wd_test_fname,
-      args = ["--experimental", "--disk-cache-dir", "../all_pyodide_wheels"],
+      args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels"],
       data = [worker_py_fname, "@all_pyodide_wheels//:whls"],
       tags = ["slow"],
     )

src/workerd/server/tests/python/py_wd_test.bzl

@@ -0,0 +1,19 @@
+load("//:build/wd_test.bzl", "wd_test")
+
+def py_wd_test(
+  src,
+  data = [],
+  name = None,
+  args = [],
+  **kwargs
+):
+    data += ["pyodide-dev.capnp.bin@rule"]
+    args += ["--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."]
+
+    wd_test(
+        src = src,
+        data = data,
+        name = name,
+        args = args,
+        **kwargs
+    )

src/workerd/server/tests/python/random/random.wd-test

@@ -12,4 +12,8 @@ const unitTests :Workerd.Config = (
       )
     ),
   ],
+
+  autogates = [
+    "workerd-autogate-pyodide-load-external",
+  ]
 );

src/workerd/server/tests/python/subdirectory/subdirectory.wd-test

@@ -14,5 +14,9 @@ const unitTests :Workerd.Config = (
       )
     ),
   ],
+
+  autogates = [
+    "workerd-autogate-pyodide-load-external",
+  ]
 );