Load Pyodide runtime from external capnproto file

These changes are needed to load the Pyodide runtime from an external capnproto
file. In workerd, we'll keep loading Pyodide in the same way, but we moved this
to WorkerdAPI. In our internal codebase, we now download the bundle and call
`setPyodideBundleData` when we load the first Python worker. Then when setting
up the module registry, we add the bundle from `pyodideBundleGlobal`.

TODO in followups:
1. Make workerd load the bundle in the same way
2. Add support for multiple versions of the bundle

src/workerd/api/modules.h

@@ -20,7 +20,9 @@ namespace workerd::api {
 template <class Registry>
 void registerModules(Registry& registry, auto featureFlags) {
   node::registerNodeJsCompatModules(registry, featureFlags);
-  pyodide::registerPyodideModules(registry, featureFlags);
+  if (featureFlags.getPythonWorkers()) {
+    pyodide::registerPyodideModules(registry, featureFlags);
+  }
   registerUnsafeModules(registry, featureFlags);
   if (featureFlags.getRttiApi()) {
     registerRTTIModule(registry);
@@ -45,11 +47,6 @@ void registerBuiltinModules(jsg::modules::ModuleRegistry::Builder& builder, auto
     builder.add(getExternalUnsafeModuleBundle<TypeWrapper>(featureFlags));
   }
 
-  if (featureFlags.getPythonWorkers()) {
-    builder.add(pyodide::getExternalPyodideModuleBundle(featureFlags));
-    builder.add(pyodide::getInternalPyodideModuleBundle(featureFlags));
-  }
-
   if (featureFlags.getRttiApi()) {
     builder.add(getExternalRttiModuleBundle<TypeWrapper>(featureFlags));
   }

src/workerd/api/pyodide/pyodide.c++

@@ -7,6 +7,17 @@
 
 namespace workerd::api::pyodide {
 
+kj::Maybe<kj::Array<unsigned char>> pyodideBundleDataGlobal = kj::none;
+kj::Maybe<kj::Own<capnp::FlatArrayMessageReader>> pyodideBundleReaderGlobal = kj::none;
+kj::Maybe<jsg::Bundle::Reader> pyodideBundleGlobal = kj::none;
+
+void setPyodideBundleData(kj::Array<unsigned char> data) {
+  pyodideBundleReaderGlobal = kj::heap<capnp::FlatArrayMessageReader>(kj::arrayPtr(
+        reinterpret_cast<const capnp::word*>(data.begin()), data.size() / sizeof(capnp::word)));
+  pyodideBundleGlobal = KJ_REQUIRE_NONNULL(pyodideBundleReaderGlobal)->getRoot<jsg::Bundle>();
+  pyodideBundleDataGlobal = kj::mv(data);
+}
+
 static int readToTarget(kj::ArrayPtr<const kj::byte> source, int offset, kj::ArrayPtr<kj::byte> buf) {
   int size = source.size();
   if (offset >= size || offset < 0) {

src/workerd/api/pyodide/pyodide.h

@@ -11,9 +11,16 @@
 #include <workerd/jsg/url.h>
 #include <workerd/server/workerd.capnp.h>
 #include <workerd/io/io-context.h>
+#include "capnp/serialize.h"
 
 namespace workerd::api::pyodide {
 
+// singleton that owns bundle
+extern kj::Maybe<jsg::Bundle::Reader> pyodideBundleGlobal;
+
+void setPyodideBundleData(kj::Array<unsigned char> data);
+
+
 struct PythonConfig {
   kj::Maybe<kj::Own<const kj::Directory>> diskCacheRoot;
   bool createSnapshot;
@@ -344,26 +351,7 @@ bool hasPythonModules(capnp::List<server::config::Worker::Module>::Reader module
   api::pyodide::SimplePythonLimiter
 
 template <class Registry> void registerPyodideModules(Registry& registry, auto featureFlags) {
-  if (featureFlags.getPythonWorkers()) {
-    // We add `pyodide:` packages here including python-entrypoint-helper.js.
-    registry.addBuiltinBundle(PYODIDE_BUNDLE, kj::none);
-    registry.template addBuiltinModule<PackagesTarReader>(
-        "pyodide-internal:packages_tar_reader", workerd::jsg::ModuleRegistry::Type::INTERNAL);
-  }
+  registry.template addBuiltinModule<PackagesTarReader>(
+      "pyodide-internal:packages_tar_reader", workerd::jsg::ModuleRegistry::Type::INTERNAL);
 }
-
-kj::Own<jsg::modules::ModuleBundle> getInternalPyodideModuleBundle(auto featureFlags) {
-  jsg::modules::ModuleBundle::BuiltinBuilder builder(
-      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY);
-  jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builder, PYODIDE_BUNDLE);
-  return builder.finish();
-}
-
-kj::Own<jsg::modules::ModuleBundle> getExternalPyodideModuleBundle(auto featureFlags) {
-  jsg::modules::ModuleBundle::BuiltinBuilder builder(
-      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN);
-  jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builder, PYODIDE_BUNDLE);
-  return builder.finish();
-}
-
 } // namespace workerd::api::pyodide

src/workerd/server/workerd-api.c++

@@ -446,7 +446,9 @@ void WorkerdApi::compileModules(
     if (hasPythonModules(confModules)) {
       KJ_REQUIRE(featureFlags.getPythonWorkers(),
           "The python_workers compatibility flag is required to use Python.");
-      // Inject pyodide bootstrap module.
+      // Inject Pyodide bundle
+      modules->addBuiltinBundle(PYODIDE_BUNDLE, kj::none);
+      // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?)
       {
         auto mainModule = confModules.begin();
         capnp::MallocMessageBuilder message;