Fetch Pyodide bundle from GCS

src/workerd/server/server.c++

@@ -33,6 +33,7 @@
 #include <workerd/util/use-perfetto-categories.h>
 #include <workerd/api/worker-rpc.h>
 #include "workerd-api.h"
+#include "workerd/api/pyodide/pyodide.h"
 #include "workerd/io/hibernation-manager.h"
 #include <stdlib.h>
 
@@ -2565,6 +2566,36 @@ void Server::abortAllActors() {
   }
 }
 
+void Server::fetchPyodideBundle() {
+  {
+    kj::Thread([] () {
+      kj::AsyncIoContext io = kj::setupAsyncIo();
+      kj::HttpHeaderTable table;
+
+      kj::TlsContext::Options options;
+      options.useSystemTrustStore = true;
+
+      kj::Own<kj::TlsContext> tls = kj::heap<kj::TlsContext>(kj::mv(options));
+      auto &network = io.provider->getNetwork();
+      auto tlsNetwork = tls->wrapNetwork(network);
+      auto &timer = io.provider->getTimer();
+
+      auto client = kj::newHttpClient(timer, table, network, *tlsNetwork);
+
+      kj::HttpHeaders headers(table);
+
+      auto req = client->request(kj::HttpMethod::GET, "https://storage.googleapis.com/cloudflare-edgeworker-python-packages/python-runtime-capnp-bin/pyodide-1.capnp.bin", headers);
+
+      auto res = req.response.wait(io.waitScope);
+      auto body = res.body->readAllBytes().wait(io.waitScope);
+
+      api::pyodide::setPyodideBundleData(kj::mv(body));
+
+    });
+  }
+
+}
+
 kj::Own<Server::Service> Server::makeWorker(kj::StringPtr name, config::Worker::Reader conf,
     capnp::List<config::Extension>::Reader extensions) {
   TRACE_EVENT("workerd", "Server::makeWorker()", "name", name.cStr());
@@ -2688,6 +2719,10 @@ kj::Own<Server::Service> Server::makeWorker(kj::StringPtr name, config::Worker::
         *observer, conf, featureFlags.asReader(), pythonConfig);
   }
 
+  // Get the pyodide bundle and register it in memory.
+  // After this, the pyodide bundle is accessible at pyodideBundleGlobal
+  fetchPyodideBundle();
+
   auto api = kj::heap<WorkerdApi>(globalContext->v8System,
                                   featureFlags.asReader(),
                                   *limitEnforcer,

src/workerd/server/server.h

@@ -227,6 +227,8 @@ class Server final: private kj::TaskSet::ErrorHandler {
                                     kj::HttpHeaderTable::Builder& headerTableBuilder,
                                     kj::ForkedPromise<void>& forkedDrainWhen,
                                     bool forTest = false);
+
+  void fetchPyodideBundle();
 };
 
 // An ActorStorage implementation which will always respond to reads as if the state is empty,

src/workerd/server/workerd-api.c++

@@ -43,6 +43,7 @@
 #include <openssl/sha.h>
 #include <openssl/hmac.h>
 #include <openssl/rand.h>
+#include <kj/compat/http.h>
 #ifdef WORKERD_EXPERIMENTAL_ENABLE_WEBGPU
 #include <workerd/api/gpu/gpu.h>
 #else
@@ -447,7 +448,7 @@ void WorkerdApi::compileModules(
       KJ_REQUIRE(featureFlags.getPythonWorkers(),
           "The python_workers compatibility flag is required to use Python.");
       // Inject Pyodide bundle
-      modules->addBuiltinBundle(PYODIDE_BUNDLE, kj::none);
+      modules->addBuiltinBundle(KJ_ASSERT_NONNULL(pyodideBundleGlobal), kj::none);
       // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?)
       {
         auto mainModule = confModules.begin();