From 61fbfeb0097df48c613959f468cc49d3e0d2a8ef Mon Sep 17 00:00:00 2001 From: Garrett Gu Date: Wed, 24 Jul 2024 16:20:50 -0500 Subject: [PATCH 1/7] Fetch Pyodide bundle from HTTP --- src/workerd/server/server.c++ | 35 ++++++++++++++++++++++++++++++ src/workerd/server/server.h | 2 ++ src/workerd/server/workerd-api.c++ | 5 ++++- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index ec32983eedb..5a9bda55cc0 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -33,6 +33,7 @@ #include #include #include "workerd-api.h" +#include "workerd/api/pyodide/pyodide.h" #include "workerd/io/hibernation-manager.h" #include @@ -2565,6 +2566,36 @@ void Server::abortAllActors() { } } +void Server::fetchPyodideBundle() { + { + kj::Thread([] () { + kj::AsyncIoContext io = kj::setupAsyncIo(); + kj::HttpHeaderTable table; + + kj::TlsContext::Options options; + options.useSystemTrustStore = true; + + kj::Own tls = kj::heap(kj::mv(options)); + auto &network = io.provider->getNetwork(); + auto tlsNetwork = tls->wrapNetwork(network); + auto &timer = io.provider->getTimer(); + + auto client = kj::newHttpClient(timer, table, network, *tlsNetwork); + + kj::HttpHeaders headers(table); + + auto req = client->request(kj::HttpMethod::GET, "https://pyodide.runtime-playground.workers.dev/python-runtime-capnp-bin/pyodide-1.capnp.bin", headers); + + auto res = req.response.wait(io.waitScope); + auto body = res.body->readAllBytes().wait(io.waitScope); + + api::pyodide::setPyodideBundleData(kj::mv(body)); + + }); + } + +} + kj::Own Server::makeWorker(kj::StringPtr name, config::Worker::Reader conf, capnp::List::Reader extensions) { TRACE_EVENT("workerd", "Server::makeWorker()", "name", name.cStr()); @@ -2688,6 +2719,10 @@ kj::Own Server::makeWorker(kj::StringPtr name, config::Worker:: *observer, conf, featureFlags.asReader(), pythonConfig); } + // Get the pyodide bundle and register it in memory. + // After this, the pyodide bundle is accessible at pyodideBundleGlobal + fetchPyodideBundle(); + auto api = kj::heap(globalContext->v8System, featureFlags.asReader(), *limitEnforcer, diff --git a/src/workerd/server/server.h b/src/workerd/server/server.h index 585bedefe9e..ecdf81921ce 100644 --- a/src/workerd/server/server.h +++ b/src/workerd/server/server.h @@ -227,6 +227,8 @@ class Server final: private kj::TaskSet::ErrorHandler { kj::HttpHeaderTable::Builder& headerTableBuilder, kj::ForkedPromise& forkedDrainWhen, bool forTest = false); + + void fetchPyodideBundle(); }; // An ActorStorage implementation which will always respond to reads as if the state is empty, diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index e7e624b0ea7..4bc102be18b 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -43,6 +43,7 @@ #include #include #include +#include #ifdef WORKERD_EXPERIMENTAL_ENABLE_WEBGPU #include #else @@ -446,7 +447,9 @@ void WorkerdApi::compileModules( if (hasPythonModules(confModules)) { KJ_REQUIRE(featureFlags.getPythonWorkers(), "The python_workers compatibility flag is required to use Python."); - // Inject pyodide bootstrap module. + // Inject Pyodide bundle + modules->addBuiltinBundle(KJ_ASSERT_NONNULL(pyodideBundleGlobal), kj::none); + // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?) { auto mainModule = confModules.begin(); capnp::MallocMessageBuilder message; From e556b730d3539ff709421f76b38ec9027745f62d Mon Sep 17 00:00:00 2001 From: Garrett Gu Date: Tue, 30 Jul 2024 13:50:10 -0500 Subject: [PATCH 2/7] Add build in bundle behind autogate Co-authored-by: Hood Chatham --- src/pyodide/BUILD.bazel | 2 ++ src/workerd/server/workerd-api.c++ | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pyodide/BUILD.bazel b/src/pyodide/BUILD.bazel index d68bec68543..b9a4560eeee 100644 --- a/src/pyodide/BUILD.bazel +++ b/src/pyodide/BUILD.bazel @@ -1,3 +1,5 @@ +package(default_visibility = ["//visibility:public"]) + load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@bazel_skylib//rules:expand_template.bzl", "expand_template") diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index 4bc102be18b..914283eb167 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -448,7 +448,9 @@ void WorkerdApi::compileModules( KJ_REQUIRE(featureFlags.getPythonWorkers(), "The python_workers compatibility flag is required to use Python."); // Inject Pyodide bundle - modules->addBuiltinBundle(KJ_ASSERT_NONNULL(pyodideBundleGlobal), kj::none); + if(util::Autogate::isEnabled(util::AutogateKey::PYODIDE_LOAD_EXTERNAL)) { + modules->addBuiltinBundle(KJ_ASSERT_NONNULL(pyodideBundleGlobal), kj::none); + } // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?) { auto mainModule = confModules.begin(); From 0e324a8525cd15fe411390482d150048d254e6af Mon Sep 17 00:00:00 2001 From: Garrett Gu Date: Mon, 5 Aug 2024 16:15:29 -0500 Subject: [PATCH 3/7] Implement split disk cache, loading and storing pyodide bundle in disk cache, using generated tip-of-tree pyodide to pre-populate disk cache --- src/workerd/api/pyodide/pyodide.h | 3 +- src/workerd/server/server.c++ | 59 +++++++++++++++++-- src/workerd/server/server.h | 12 ++-- src/workerd/server/tests/python/BUILD.bazel | 26 +++++--- .../server/tests/python/import_tests.bzl | 5 +- src/workerd/server/workerd-api.c++ | 11 ++-- src/workerd/server/workerd.c++ | 14 ++++- src/workerd/tests/test-fixture.c++ | 2 +- 8 files changed, 101 insertions(+), 31 deletions(-) diff --git a/src/workerd/api/pyodide/pyodide.h b/src/workerd/api/pyodide/pyodide.h index 00d2de0459d..10c8635494b 100644 --- a/src/workerd/api/pyodide/pyodide.h +++ b/src/workerd/api/pyodide/pyodide.h @@ -23,7 +23,8 @@ jsg::Bundle::Reader getPyodideBundle(kj::StringPtr version); struct PythonConfig { - kj::Maybe> diskCacheRoot; + kj::Maybe> packageDiskCacheRoot; + kj::Maybe> pyodideDiskCacheRoot; bool createSnapshot; bool createBaselineSnapshot; }; diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 5a9bda55cc0..3b86e750607 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -2566,9 +2566,50 @@ void Server::abortAllActors() { } } -void Server::fetchPyodideBundle() { +kj::Path getPyodideBundlePath(kj::String &version) { + return kj::Path(kj::str("pyodide-", version, ".capnp.bin")); +} + +kj::Maybe> getPyodideBundleFile(kj::Maybe> &maybeDir, kj::String &version) { + KJ_IF_SOME(dir, maybeDir) { + kj::Path filename = getPyodideBundlePath(version); + auto file = dir->tryOpenFile(filename); + + return file; + } + + return kj::none; +} + +void writePyodideBundleFileToDisk(kj::Maybe> &maybeDir, kj::String &version, kj::ArrayPtr bytes) { + KJ_IF_SOME(dir, maybeDir) { + kj::Path filename = getPyodideBundlePath(version); + auto replacer = dir->replaceFile(filename, kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + + replacer->get().writeAll(bytes); + replacer->commit(); + } +} + +void Server::fetchPyodideBundle(api::pyodide::PythonConfig& pyConfig) { + auto version = kj::str("2"); //TODO: hardcoded version number, should probably get this from pyConfig eventually + + if(api::pyodide::hasPyodideBundle(kj::str(version))) { + KJ_LOG(WARNING, "Pyodide version ", kj::str(version), " already exists in pyodide bundle table"); + return; + } + + auto maybePyodideBundleFile = getPyodideBundleFile(pyConfig.pyodideDiskCacheRoot, version); + KJ_IF_SOME(pyodideBundleFile, maybePyodideBundleFile) { + auto body = pyodideBundleFile->readAllBytes(); + api::pyodide::setPyodideBundleData(kj::str(version), kj::mv(body)); + + return; + } + { - kj::Thread([] () { + KJ_LOG(INFO, "Loading Pyodide package from internet..."); + kj::Thread([&] () { kj::AsyncIoContext io = kj::setupAsyncIo(); kj::HttpHeaderTable table; @@ -2584,16 +2625,22 @@ void Server::fetchPyodideBundle() { kj::HttpHeaders headers(table); - auto req = client->request(kj::HttpMethod::GET, "https://pyodide.runtime-playground.workers.dev/python-runtime-capnp-bin/pyodide-1.capnp.bin", headers); + kj::String url = kj::str("https://pyodide.runtime-playground.workers.dev/python-runtime-capnp-bin/pyodide-", version, ".capnp.bin"); + + auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers); auto res = req.response.wait(io.waitScope); auto body = res.body->readAllBytes().wait(io.waitScope); - api::pyodide::setPyodideBundleData(kj::mv(body)); + writePyodideBundleFileToDisk(pyConfig.pyodideDiskCacheRoot, version, body); + + api::pyodide::setPyodideBundleData(kj::str(version), kj::mv(body)); }); } + KJ_LOG(INFO, "Loaded Pyodide package from internet"); + } kj::Own Server::makeWorker(kj::StringPtr name, config::Worker::Reader conf, @@ -2720,8 +2767,8 @@ kj::Own Server::makeWorker(kj::StringPtr name, config::Worker:: } // Get the pyodide bundle and register it in memory. - // After this, the pyodide bundle is accessible at pyodideBundleGlobal - fetchPyodideBundle(); + // After this, the pyodide bundle is accessible through getPyodideBundle + fetchPyodideBundle(pythonConfig); auto api = kj::heap(globalContext->v8System, featureFlags.asReader(), diff --git a/src/workerd/server/server.h b/src/workerd/server/server.h index ecdf81921ce..23e67de4bf4 100644 --- a/src/workerd/server/server.h +++ b/src/workerd/server/server.h @@ -61,8 +61,11 @@ class Server final: private kj::TaskSet::ErrorHandler { void enableControl(uint fd) { controlOverride = kj::heap(fd); } - void setPythonDiskCacheRoot(kj::Maybe> &&dkr) { - pythonConfig.diskCacheRoot = kj::mv(dkr); + void setPackageDiskCacheRoot(kj::Maybe> &&dkr) { + pythonConfig.packageDiskCacheRoot = kj::mv(dkr); + } + void setPyodideDiskCacheRoot(kj::Maybe> &&dkr) { + pythonConfig.pyodideDiskCacheRoot = kj::mv(dkr); } void setPythonCreateSnapshot() { pythonConfig.createSnapshot = true; @@ -103,7 +106,8 @@ class Server final: private kj::TaskSet::ErrorHandler { kj::EntropySource& entropySource; kj::Function reportConfigError; PythonConfig pythonConfig = PythonConfig { - .diskCacheRoot = kj::none, + .packageDiskCacheRoot = kj::none, + .pyodideDiskCacheRoot = kj::none, .createSnapshot = false, .createBaselineSnapshot = false }; @@ -228,7 +232,7 @@ class Server final: private kj::TaskSet::ErrorHandler { kj::ForkedPromise& forkedDrainWhen, bool forTest = false); - void fetchPyodideBundle(); + void fetchPyodideBundle(PythonConfig& pyConfig); }; // An ActorStorage implementation which will always respond to reads as if the state is empty, diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel index b94b998ca84..5b3de9e860b 100644 --- a/src/workerd/server/tests/python/BUILD.bazel +++ b/src/workerd/server/tests/python/BUILD.bazel @@ -1,42 +1,50 @@ load("//:build/wd_test.bzl", "wd_test") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +copy_file( + name = "pyodide-2.capnp.bin@rule", + src = "//src/pyodide:pyodide.capnp.bin", + out = "pyodide-bundle-cache/pyodide-2.capnp.bin" +) + wd_test( src = "hello/hello.wd-test", - args = ["--experimental"], + args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-2.capnp.bin@rule)/.."], data = glob( [ "hello/*", ], exclude = ["**/*.wd-test"], - ), + ) + ["pyodide-2.capnp.bin@rule"], ) wd_test( src = "env-param/env.wd-test", - args = ["--experimental"], + args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-2.capnp.bin@rule)/.."], data = glob( [ "env-param/*", ], exclude = ["**/*.wd-test"], - ), + ) + ["pyodide-2.capnp.bin@rule"], ) wd_test( src = "random/random.wd-test", - args = ["--experimental"], + args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-2.capnp.bin@rule)/.."], data = glob( [ "random/*", ], exclude = ["**/*.wd-test"], - ), + ) + ["pyodide-2.capnp.bin@rule"], ) # Disabled because it tests the same thing as the import test defined in import_tests.bzl # wd_test( # src = "langchain/langchain.wd-test", -# args = ["--experimental", "--disk-cache-dir", "../all_pyodide_wheels"], +# args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels"], # data = glob( # [ # "langchain/*", @@ -47,13 +55,13 @@ wd_test( wd_test( src = "subdirectory/subdirectory.wd-test", - args = ["--experimental"], + args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-2.capnp.bin@rule)/.."], data = glob( [ "subdirectory/**", ], exclude = ["**/*.wd-test"], - ), + ) + ["pyodide-2.capnp.bin@rule"], ) load("//src/workerd/server/tests/python:import_tests.bzl", "gen_import_tests") diff --git a/src/workerd/server/tests/python/import_tests.bzl b/src/workerd/server/tests/python/import_tests.bzl index 02fa5f1513b..b1d43a53725 100644 --- a/src/workerd/server/tests/python/import_tests.bzl +++ b/src/workerd/server/tests/python/import_tests.bzl @@ -1,4 +1,5 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") + load("//:build/wd_test.bzl", "wd_test") def generate_import_py_file(imports): @@ -51,7 +52,7 @@ def gen_import_tests(to_test): wd_test( src = wd_test_fname, - args = ["--experimental", "--disk-cache-dir", "../all_pyodide_wheels"], - data = [worker_py_fname, "@all_pyodide_wheels//:whls"], + args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-2.capnp.bin@rule)/.."], + data = [worker_py_fname, "@all_pyodide_wheels//:whls", "pyodide-2.capnp.bin@rule"], tags = ["slow"], ) diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index 914283eb167..6f8ccc2b70a 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -117,7 +117,8 @@ JSG_DECLARE_ISOLATE_TYPE(JsgWorkerdIsolate, static PythonConfig defaultConfig { - .diskCacheRoot = kj::none, + .packageDiskCacheRoot = kj::none, + .pyodideDiskCacheRoot = kj::none, .createSnapshot = false, .createBaselineSnapshot = false, }; @@ -449,7 +450,7 @@ void WorkerdApi::compileModules( "The python_workers compatibility flag is required to use Python."); // Inject Pyodide bundle if(util::Autogate::isEnabled(util::AutogateKey::PYODIDE_LOAD_EXTERNAL)) { - modules->addBuiltinBundle(KJ_ASSERT_NONNULL(pyodideBundleGlobal), kj::none); + modules->addBuiltinBundle(getPyodideBundle(kj::str("2")), kj::none); } // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?) { @@ -517,7 +518,7 @@ void WorkerdApi::compileModules( using ObjectModuleInfo = jsg::ModuleRegistry::ObjectModuleInfo; using ResolveMethod = jsg::ModuleRegistry::ResolveMethod; auto specifier = "pyodide-internal:disk_cache"; - auto diskCache = jsg::alloc(impl->pythonConfig.diskCacheRoot); + auto diskCache = jsg::alloc(impl->pythonConfig.packageDiskCacheRoot); modules->addBuiltinModule( specifier, [specifier = kj::str(specifier), diskCache = kj::mv(diskCache)]( @@ -940,9 +941,9 @@ kj::Own WorkerdApi::initializeBundleModuleRegistry jsg::modules::Module::newJsgObjectModuleHandler< DiskCache, JsgWorkerdIsolate_TypeWrapper>( - [diskCache=jsg::alloc(pythonConfig.diskCacheRoot)] + [packageDiskCache=jsg::alloc(pythonConfig.packageDiskCacheRoot)] (jsg::Lock& js) mutable -> jsg::Ref { - return diskCache.addRef(); + return packageDiskCache.addRef(); })); // Inject a (disabled) SimplePythonLimiter pyodideBundleBuilder.addSynthetic(limiterSpecifier, diff --git a/src/workerd/server/workerd.c++ b/src/workerd/server/workerd.c++ index f260ec05d1e..474396ee8f4 100644 --- a/src/workerd/server/workerd.c++ +++ b/src/workerd/server/workerd.c++ @@ -731,8 +731,10 @@ public: .addOption({"experimental"}, [this]() { server->allowExperimental(); return true; }, "Permit the use of experimental features which may break backwards " "compatibility in a future release.") - .addOptionWithArg({"disk-cache-dir"}, CLI_METHOD(setPythonDiskCacheDir), "", + .addOptionWithArg({"pyodide-package-disk-cache-dir"}, CLI_METHOD(setPackageDiskCacheDir), "", "Use as a disk cache to avoid repeatedly fetching packages from the internet. ") + .addOptionWithArg({"pyodide-bundle-disk-cache-dir"}, CLI_METHOD(setPyodideDiskCacheDir), "", + "Use as a disk cache to avoid repeatedly fetching Pyodide bundles from the internet. ") .addOption({"python-save-snapshot"}, [this]() { server->setPythonCreateSnapshot(); return true; }, "Save a dedicated snapshot to the disk cache") .addOption({"python-save-baseline-snapshot"}, [this]() { server->setPythonCreateBaselineSnapshot(); return true; }, @@ -937,10 +939,16 @@ public: server->enableControl(fd); } - void setPythonDiskCacheDir(kj::StringPtr pathStr) { + void setPackageDiskCacheDir(kj::StringPtr pathStr) { kj::Path path = fs->getCurrentPath().eval(pathStr); kj::Maybe> dir = fs->getRoot().tryOpenSubdir(path, kj::WriteMode::MODIFY); - server->setPythonDiskCacheRoot(kj::mv(dir)); + server->setPackageDiskCacheRoot(kj::mv(dir)); + } + + void setPyodideDiskCacheDir(kj::StringPtr pathStr) { + kj::Path path = fs->getCurrentPath().eval(pathStr); + kj::Maybe> dir = fs->getRoot().tryOpenSubdir(path, kj::WriteMode::MODIFY); + server->setPyodideDiskCacheRoot(kj::mv(dir)); } void watch() { diff --git a/src/workerd/tests/test-fixture.c++ b/src/workerd/tests/test-fixture.c++ index 13081b2b023..394cf8e9be4 100644 --- a/src/workerd/tests/test-fixture.c++ +++ b/src/workerd/tests/test-fixture.c++ @@ -264,7 +264,7 @@ class MockActorLoopback : public Worker::Actor::Loopback, public kj::Refcounted using api::pyodide::PythonConfig; -PythonConfig defaultPythonConfig { .diskCacheRoot = kj::none, .createSnapshot = false, .createBaselineSnapshot = false }; +PythonConfig defaultPythonConfig { .packageDiskCacheRoot = kj::none, .pyodideDiskCacheRoot = kj::none, .createSnapshot = false, .createBaselineSnapshot = false }; TestFixture::TestFixture(SetupParams&& params) : waitScope(params.waitScope), From 05f3d6df66f97569987d87105f5429b9abbf346a Mon Sep 17 00:00:00 2001 From: Garrett Gu Date: Tue, 6 Aug 2024 10:06:50 -0500 Subject: [PATCH 4/7] address feedback, rename version to dev, only fetch pyodide when needed --- src/pyodide/BUILD.bazel | 2 - src/workerd/server/server.c++ | 81 ---------------- src/workerd/server/server.h | 2 - src/workerd/server/tests/python/BUILD.bazel | 24 ++--- .../server/tests/python/import_tests.bzl | 4 +- src/workerd/server/workerd-api.c++ | 93 ++++++++++++++++++- 6 files changed, 106 insertions(+), 100 deletions(-) diff --git a/src/pyodide/BUILD.bazel b/src/pyodide/BUILD.bazel index b9a4560eeee..d68bec68543 100644 --- a/src/pyodide/BUILD.bazel +++ b/src/pyodide/BUILD.bazel @@ -1,5 +1,3 @@ -package(default_visibility = ["//visibility:public"]) - load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@bazel_skylib//rules:expand_template.bzl", "expand_template") diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 3b86e750607..b662285ee60 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -2566,83 +2566,6 @@ void Server::abortAllActors() { } } -kj::Path getPyodideBundlePath(kj::String &version) { - return kj::Path(kj::str("pyodide-", version, ".capnp.bin")); -} - -kj::Maybe> getPyodideBundleFile(kj::Maybe> &maybeDir, kj::String &version) { - KJ_IF_SOME(dir, maybeDir) { - kj::Path filename = getPyodideBundlePath(version); - auto file = dir->tryOpenFile(filename); - - return file; - } - - return kj::none; -} - -void writePyodideBundleFileToDisk(kj::Maybe> &maybeDir, kj::String &version, kj::ArrayPtr bytes) { - KJ_IF_SOME(dir, maybeDir) { - kj::Path filename = getPyodideBundlePath(version); - auto replacer = dir->replaceFile(filename, kj::WriteMode::CREATE | kj::WriteMode::MODIFY); - - replacer->get().writeAll(bytes); - replacer->commit(); - } -} - -void Server::fetchPyodideBundle(api::pyodide::PythonConfig& pyConfig) { - auto version = kj::str("2"); //TODO: hardcoded version number, should probably get this from pyConfig eventually - - if(api::pyodide::hasPyodideBundle(kj::str(version))) { - KJ_LOG(WARNING, "Pyodide version ", kj::str(version), " already exists in pyodide bundle table"); - return; - } - - auto maybePyodideBundleFile = getPyodideBundleFile(pyConfig.pyodideDiskCacheRoot, version); - KJ_IF_SOME(pyodideBundleFile, maybePyodideBundleFile) { - auto body = pyodideBundleFile->readAllBytes(); - api::pyodide::setPyodideBundleData(kj::str(version), kj::mv(body)); - - return; - } - - { - KJ_LOG(INFO, "Loading Pyodide package from internet..."); - kj::Thread([&] () { - kj::AsyncIoContext io = kj::setupAsyncIo(); - kj::HttpHeaderTable table; - - kj::TlsContext::Options options; - options.useSystemTrustStore = true; - - kj::Own tls = kj::heap(kj::mv(options)); - auto &network = io.provider->getNetwork(); - auto tlsNetwork = tls->wrapNetwork(network); - auto &timer = io.provider->getTimer(); - - auto client = kj::newHttpClient(timer, table, network, *tlsNetwork); - - kj::HttpHeaders headers(table); - - kj::String url = kj::str("https://pyodide.runtime-playground.workers.dev/python-runtime-capnp-bin/pyodide-", version, ".capnp.bin"); - - auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers); - - auto res = req.response.wait(io.waitScope); - auto body = res.body->readAllBytes().wait(io.waitScope); - - writePyodideBundleFileToDisk(pyConfig.pyodideDiskCacheRoot, version, body); - - api::pyodide::setPyodideBundleData(kj::str(version), kj::mv(body)); - - }); - } - - KJ_LOG(INFO, "Loaded Pyodide package from internet"); - -} - kj::Own Server::makeWorker(kj::StringPtr name, config::Worker::Reader conf, capnp::List::Reader extensions) { TRACE_EVENT("workerd", "Server::makeWorker()", "name", name.cStr()); @@ -2766,10 +2689,6 @@ kj::Own Server::makeWorker(kj::StringPtr name, config::Worker:: *observer, conf, featureFlags.asReader(), pythonConfig); } - // Get the pyodide bundle and register it in memory. - // After this, the pyodide bundle is accessible through getPyodideBundle - fetchPyodideBundle(pythonConfig); - auto api = kj::heap(globalContext->v8System, featureFlags.asReader(), *limitEnforcer, diff --git a/src/workerd/server/server.h b/src/workerd/server/server.h index 23e67de4bf4..b451f8b0488 100644 --- a/src/workerd/server/server.h +++ b/src/workerd/server/server.h @@ -231,8 +231,6 @@ class Server final: private kj::TaskSet::ErrorHandler { kj::HttpHeaderTable::Builder& headerTableBuilder, kj::ForkedPromise& forkedDrainWhen, bool forTest = false); - - void fetchPyodideBundle(PythonConfig& pyConfig); }; // An ActorStorage implementation which will always respond to reads as if the state is empty, diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel index 5b3de9e860b..97a212360f5 100644 --- a/src/workerd/server/tests/python/BUILD.bazel +++ b/src/workerd/server/tests/python/BUILD.bazel @@ -3,65 +3,65 @@ load("//:build/wd_test.bzl", "wd_test") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") copy_file( - name = "pyodide-2.capnp.bin@rule", + name = "pyodide-dev.capnp.bin@rule", src = "//src/pyodide:pyodide.capnp.bin", - out = "pyodide-bundle-cache/pyodide-2.capnp.bin" + out = "pyodide-bundle-cache/pyodide-dev.capnp.bin" ) wd_test( src = "hello/hello.wd-test", - args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-2.capnp.bin@rule)/.."], + args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], data = glob( [ "hello/*", ], exclude = ["**/*.wd-test"], - ) + ["pyodide-2.capnp.bin@rule"], + ) + ["pyodide-dev.capnp.bin@rule"], ) wd_test( src = "env-param/env.wd-test", - args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-2.capnp.bin@rule)/.."], + args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], data = glob( [ "env-param/*", ], exclude = ["**/*.wd-test"], - ) + ["pyodide-2.capnp.bin@rule"], + ) + ["pyodide-dev.capnp.bin@rule"], ) wd_test( src = "random/random.wd-test", - args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-2.capnp.bin@rule)/.."], + args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], data = glob( [ "random/*", ], exclude = ["**/*.wd-test"], - ) + ["pyodide-2.capnp.bin@rule"], + ) + ["pyodide-dev.capnp.bin@rule"], ) # Disabled because it tests the same thing as the import test defined in import_tests.bzl # wd_test( # src = "langchain/langchain.wd-test", -# args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels"], +# args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], # data = glob( # [ # "langchain/*", # ], # exclude = ["**/*.wd-test"], -# ) + ["@all_pyodide_wheels//:whls"], +# ) + ["@all_pyodide_wheels//:whls", "pyodide-dev.capnp.bin@rule"], # ) wd_test( src = "subdirectory/subdirectory.wd-test", - args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-2.capnp.bin@rule)/.."], + args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], data = glob( [ "subdirectory/**", ], exclude = ["**/*.wd-test"], - ) + ["pyodide-2.capnp.bin@rule"], + ) + ["pyodide-dev.capnp.bin@rule"], ) load("//src/workerd/server/tests/python:import_tests.bzl", "gen_import_tests") diff --git a/src/workerd/server/tests/python/import_tests.bzl b/src/workerd/server/tests/python/import_tests.bzl index b1d43a53725..fdb92d3aa3c 100644 --- a/src/workerd/server/tests/python/import_tests.bzl +++ b/src/workerd/server/tests/python/import_tests.bzl @@ -52,7 +52,7 @@ def gen_import_tests(to_test): wd_test( src = wd_test_fname, - args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-2.capnp.bin@rule)/.."], - data = [worker_py_fname, "@all_pyodide_wheels//:whls", "pyodide-2.capnp.bin@rule"], + args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], + data = [worker_py_fname, "@all_pyodide_wheels//:whls", "pyodide-dev.capnp.bin@rule"], tags = ["slow"], ) diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index 6f8ccc2b70a..c3a10755dbd 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -44,6 +44,8 @@ #include #include #include +#include +#include #ifdef WORKERD_EXPERIMENTAL_ENABLE_WEBGPU #include #else @@ -434,6 +436,89 @@ kj::Maybe WorkerdApi::tryCompileModule( KJ_UNREACHABLE; } +kj::Path getPyodideBundleFileName(kj::StringPtr version) { + return kj::Path(kj::str("pyodide-", version, ".capnp.bin")); +} + +kj::Maybe> getPyodideBundleFile(const kj::Maybe> &maybeDir, kj::StringPtr version) { + KJ_IF_SOME(dir, maybeDir) { + kj::Path filename = getPyodideBundleFileName(version); + auto file = dir->tryOpenFile(filename); + + return file; + } + + return kj::none; +} + +void writePyodideBundleFileToDisk(const kj::Maybe> &maybeDir, kj::StringPtr version, kj::ArrayPtr bytes) { + KJ_IF_SOME(dir, maybeDir) { + kj::Path filename = getPyodideBundleFileName(version); + auto replacer = dir->replaceFile(filename, kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + + replacer->get().writeAll(bytes); + replacer->commit(); + } +} + +bool fetchPyodideBundle(const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version) { + if(api::pyodide::hasPyodideBundle(version)) { + KJ_LOG(WARNING, "Pyodide version ", version, " already exists in pyodide bundle table"); + return true; + } + + auto maybePyodideBundleFile = getPyodideBundleFile(pyConfig.pyodideDiskCacheRoot, version); + KJ_IF_SOME(pyodideBundleFile, maybePyodideBundleFile) { + auto body = pyodideBundleFile->readAllBytes(); + api::pyodide::setPyodideBundleData(kj::str(version), kj::mv(body)); + + return true; + } + + if (version == "dev") { + // the "dev" version is special and indicates we're using the tip-of-tree version built for testing + // so we shouldn't fetch it from the internet, only check for its existence in the disk cache + return false; + } + + { + KJ_LOG(INFO, "Loading Pyodide package from internet..."); + kj::Thread([&] () { + kj::AsyncIoContext io = kj::setupAsyncIo(); + kj::HttpHeaderTable table; + + kj::TlsContext::Options options; + options.useSystemTrustStore = true; + + kj::Own tls = kj::heap(kj::mv(options)); + auto &network = io.provider->getNetwork(); + auto tlsNetwork = tls->wrapNetwork(network); + auto &timer = io.provider->getTimer(); + + auto client = kj::newHttpClient(timer, table, network, *tlsNetwork); + + kj::HttpHeaders headers(table); + + kj::String url = kj::str("https://pyodide.runtime-playground.workers.dev/python-runtime-capnp-bin/pyodide-", version, ".capnp.bin"); + + auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers); + + auto res = req.response.wait(io.waitScope); + auto body = res.body->readAllBytes().wait(io.waitScope); + + writePyodideBundleFileToDisk(pyConfig.pyodideDiskCacheRoot, version, body); + + api::pyodide::setPyodideBundleData(kj::str(version), kj::mv(body)); + + }); + } + + KJ_LOG(INFO, "Loaded Pyodide package from internet"); + + return true; + +} + void WorkerdApi::compileModules( jsg::Lock& lockParam, config::Worker::Reader conf, Worker::ValidationErrorReporter& errorReporter, @@ -450,7 +535,13 @@ void WorkerdApi::compileModules( "The python_workers compatibility flag is required to use Python."); // Inject Pyodide bundle if(util::Autogate::isEnabled(util::AutogateKey::PYODIDE_LOAD_EXTERNAL)) { - modules->addBuiltinBundle(getPyodideBundle(kj::str("2")), kj::none); + if (fetchPyodideBundle(impl->pythonConfig, "dev"_kj)) { + modules->addBuiltinBundle(getPyodideBundle("dev"_kj), kj::none); + } else { + // TODO: hardcoded version number + KJ_REQUIRE(fetchPyodideBundle(impl->pythonConfig, "2"_kj), "Failed to get both dev and hardcoded Pyodide version"); + modules->addBuiltinBundle(getPyodideBundle("2"_kj), kj::none); + } } // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?) { From 928869a31f6c2107cda9ccfe32b8232aef1b9ffb Mon Sep 17 00:00:00 2001 From: Garrett Gu Date: Wed, 7 Aug 2024 14:44:23 -0500 Subject: [PATCH 5/7] get pyodide version from compat flag annotations, set pyodide-load-external autogate on all python tests --- src/workerd/io/compatibility-date.c++ | 60 +++++++++++++++++++ src/workerd/io/compatibility-date.h | 3 + .../server/tests/python/env-param/env.wd-test | 4 ++ .../server/tests/python/hello/hello.wd-test | 4 ++ .../server/tests/python/import_tests.bzl | 4 ++ .../tests/python/langchain/langchain.wd-test | 4 ++ .../server/tests/python/random/random.wd-test | 4 ++ .../python/subdirectory/subdirectory.wd-test | 4 ++ src/workerd/server/workerd-api.c++ | 9 +-- 9 files changed, 92 insertions(+), 4 deletions(-) diff --git a/src/workerd/io/compatibility-date.c++ b/src/workerd/io/compatibility-date.c++ index 5cef6518470..c3eae0f4b27 100644 --- a/src/workerd/io/compatibility-date.c++ +++ b/src/workerd/io/compatibility-date.c++ @@ -296,4 +296,64 @@ kj::Maybe normalizeCompatDate(kj::StringPtr date) { return CompatDate::parse(date).map([](auto v) { return v.toString(); }); } +struct PythonSnapshotParsedField { + PythonSnapshotRelease::Reader pythonSnapshotRelease; + capnp::StructSchema::Field field; +}; + +kj::Array makePythonSnapshotFieldTable( + capnp::StructSchema::FieldList fields) { + kj::Vector table(fields.size()); + + for (auto field: fields) { + kj::Maybe maybePythonSnapshotRelease; + + for (auto annotation: field.getProto().getAnnotations()) { + if (annotation.getId() == PYTHON_SNAPSHOT_RELEASE_ANNOTATION_ID) { + maybePythonSnapshotRelease = + annotation.getValue().getStruct().getAs(); + } + } + + KJ_IF_SOME(pythonSnapshotRelease, maybePythonSnapshotRelease) { + table.add(PythonSnapshotParsedField{ + .pythonSnapshotRelease = pythonSnapshotRelease, + .field = field, + }); + } + } + + return table.releaseAsArray(); +} + +kj::Maybe getPythonSnapshotRelease( + CompatibilityFlags::Reader featureFlags) { + uint latestFieldOrdinal = 0; + kj::Maybe result; + + static auto fieldTable = + makePythonSnapshotFieldTable(capnp::Schema::from().getFields()); + + for (auto field: fieldTable) { + bool isEnabled = capnp::toDynamic(featureFlags).get(field.field).as(); + if (!isEnabled) { + continue; + } + + // We pick the flag with the highest ordinal value that is enabled and has a + // pythonSnapshotRelease annotation. + // + // The fieldTable is probably ordered by the ordinal anyway, but doesn't hurt to be explicit + // here. + // + // TODO(later): make sure this is well tested once we have more than one compat flag. + if (latestFieldOrdinal < field.field.getIndex()) { + latestFieldOrdinal = field.field.getIndex(); + result = field.pythonSnapshotRelease; + } + } + + return result; +} + } // namespace workerd diff --git a/src/workerd/io/compatibility-date.h b/src/workerd/io/compatibility-date.h index 2ebfbd45068..c182edf76a0 100644 --- a/src/workerd/io/compatibility-date.h +++ b/src/workerd/io/compatibility-date.h @@ -50,6 +50,9 @@ kj::Maybe normalizeCompatDate(kj::StringPtr date); // Returns the current date as a string formatted by CompatDate. kj::String currentDateStr(); +kj::Maybe getPythonSnapshotRelease( + CompatibilityFlags::Reader featureFlags); + // These values come from src/workerd/io/compatibility-date.capnp static constexpr uint64_t COMPAT_ENABLE_FLAG_ANNOTATION_ID = 0xb6dabbc87cd1b03eull; static constexpr uint64_t COMPAT_DISABLE_FLAG_ANNOTATION_ID = 0xd145cf1adc42577cull; diff --git a/src/workerd/server/tests/python/env-param/env.wd-test b/src/workerd/server/tests/python/env-param/env.wd-test index 4437452ad1a..b2c27541a52 100644 --- a/src/workerd/server/tests/python/env-param/env.wd-test +++ b/src/workerd/server/tests/python/env-param/env.wd-test @@ -18,4 +18,8 @@ const unitTests :Workerd.Config = ( ) ), ], + + autogates = [ + "workerd-autogate-pyodide-load-external", + ] ); diff --git a/src/workerd/server/tests/python/hello/hello.wd-test b/src/workerd/server/tests/python/hello/hello.wd-test index 612bdd1645b..7b10cd9986f 100644 --- a/src/workerd/server/tests/python/hello/hello.wd-test +++ b/src/workerd/server/tests/python/hello/hello.wd-test @@ -12,4 +12,8 @@ const unitTests :Workerd.Config = ( ) ), ], + + autogates = [ + "workerd-autogate-pyodide-load-external", + ] ); diff --git a/src/workerd/server/tests/python/import_tests.bzl b/src/workerd/server/tests/python/import_tests.bzl index fdb92d3aa3c..a5b65f4c7c8 100644 --- a/src/workerd/server/tests/python/import_tests.bzl +++ b/src/workerd/server/tests/python/import_tests.bzl @@ -27,6 +27,10 @@ const unitTests :Workerd.Config = ( ) ), ], + + autogates = [ + "workerd-autogate-pyodide-load-external", + ] );""" def generate_wd_test_file(requirement): diff --git a/src/workerd/server/tests/python/langchain/langchain.wd-test b/src/workerd/server/tests/python/langchain/langchain.wd-test index ba9287eccdb..39c617f0f73 100644 --- a/src/workerd/server/tests/python/langchain/langchain.wd-test +++ b/src/workerd/server/tests/python/langchain/langchain.wd-test @@ -16,4 +16,8 @@ const unitTests :Workerd.Config = ( ) ), ], + + autogates = [ + "workerd-autogate-pyodide-load-external", + ] ); diff --git a/src/workerd/server/tests/python/random/random.wd-test b/src/workerd/server/tests/python/random/random.wd-test index 612bdd1645b..7b10cd9986f 100644 --- a/src/workerd/server/tests/python/random/random.wd-test +++ b/src/workerd/server/tests/python/random/random.wd-test @@ -12,4 +12,8 @@ const unitTests :Workerd.Config = ( ) ), ], + + autogates = [ + "workerd-autogate-pyodide-load-external", + ] ); diff --git a/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test b/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test index 19df255fbeb..92f00b095d3 100644 --- a/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test +++ b/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test @@ -14,5 +14,9 @@ const unitTests :Workerd.Config = ( ) ), ], + + autogates = [ + "workerd-autogate-pyodide-load-external", + ] ); diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index c3a10755dbd..c9733fdc80a 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -499,7 +499,7 @@ bool fetchPyodideBundle(const api::pyodide::PythonConfig& pyConfig, kj::StringPt kj::HttpHeaders headers(table); - kj::String url = kj::str("https://pyodide.runtime-playground.workers.dev/python-runtime-capnp-bin/pyodide-", version, ".capnp.bin"); + kj::String url = kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide-", version, ".capnp.bin"); auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers); @@ -538,9 +538,10 @@ void WorkerdApi::compileModules( if (fetchPyodideBundle(impl->pythonConfig, "dev"_kj)) { modules->addBuiltinBundle(getPyodideBundle("dev"_kj), kj::none); } else { - // TODO: hardcoded version number - KJ_REQUIRE(fetchPyodideBundle(impl->pythonConfig, "2"_kj), "Failed to get both dev and hardcoded Pyodide version"); - modules->addBuiltinBundle(getPyodideBundle("2"_kj), kj::none); + auto version = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags)).getPyodide(); + + KJ_REQUIRE(fetchPyodideBundle(impl->pythonConfig, version), "Failed to get Pyodide bundle"); + modules->addBuiltinBundle(getPyodideBundle(version), kj::none); } } // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?) From d24fab896744d5d104727a88aca0f66b1dddfd83 Mon Sep 17 00:00:00 2001 From: Garrett Gu Date: Tue, 6 Aug 2024 15:08:02 -0500 Subject: [PATCH 6/7] refactor bazel test setup --- src/workerd/server/tests/python/BUILD.bazel | 42 ++++++++----------- .../server/tests/python/import_tests.bzl | 8 ++-- .../tests/python/langchain/langchain.wd-test | 23 ---------- .../server/tests/python/langchain/worker.py | 12 ------ .../server/tests/python/py_wd_test.bzl | 19 +++++++++ 5 files changed, 41 insertions(+), 63 deletions(-) delete mode 100644 src/workerd/server/tests/python/langchain/langchain.wd-test delete mode 100644 src/workerd/server/tests/python/langchain/worker.py create mode 100644 src/workerd/server/tests/python/py_wd_test.bzl diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel index 97a212360f5..2f9d85c9966 100644 --- a/src/workerd/server/tests/python/BUILD.bazel +++ b/src/workerd/server/tests/python/BUILD.bazel @@ -1,67 +1,61 @@ load("//:build/wd_test.bzl", "wd_test") +load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test") + load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +# pyodide-dev.capnp.bin represents a custom pyodide version "dev" that is generated +# at build time using the latest contents of the src/pyodide directory. +# This is used to run tests to ensure that they are always run against the latest build of +# the Pyodide bundle. copy_file( name = "pyodide-dev.capnp.bin@rule", src = "//src/pyodide:pyodide.capnp.bin", out = "pyodide-bundle-cache/pyodide-dev.capnp.bin" ) -wd_test( +py_wd_test( src = "hello/hello.wd-test", - args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], + args = ["--experimental"], data = glob( [ "hello/*", ], exclude = ["**/*.wd-test"], - ) + ["pyodide-dev.capnp.bin@rule"], + ), ) -wd_test( +py_wd_test( src = "env-param/env.wd-test", - args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], + args = ["--experimental"], data = glob( [ "env-param/*", ], exclude = ["**/*.wd-test"], - ) + ["pyodide-dev.capnp.bin@rule"], + ), ) -wd_test( +py_wd_test( src = "random/random.wd-test", - args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], + args = ["--experimental"], data = glob( [ "random/*", ], exclude = ["**/*.wd-test"], - ) + ["pyodide-dev.capnp.bin@rule"], + ), ) -# Disabled because it tests the same thing as the import test defined in import_tests.bzl -# wd_test( -# src = "langchain/langchain.wd-test", -# args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], -# data = glob( -# [ -# "langchain/*", -# ], -# exclude = ["**/*.wd-test"], -# ) + ["@all_pyodide_wheels//:whls", "pyodide-dev.capnp.bin@rule"], -# ) - -wd_test( +py_wd_test( src = "subdirectory/subdirectory.wd-test", - args = ["--experimental", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], + args = ["--experimental"], data = glob( [ "subdirectory/**", ], exclude = ["**/*.wd-test"], - ) + ["pyodide-dev.capnp.bin@rule"], + ), ) load("//src/workerd/server/tests/python:import_tests.bzl", "gen_import_tests") diff --git a/src/workerd/server/tests/python/import_tests.bzl b/src/workerd/server/tests/python/import_tests.bzl index a5b65f4c7c8..850dc807be8 100644 --- a/src/workerd/server/tests/python/import_tests.bzl +++ b/src/workerd/server/tests/python/import_tests.bzl @@ -1,6 +1,6 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") -load("//:build/wd_test.bzl", "wd_test") +load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test") def generate_import_py_file(imports): res = "" @@ -54,9 +54,9 @@ def gen_import_tests(to_test): tags = ["slow"], ) - wd_test( + py_wd_test( src = wd_test_fname, - args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels", "--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."], - data = [worker_py_fname, "@all_pyodide_wheels//:whls", "pyodide-dev.capnp.bin@rule"], + args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels"], + data = [worker_py_fname, "@all_pyodide_wheels//:whls"], tags = ["slow"], ) diff --git a/src/workerd/server/tests/python/langchain/langchain.wd-test b/src/workerd/server/tests/python/langchain/langchain.wd-test deleted file mode 100644 index 39c617f0f73..00000000000 --- a/src/workerd/server/tests/python/langchain/langchain.wd-test +++ /dev/null @@ -1,23 +0,0 @@ -using Workerd = import "/workerd/workerd.capnp"; - -const unitTests :Workerd.Config = ( - services = [ - ( name = "python-langchain", - worker = ( - modules = [ - (name = "worker.py", pythonModule = embed "./worker.py"), - (name = "aiohttp", pythonRequirement = "aiohttp"), - (name = "ssl", pythonRequirement = "ssl"), - (name = "langchain_core", pythonRequirement = ""), - (name = "langchain_openai", pythonRequirement = ""), - ], - compatibilityDate = "2024-01-15", - compatibilityFlags = ["python_workers"], - ) - ), - ], - - autogates = [ - "workerd-autogate-pyodide-load-external", - ] -); diff --git a/src/workerd/server/tests/python/langchain/worker.py b/src/workerd/server/tests/python/langchain/worker.py deleted file mode 100644 index 7b1ab338055..00000000000 --- a/src/workerd/server/tests/python/langchain/worker.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -This just tests that we can import all the dependencies and set up the -ChatOpenAI session. We don't make any requests here, so it doesn't check that -part. - -TODO: update this to test that something happened -""" - -def test(): - from langchain_core.prompts import PromptTemplate - from langchain_openai import OpenAI - diff --git a/src/workerd/server/tests/python/py_wd_test.bzl b/src/workerd/server/tests/python/py_wd_test.bzl new file mode 100644 index 00000000000..3c768cadee9 --- /dev/null +++ b/src/workerd/server/tests/python/py_wd_test.bzl @@ -0,0 +1,19 @@ +load("//:build/wd_test.bzl", "wd_test") + +def py_wd_test( + src, + data = [], + name = None, + args = [], + **kwargs +): + data += ["pyodide-dev.capnp.bin@rule"] + args += ["--pyodide-bundle-disk-cache-dir", "$(location pyodide-dev.capnp.bin@rule)/.."] + + wd_test( + src = src, + data = data, + name = name, + args = args, + **kwargs + ) From c3d0a86b934798b590fb0a4ce9e32b66e3f708fb Mon Sep 17 00:00:00 2001 From: Garrett Gu Date: Thu, 8 Aug 2024 09:18:34 -0500 Subject: [PATCH 7/7] Change versioning scheme, enable load-external autogate on samples --- samples/pyodide-fastapi/config.capnp | 4 ++++ samples/pyodide-langchain/config.capnp | 4 ++++ samples/pyodide-secret/config.capnp | 1 + samples/pyodide/config.capnp | 4 ++++ samples/repl-server-python/config.capnp | 4 ++++ src/workerd/server/workerd-api.c++ | 17 +++++++++++++---- 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/samples/pyodide-fastapi/config.capnp b/samples/pyodide-fastapi/config.capnp index 114e65afaa2..f117a47ab25 100644 --- a/samples/pyodide-fastapi/config.capnp +++ b/samples/pyodide-fastapi/config.capnp @@ -13,6 +13,10 @@ const config :Workerd.Config = ( service = "main" ), ], + + autogates = [ + "workerd-autogate-pyodide-load-external", + ] ); const mainWorker :Workerd.Worker = ( diff --git a/samples/pyodide-langchain/config.capnp b/samples/pyodide-langchain/config.capnp index 7d1f43b8b4f..87fb71aa7be 100644 --- a/samples/pyodide-langchain/config.capnp +++ b/samples/pyodide-langchain/config.capnp @@ -13,6 +13,10 @@ const config :Workerd.Config = ( service = "main" ), ], + + autogates = [ + "workerd-autogate-pyodide-load-external", + ] ); const mainWorker :Workerd.Worker = ( diff --git a/samples/pyodide-secret/config.capnp b/samples/pyodide-secret/config.capnp index bc4eb372c4f..ab1d15e25cf 100644 --- a/samples/pyodide-secret/config.capnp +++ b/samples/pyodide-secret/config.capnp @@ -17,6 +17,7 @@ const config :Workerd.Config = ( # Pyodide is included as a builtin wasm module so it requires the # corresponding autogate flag. "workerd-autogate-builtin-wasm-modules", + "workerd-autogate-pyodide-load-external", ] ); diff --git a/samples/pyodide/config.capnp b/samples/pyodide/config.capnp index c008aacb47f..14c3b2729be 100644 --- a/samples/pyodide/config.capnp +++ b/samples/pyodide/config.capnp @@ -13,6 +13,10 @@ const config :Workerd.Config = ( service = "main" ), ], + + autogates = [ + "workerd-autogate-pyodide-load-external", + ] ); const mainWorker :Workerd.Worker = ( diff --git a/samples/repl-server-python/config.capnp b/samples/repl-server-python/config.capnp index c008aacb47f..14c3b2729be 100644 --- a/samples/repl-server-python/config.capnp +++ b/samples/repl-server-python/config.capnp @@ -13,6 +13,10 @@ const config :Workerd.Config = ( service = "main" ), ], + + autogates = [ + "workerd-autogate-pyodide-load-external", + ] ); const mainWorker :Workerd.Worker = ( diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index c9733fdc80a..4c1ce9ecfdd 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -440,7 +440,8 @@ kj::Path getPyodideBundleFileName(kj::StringPtr version) { return kj::Path(kj::str("pyodide-", version, ".capnp.bin")); } -kj::Maybe> getPyodideBundleFile(const kj::Maybe> &maybeDir, kj::StringPtr version) { +kj::Maybe> getPyodideBundleFile( + const kj::Maybe> &maybeDir, kj::StringPtr version) { KJ_IF_SOME(dir, maybeDir) { kj::Path filename = getPyodideBundleFileName(version); auto file = dir->tryOpenFile(filename); @@ -451,7 +452,10 @@ kj::Maybe> getPyodideBundleFile(const kj::Maybe< return kj::none; } -void writePyodideBundleFileToDisk(const kj::Maybe> &maybeDir, kj::StringPtr version, kj::ArrayPtr bytes) { +void writePyodideBundleFileToDisk( + const kj::Maybe> &maybeDir, + kj::StringPtr version, + kj::ArrayPtr bytes) { KJ_IF_SOME(dir, maybeDir) { kj::Path filename = getPyodideBundleFileName(version); auto replacer = dir->replaceFile(filename, kj::WriteMode::CREATE | kj::WriteMode::MODIFY); @@ -462,7 +466,7 @@ void writePyodideBundleFileToDisk(const kj::Maybe> } bool fetchPyodideBundle(const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version) { - if(api::pyodide::hasPyodideBundle(version)) { + if (api::pyodide::hasPyodideBundle(version)) { KJ_LOG(WARNING, "Pyodide version ", version, " already exists in pyodide bundle table"); return true; } @@ -538,7 +542,12 @@ void WorkerdApi::compileModules( if (fetchPyodideBundle(impl->pythonConfig, "dev"_kj)) { modules->addBuiltinBundle(getPyodideBundle("dev"_kj), kj::none); } else { - auto version = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags)).getPyodide(); + auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags)); + auto pyodide = pythonRelease.getPyodide(); + auto pyodideRevision = pythonRelease.getPyodideRevision(); + auto backport = pythonRelease.getBackport(); + + auto version = kj::str(pyodide, "_", pyodideRevision, "_", backport); KJ_REQUIRE(fetchPyodideBundle(impl->pythonConfig, version), "Failed to get Pyodide bundle"); modules->addBuiltinBundle(getPyodideBundle(version), kj::none);