More implementation for the new module registry (#2298)

src/workerd/api/modules.h

@@ -12,6 +12,7 @@
 #include <workerd/api/unsafe.h>
 #include <workerd/api/worker-rpc.h>
 #include <workerd/io/worker.h>
+#include <workerd/jsg/modules-new.h>
 #include <cloudflare/cloudflare.capnp.h>
 
 namespace workerd::api {
@@ -32,4 +33,40 @@ void registerModules(Registry& registry, auto featureFlags) {
   registerRpcModules(registry, featureFlags);
 }
 
+template <class TypeWrapper>
+void registerBuiltinModules(jsg::modules::ModuleRegistry::Builder& builder, auto featureFlags) {
+  builder.add(node::getInternalNodeJsCompatModuleBundle<TypeWrapper>(featureFlags));
+  builder.add(node::getExternalNodeJsCompatModuleBundle(featureFlags));
+  builder.add(getInternalSocketModuleBundle<TypeWrapper>(featureFlags));
+  builder.add(getInternalRpcModuleBundle<TypeWrapper>(featureFlags));
+
+  builder.add(getInternalUnsafeModuleBundle<TypeWrapper>(featureFlags));
+  if (featureFlags.getUnsafeModule()) {
+    builder.add(getExternalUnsafeModuleBundle<TypeWrapper>(featureFlags));
+  }
+
+  if (featureFlags.getPythonWorkers()) {
+    builder.add(pyodide::getExternalPyodideModuleBundle(featureFlags));
+    builder.add(pyodide::getInternalPyodideModuleBundle(featureFlags));
+  }
+
+  if (featureFlags.getRttiApi()) {
+    builder.add(getExternalRttiModuleBundle<TypeWrapper>(featureFlags));
+  }
+
+  {
+    jsg::modules::ModuleBundle::BuiltinBuilder builtinsBuilder(
+        jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY);
+    jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builtinsBuilder, CLOUDFLARE_BUNDLE);
+    builder.add(builtinsBuilder.finish());
+  }
+
+  {
+    jsg::modules::ModuleBundle::BuiltinBuilder builtinsBuilder(
+        jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN);
+    jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builtinsBuilder, CLOUDFLARE_BUNDLE);
+    builder.add(builtinsBuilder.finish());
+  }
+}
+
 }  // namespace workerd::api

src/workerd/api/node/async-hooks.h

@@ -217,6 +217,8 @@ class AsyncResource final: public jsg::Object {
 // Node.js.
 class AsyncHooksModule final: public jsg::Object {
 public:
+  AsyncHooksModule() = default;
+  AsyncHooksModule(jsg::Lock&, const jsg::Url&) {}
 
   JSG_RESOURCE_TYPE(AsyncHooksModule) {
     JSG_NESTED_TYPE(AsyncLocalStorage);

src/workerd/api/node/buffer.h

@@ -10,6 +10,8 @@ namespace workerd::api::node {
 // Implements utilities in support of the Node.js Buffer
 class BufferUtil final: public jsg::Object {
 public:
+  BufferUtil() = default;
+  BufferUtil(jsg::Lock&, const jsg::Url&) {}
 
   uint32_t byteLength(jsg::Lock& js, jsg::JsString str);
 

src/workerd/api/node/crypto.h

@@ -179,6 +179,9 @@ class CryptoImpl final: public jsg::Object {
     JSG_STRUCT(key, format, type, passphrase);
   };
 
+  CryptoImpl() = default;
+  CryptoImpl(jsg::Lock&, const jsg::Url&) {}
+
   kj::OneOf<kj::String, kj::Array<kj::byte>, SubtleCrypto::JsonWebKey> exportKey(
       jsg::Lock& js,
       jsg::Ref<CryptoKey> key,

src/workerd/api/node/diagnostics-channel.h

@@ -77,6 +77,9 @@ class Channel : public jsg::Object {
 
 class DiagnosticsChannelModule : public jsg::Object {
 public:
+  DiagnosticsChannelModule() = default;
+  DiagnosticsChannelModule(jsg::Lock&, const jsg::Url&) {}
+
   bool hasSubscribers(jsg::Lock& js, jsg::Name name);
   jsg::Ref<Channel> channel(jsg::Lock& js, jsg::Name name);
   void subscribe(jsg::Lock& js, jsg::Name name, jsg::Identified<Channel::MessageCallback> callback);

src/workerd/api/node/node.h

@@ -6,7 +6,9 @@
 #include "diagnostics-channel.h"
 #include "util.h"
 #include <workerd/jsg/jsg.h>
+#include <workerd/jsg/url.h>
 #include <workerd/jsg/modules.h>
+#include <workerd/jsg/modules-new.h>
 #include <capnp/dynamic.h>
 #include <node/node.capnp.h>
 
@@ -17,6 +19,9 @@ namespace workerd::api::node {
 // built-ins
 class CompatibilityFlags : public jsg::Object {
 public:
+  CompatibilityFlags() = default;
+  CompatibilityFlags(jsg::Lock&, const jsg::Url&) {}
+
   JSG_RESOURCE_TYPE(CompatibilityFlags, workerd::CompatibilityFlags::Reader flags) {
     // Not your typical JSG_RESOURCE_TYPE definition.. here we are iterating
     // through all of the compatibility flags and registering each as read-only
@@ -31,10 +36,6 @@ class CompatibilityFlags : public jsg::Object {
   }
 };
 
-template <class Registry>
-void registerNodeJsCompatModules(
-    Registry& registry, auto featureFlags) {
-
 #define NODEJS_MODULES(V)                                                       \
   V(CompatibilityFlags, "workerd:compatibility-flags")                          \
   V(AsyncHooksModule, "node-internal:async_hooks")                              \
@@ -48,6 +49,13 @@ void registerNodeJsCompatModules(
 // flag. Once they are ready to ship, move them up to the NODEJS_MODULES list.
 #define NODEJS_MODULES_EXPERIMENTAL(V)
 
+bool isNodeJsCompatEnabled(auto featureFlags) {
+  return featureFlags.getNodeJsCompat() || featureFlags.getNodeJsCompatV2();
+}
+
+template <class Registry>
+void registerNodeJsCompatModules(
+    Registry& registry, auto featureFlags) {
 #define V(T, N)                                                                 \
   registry.template addBuiltinModule<T>(N, workerd::jsg::ModuleRegistry::Type::INTERNAL);
 
@@ -58,10 +66,8 @@ void registerNodeJsCompatModules(
   }
 
 #undef V
-#undef NODEJS_MODULES
 
-  bool nodeJsCompatEnabled = featureFlags.getNodeJsCompat() ||
-                             featureFlags.getNodeJsCompatV2();
+  bool nodeJsCompatEnabled = isNodeJsCompatEnabled(featureFlags);
 
   // If the `nodejs_compat` flag isn't enabled, only register internal modules.
   // We need these for `console.log()`ing when running `workerd` locally.
@@ -84,6 +90,46 @@ void registerNodeJsCompatModules(
   }
 }
 
+template <class TypeWrapper>
+kj::Own<jsg::modules::ModuleBundle> getInternalNodeJsCompatModuleBundle(auto featureFlags) {
+  jsg::modules::ModuleBundle::BuiltinBuilder builder(
+      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY);
+#define V(M, N) static const auto k##M##Specifier = N##_url;        \
+                builder.addObject<M, TypeWrapper>(k##M##Specifier);
+  NODEJS_MODULES(V)
+  if (featureFlags.getWorkerdExperimental()) {
+    NODEJS_MODULES_EXPERIMENTAL(V)
+  }
+#undef V
+  jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builder, NODE_BUNDLE);
+  return builder.finish();
+}
+
+kj::Own<jsg::modules::ModuleBundle> getExternalNodeJsCompatModuleBundle(auto featureFlags) {
+  jsg::modules::ModuleBundle::BuiltinBuilder builder(
+      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN);
+  if (isNodeJsCompatEnabled(featureFlags)) {
+    jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builder, NODE_BUNDLE);
+  } else if (featureFlags.getNodeJsAls()) {
+    // The AsyncLocalStorage API can be enabled independently of the rest
+    // of the nodejs_compat layer.
+    jsg::Bundle::Reader reader = NODE_BUNDLE;
+    for (auto module : reader.getModules()) {
+      auto specifier = module.getName();
+      if (specifier == "node:async_hooks") {
+        KJ_DASSERT(module.getType() == jsg::ModuleType::BUILTIN);
+        KJ_DASSERT(module.which() == workerd::jsg::Module::SRC);
+        auto specifier = KJ_ASSERT_NONNULL(jsg::Url::tryParse(module.getName()));
+        builder.addEsm(specifier, module.getSrc().asChars());
+      }
+    }
+  }
+  return builder.finish();
+}
+
+#undef NODEJS_MODULES
+}  // namespace workerd::api::node
+
 #define EW_NODE_ISOLATE_TYPES              \
   api::node::CompatibilityFlags,           \
   EW_NODE_BUFFER_ISOLATE_TYPES,            \
@@ -92,4 +138,3 @@ void registerNodeJsCompatModules(
   EW_NODE_ASYNCHOOKS_ISOLATE_TYPES,        \
   EW_NODE_UTIL_ISOLATE_TYPES
 
-}  // namespace workerd::api::node

src/workerd/api/node/util.h

@@ -152,6 +152,9 @@ class MIMEType final: public jsg::Object {
 
 class UtilModule final: public jsg::Object {
 public:
+  UtilModule() = default;
+  UtilModule(jsg::Lock&, const jsg::Url&) {}
+
   jsg::Name getResourceTypeInspect(jsg::Lock& js);
 
   // `getOwnNonIndexProperties()` `filter`s

src/workerd/api/pyodide/pyodide.h

@@ -7,6 +7,8 @@
 #include <pyodide/generated/pyodide_extra.capnp.h>
 #include <pyodide/pyodide.capnp.h>
 #include <workerd/jsg/jsg.h>
+#include <workerd/jsg/modules-new.h>
+#include <workerd/jsg/url.h>
 #include <workerd/server/workerd.capnp.h>
 #include <workerd/io/io-context.h>
 
@@ -350,4 +352,18 @@ template <class Registry> void registerPyodideModules(Registry& registry, auto f
   }
 }
 
+kj::Own<jsg::modules::ModuleBundle> getInternalPyodideModuleBundle(auto featureFlags) {
+  jsg::modules::ModuleBundle::BuiltinBuilder builder(
+      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY);
+  jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builder, PYODIDE_BUNDLE);
+  return builder.finish();
+}
+
+kj::Own<jsg::modules::ModuleBundle> getExternalPyodideModuleBundle(auto featureFlags) {
+  jsg::modules::ModuleBundle::BuiltinBuilder builder(
+      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN);
+  jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builder, PYODIDE_BUNDLE);
+  return builder.finish();
+}
+
 } // namespace workerd::api::pyodide

src/workerd/api/rtti.h

@@ -8,12 +8,17 @@
 #include <kj/string.h>
 #include <workerd/io/compatibility-date.h>
 #include <workerd/jsg/modules.h>
+#include <workerd/jsg/modules-new.h>
+#include <workerd/jsg/url.h>
 #include <workerd/jsg/rtti.h>
 
 namespace workerd::api {
 
 class RTTIModule final: public jsg::Object {
 public:
+  RTTIModule() = default;
+  RTTIModule(jsg::Lock&, const jsg::Url&) {}
+
   kj::Array<byte> exportTypes(kj::String compatDate, kj::Array<kj::String> compatFlags);
 
   JSG_RESOURCE_TYPE(RTTIModule) {
@@ -27,6 +32,15 @@ void registerRTTIModule(Registry& registry) {
     workerd::jsg::ModuleRegistry::Type::BUILTIN);
 }
 
+template <typename TypeWrapper>
+kj::Own<jsg::modules::ModuleBundle> getExternalRttiModuleBundle(auto featureFlags) {
+  jsg::modules::ModuleBundle::BuiltinBuilder builder(
+      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN);
+  static const auto kSpecifier = "internal:rtti"_url;
+  builder.addObject<RTTIModule, TypeWrapper>(kSpecifier);
+  return builder.finish();
+}
+
 #define EW_RTTI_ISOLATE_TYPES api::RTTIModule
 
 } // namespace workerd::api

src/workerd/api/sockets.h

@@ -6,6 +6,8 @@
 
 #include <workerd/jsg/jsg.h>
 #include <workerd/jsg/modules.h>
+#include <workerd/jsg/modules-new.h>
+#include <workerd/jsg/url.h>
 #include "streams.h"
 
 namespace workerd::api {
@@ -223,6 +225,9 @@ jsg::Ref<Socket> connectImpl(
 
 class SocketsModule final: public jsg::Object {
 public:
+  SocketsModule() = default;
+  SocketsModule(jsg::Lock&, const jsg::Url&) {}
+
   jsg::Ref<Socket> connect(jsg::Lock& js, AnySocketAddress address,
     jsg::Optional<SocketOptions> options) {
     return connectImpl(js, kj::none, kj::mv(address), kj::mv(options));
@@ -238,7 +243,15 @@ void registerSocketsModule(
     Registry& registry, auto featureFlags) {
   registry.template addBuiltinModule<SocketsModule>("cloudflare-internal:sockets",
     workerd::jsg::ModuleRegistry::Type::INTERNAL);
+}
 
+template <typename TypeWrapper>
+kj::Own<jsg::modules::ModuleBundle> getInternalSocketModuleBundle(auto featureFlags) {
+  jsg::modules::ModuleBundle::BuiltinBuilder builder(
+      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY);
+  static const auto kSpecifier = "cloudflare-internal:sockets"_url;
+  builder.addObject<SocketsModule, TypeWrapper>(kSpecifier);
+  return builder.finish();
 }
 
 #define EW_SOCKETS_ISOLATE_TYPES     \

src/workerd/api/unsafe.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <workerd/jsg/jsg.h>
+#include <workerd/jsg/modules-new.h>
+#include <workerd/jsg/url.h>
 #include <workerd/io/io-context.h>
 
 namespace workerd::api {
@@ -9,6 +11,7 @@ namespace workerd::api {
 class UnsafeEval: public jsg::Object {
 public:
   UnsafeEval() = default;
+  UnsafeEval(jsg::Lock&, const jsg::Url&) {}
 
   // A non-capturing eval. Compile and evaluates the given script, returning whatever
   // value is returned by the script. This version of eval intentionally does not
@@ -62,6 +65,8 @@ class UnsafeEval: public jsg::Object {
 
 class UnsafeModule: public jsg::Object {
 public:
+  UnsafeModule() = default;
+  UnsafeModule(jsg::Lock&, const jsg::Url&) {}
   jsg::Promise<void> abortAllDurableObjects(jsg::Lock& js);
 
   JSG_RESOURCE_TYPE(UnsafeModule) {
@@ -84,4 +89,25 @@ template <class Registry> void registerUnsafeModules(Registry& registry, auto fe
   registry.template addBuiltinModule<UnsafeEval>("internal:unsafe-eval",
                                                  workerd::jsg::ModuleRegistry::Type::INTERNAL);
 }
+
+template <typename TypeWrapper>
+kj::Own<jsg::modules::ModuleBundle> getInternalUnsafeModuleBundle(auto featureFlags) {
+  jsg::modules::ModuleBundle::BuiltinBuilder builder(
+      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY);
+  static const auto kSpecifier = "internal:unsafe-eval"_url;
+  builder.addObject<UnsafeEval, TypeWrapper>(kSpecifier);
+  return builder.finish();
+}
+
+template <typename TypeWrapper>
+kj::Own<jsg::modules::ModuleBundle> getExternalUnsafeModuleBundle(auto featureFlags) {
+  jsg::modules::ModuleBundle::BuiltinBuilder builder(
+      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN);
+  static const auto kSpecifier = "workerd:unsafe-eval"_url;
+  builder.addObject<UnsafeEval, TypeWrapper>(kSpecifier);
+
+  static const auto kUnsafeSpecifier = "workerd:unsafe"_url;
+  builder.addObject<UnsafeModule, TypeWrapper>(kUnsafeSpecifier);
+  return builder.finish();
+}
 } // namespace workerd::api

src/workerd/api/worker-rpc.h

@@ -17,6 +17,8 @@
 #include <workerd/jsg/jsg.h>
 #include <workerd/jsg/ser.h>
 #include <workerd/jsg/function.h>
+#include <workerd/jsg/modules-new.h>
+#include <workerd/jsg/url.h>
 #include <workerd/io/io-context.h>
 #include <workerd/io/worker-interface.capnp.h>
 
@@ -480,6 +482,9 @@ class DurableObjectBase: public jsg::Object {
 // for extending.
 class EntrypointsModule: public jsg::Object {
 public:
+  EntrypointsModule() = default;
+  EntrypointsModule(jsg::Lock&, const jsg::Url&) {}
+
   JSG_RESOURCE_TYPE(EntrypointsModule) {
     JSG_NESTED_TYPE(WorkerEntrypoint);
     JSG_NESTED_TYPE_NAMED(DurableObjectBase, DurableObject);
@@ -505,4 +510,12 @@ void registerRpcModules(Registry& registry, CompatibilityFlags::Reader flags) {
       "cloudflare-internal:workers", workerd::jsg::ModuleRegistry::Type::INTERNAL);
 }
 
+template <typename TypeWrapper>
+kj::Own<jsg::modules::ModuleBundle> getInternalRpcModuleBundle(auto featureFlags) {
+  jsg::modules::ModuleBundle::BuiltinBuilder builder(
+      jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY);
+  static const auto kSpecifier = "cloudflare-internal:workers"_url;
+  builder.addObject<EntrypointsModule, TypeWrapper>(kSpecifier);
+  return builder.finish();
+}
 }; // namespace workerd::api

src/workerd/io/compatibility-date.capnp

@@ -488,4 +488,10 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
   # always been configured to accept publicly-routable internet hosts only; hostnames which map
   # to private IP addresses (as defined in e.g. RFC 1918) will be rejected. Thus, workerd has
   # always been SSRF-safe by default.
+
+  newModuleRegistry @52 :Bool
+      $compatEnableFlag("new_module_registry")
+      $compatDisableFlag("legacy_module_registry")
+      $experimental;
+  # Enables of the new module registry implementation.
 }