[types] Generate @cloudflare/workers-types with a Worker (#2405)

cloudflare · Jul 18, 2024 · 0455ed3 · 0455ed3
1 parent 476100a
commit 0455ed3
Show file tree

Hide file tree

Showing 22 changed files with 631 additions and 903 deletions.
diff --git a/.github/workflows/npm.yml b/.github/workflows/npm.yml
@@ -1,7 +1,7 @@
 name: Publish to NPM
 on:
-# Since we still need to manually upload binaries, use manual run
-# Ideally this would trigger off `release`
+  # Since we still need to manually upload binaries, use manual run
+  # Ideally this would trigger off `release`
   workflow_dispatch:
     inputs:
       patch:
@@ -57,7 +57,7 @@ jobs:
           fileName: workerd-${{ matrix.arch }}.gz
           tarBall: false
           zipBall: false
-          out-file-path: "release-downloads"
+          out-file-path: 'release-downloads'
           token: ${{ secrets.GITHUB_TOKEN }}
         # release-downloader does not support .gz files (unlike .tar.gz), decompress manually
         # Using the -N flag the right file name should be restored
@@ -87,6 +87,28 @@ jobs:
         with:
           node-version: 18
 
+      - name: Cache
+        id: cache
+        uses: actions/cache@v4
+        # Use same cache and build configuration as release build, this allows us to keep download
+        # sizes small and generate types with optimization enabled, should be slightly faster.
+        with:
+          path: ~/bazel-disk-cache
+          key: bazel-disk-cache-release-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.bazelversion', '.bazelrc', 'WORKSPACE') }}
+      - name: Setup Linux
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          wget https://apt.llvm.org/llvm.sh
+          sed -i '/apt-get install/d' llvm.sh
+          chmod +x llvm.sh
+          sudo ./llvm.sh 15
+          sudo apt-get install -y --no-install-recommends clang-15 lld-15 libunwind-15 libc++abi1-15 libc++1-15 libc++-15-dev
+          echo "build:linux --action_env=CC=/usr/lib/llvm-15/bin/clang --action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
+          echo "build:linux --host_action_env=CC=/usr/lib/llvm-15/bin/clang --host_action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
+      - name: Build type generating Worker
+        run: |
+          bazel build --disk_cache=~/bazel-disk-cache --strip=always --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev --config=release_linux //types:types_worker
+
       - name: Modify package.json version
         run: node npm/scripts/bump-version.mjs npm/workerd/package.json
         env:

diff --git a/npm/scripts/build-shim-package.mjs b/npm/scripts/build-shim-package.mjs
@@ -17,14 +17,17 @@ function buildNeutralLib() {
     '@cloudflare/workerd-darwin-64': process.env.WORKERD_VERSION,
     '@cloudflare/workerd-linux-arm64': process.env.WORKERD_VERSION,
     '@cloudflare/workerd-linux-64': process.env.WORKERD_VERSION,
-    '@cloudflare/workerd-windows-64': process.env.WORKERD_VERSION
+    '@cloudflare/workerd-windows-64': process.env.WORKERD_VERSION,
   };
   fs.writeFileSync(pjPath, JSON.stringify(package_json, null, 2) + '\n');
 
   const capnpPath = path.join('src', 'workerd', 'server', 'workerd.capnp');
 
-  fs.copyFileSync(capnpPath, path.join('npm', 'workerd', 'workerd.capnp'))
+  fs.copyFileSync(capnpPath, path.join('npm', 'workerd', 'workerd.capnp'));
 
+  const typeWorkerPath = path.join('bazel-bin', 'types', 'dist', 'index.mjs');
+
+  fs.copyFileSync(typeWorkerPath, path.join('npm', 'workerd', 'worker.mjs'));
 }
 
 buildNeutralLib();
diff --git a/src/workerd/api/rtti.c++ b/src/workerd/api/rtti.c++
@@ -29,6 +29,11 @@
 #include <workerd/api/urlpattern.h>
 #include <workerd/api/node/node.h>
 #include <workerd/jsg/modules.capnp.h>
+#include <workerd/api/hyperdrive.h>
+#include <workerd/api/eventsource.h>
+#include <workerd/api/unsafe.h>
+#include <workerd/api/memory-cache.h>
+#include <workerd/api/worker-rpc.h>
 
 #include <cloudflare/cloudflare.capnp.h>
 
@@ -53,6 +58,10 @@
   F("form-data", EW_FORMDATA_ISOLATE_TYPES)                                    \
   F("html-rewriter", EW_HTML_REWRITER_ISOLATE_TYPES)                           \
   F("http", EW_HTTP_ISOLATE_TYPES)                                             \
+  F("hyperdrive", EW_HYPERDRIVE_ISOLATE_TYPES)                                 \
+  F("unsafe", EW_UNSAFE_ISOLATE_TYPES)                                         \
+  F("memory-cache", EW_MEMORY_CACHE_ISOLATE_TYPES)                             \
+  F("pyodide", EW_PYODIDE_ISOLATE_TYPES)                                       \
   F("kv", EW_KV_ISOLATE_TYPES)                                                 \
   F("queue", EW_QUEUE_ISOLATE_TYPES)                                           \
   F("r2-admin", EW_R2_PUBLIC_BETA_ADMIN_ISOLATE_TYPES)                         \
@@ -69,7 +78,8 @@
   F("sockets", EW_SOCKETS_ISOLATE_TYPES)                                       \
   F("node", EW_NODE_ISOLATE_TYPES)                                             \
   F("rtti", EW_RTTI_ISOLATE_TYPES)                                             \
-  F("webgpu", EW_WEBGPU_ISOLATE_TYPES)
+  F("webgpu", EW_WEBGPU_ISOLATE_TYPES)                                         \
+  F("eventsource", EW_EVENTSOURCE_ISOLATE_TYPES)
 
 namespace workerd::api {
 
@@ -148,14 +158,44 @@ CompatibilityFlags::Reader compileFlags(capnp::MessageBuilder &message, kj::Stri
   return kj::mv(reader);
 }
 
+CompatibilityFlags::Reader compileAllFlags(capnp::MessageBuilder &message) {
+  auto output = message.initRoot<CompatibilityFlags>();
+  auto schema = capnp::Schema::from<CompatibilityFlags>();
+  auto dynamicOutput = capnp::toDynamic(output);
+  for (auto field: schema.getFields()) {
+    bool isNode = false;
+
+    kj::StringPtr enableFlagName;
+
+    for (auto annotation: field.getProto().getAnnotations()) {
+      if (annotation.getId() == COMPAT_ENABLE_FLAG_ANNOTATION_ID) {
+        enableFlagName = annotation.getValue().getText();
+        // Exclude nodejs_compat, since the type generation scripts don't support node:* imports
+        // TODO: Figure out typing for node compat
+        isNode = enableFlagName == "nodejs_compat" ||
+                  enableFlagName == "nodejs_compat_v2";
+      }
+    }
+
+    dynamicOutput.set(field, !isNode);
+  }
+  auto reader = output.asReader();
+  return kj::mv(reader);
+}
+
 struct TypesEncoder {
 public:
+  TypesEncoder(): compatFlags(kj::heapArray<kj::String>(0)) {}
   TypesEncoder(kj::String compatDate, kj::Array<kj::String> compatFlags): compatDate(kj::mv(compatDate)), compatFlags(kj::mv(compatFlags)) {}
 
   kj::Array<byte> encode() {
     capnp::MallocMessageBuilder flagsMessage;
-    CompatibilityFlags::Reader flags = compileFlags(flagsMessage, compatDate, false, compatFlags);
-
+    CompatibilityFlags::Reader flags;
+    KJ_IF_SOME(date, compatDate) {
+      flags = compileFlags(flagsMessage, date, true, compatFlags);
+    } else {
+      flags = compileAllFlags(flagsMessage);
+    }
     capnp::MallocMessageBuilder message;
     auto root = message.initRoot<jsg::rtti::StructureGroups>();
 
@@ -221,7 +261,7 @@ private:
     KJ_ASSERT(structureIndex == structuresSize);
   }
 
-  kj::String compatDate;
+  kj::Maybe<kj::String> compatDate;
   kj::Array<kj::String> compatFlags;
 
   unsigned int groupsIndex = 0;
@@ -235,4 +275,9 @@ kj::Array<byte> RTTIModule::exportTypes(kj::String compatDate, kj::Array<kj::Str
   return encoder.encode();
 }
 
+kj::Array<byte> RTTIModule::exportExperimentalTypes() {
+  TypesEncoder encoder;
+  return encoder.encode();
+}
+
 } // namespace workerd::api
diff --git a/src/workerd/api/rtti.h b/src/workerd/api/rtti.h
@@ -20,9 +20,11 @@ class RTTIModule final: public jsg::Object {
   RTTIModule(jsg::Lock&, const jsg::Url&) {}
 
   kj::Array<byte> exportTypes(kj::String compatDate, kj::Array<kj::String> compatFlags);
+  kj::Array<byte> exportExperimentalTypes();
 
   JSG_RESOURCE_TYPE(RTTIModule) {
     JSG_METHOD(exportTypes);
+    JSG_METHOD(exportExperimentalTypes);
   }
 };
 

diff --git a/src/workerd/tools/BUILD.bazel b/src/workerd/tools/BUILD.bazel
@@ -5,133 +5,26 @@ load("//:build/run_binary_target.bzl", "run_binary_target")
 load("//:build/wd_cc_binary.bzl", "wd_cc_binary")
 load("//:build/wd_cc_library.bzl", "wd_cc_library")
 
-# ========================================================================================
-# C++ deps for API extraction
-#
-# Both `api_encoder` and `param_extractor` need access to the API surface
-# of `workerd`, so this target allows them to both have the same deps
-
-wd_cc_library(
-    name = "api_encoder_lib",
-    deps = [
-        "//src/workerd/api:html-rewriter",
-        "//src/workerd/io",
-        "//src/workerd/jsg",
-        "//src/workerd/jsg:rtti",
-        "@capnp-cpp//src/capnp",
-    ],
-    target_compatible_with = select({
-        "@platforms//os:windows": ["@platforms//:incompatible"],
-        "//conditions:default": [],
-    }),
-)
-
-api_encoder_src = "api-encoder.c++"
-
-# ========================================================================================
-# API Encoder
-#
-# Encodes runtime API type information into a capnp schema
-
-wd_cc_binary(
-    name = "api_encoder_bin",
-    srcs = [api_encoder_src],
-    deps = [":api_encoder_lib"],
-    # Use dynamic linkage where possible to reduce binary size – unlike the workerd binary, we
-    # shouldn't need to distribute the api encoder.
-    linkstatic = 0,
-    # The dependent targets are not Windows-compatible, no need to compile this.
-    target_compatible_with = select({
-        "@platforms//os:windows": ["@platforms//:incompatible"],
-        "//conditions:default": [],
-    }),
-)
-
-# All compatibility dates that changed public facing types.
-# Remember to update `npm/workers-types/README.md` when adding new dates here.
-compat_dates = [
-    # Oldest compatibility date, with no flags enabled
-    ("2021-01-01", "oldest"),
-    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#formdata-parsing-supports-file
-    ("2021-11-03", "2021-11-03"),
-    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#settersgetters-on-api-object-prototypes
-    ("2022-01-31", "2022-01-31"),
-    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#global-navigator
-    ("2022-03-21", "2022-03-21"),
-    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#r2-bucket-list-respects-the-include-option
-    ("2022-08-04", "2022-08-04"),
-    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#new-url-parser-implementation
-    ("2022-10-31", "2022-10-31"),
-    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#streams-constructors
-    # https://developers.cloudflare.com/workers/platform/compatibility-dates/#compliant-transformstream-constructor
-    ("2022-11-30", "2022-11-30"),
-    # https://github.com/cloudflare/workerd/blob/fcb6f33d10c71975cb2ce68dbf1924a1eeadbd8a/src/workerd/io/compatibility-date.capnp#L275-L280 (http_headers_getsetcookie)
-    ("2023-03-01", "2023-03-01"),
-    # https://github.com/cloudflare/workerd/blob/fcb6f33d10c71975cb2ce68dbf1924a1eeadbd8a/src/workerd/io/compatibility-date.capnp#L307-L312 (urlsearchparams_delete_has_value_arg)
-    ("2023-07-01", "2023-07-01"),
-    # Latest compatibility date (note these types should be the same as the previous entry)
-    (None, "experimental"),
-]
-
-filegroup(
-    name = "api_encoder",
-    srcs = [
-        "//src/workerd/tools:api_encoder_" + label
-        for (date, label) in compat_dates
-    ],
-    tags = ["no-arm64"],
-    target_compatible_with = select({
-        "@platforms//os:windows": ["@platforms//:incompatible"],
-        "//conditions:default": [],
-    }),
-    visibility = ["//visibility:public"],
-)
-
-[
-    run_binary_target(
-        name = "api_encoder_" + label,
-        outs = [label + ".api.capnp.bin"],
-        args = [
-            "--output",
-            "$(location " + label + ".api.capnp.bin)",
-        ] + ([
-            "--compatibility-date",
-            date,
-        ] if date else []),
-        # Cross-compiling is not supported as this runs in target cfg.
-        tags = ["no-arm64"],
-        target_compatible_with = select({
-            "@platforms//os:windows": ["@platforms//:incompatible"],
-            "//conditions:default": [],
-        }),
-        tool = "api_encoder_bin",
-        visibility = ["//visibility:public"],
-    )
-    for (date, label) in compat_dates
-]
-
 # ========================================================================================
 # Parameter Name Extractor
 #
 # Extracts the parameter names of functions, methods, etc. of the runtime API,
-# since they're not encoded in the type information generated by `api_encoder`
-
-cc_library(
-    name = "compile_api_headers_only",
-    defines = ["API_ENCODER_HDRS_ONLY=1"],
-)
+# since they're not encoded in the type information generated by the RTTI dump
 
 cc_ast_dump(
     name = "dump_api_ast",
-    src = api_encoder_src,
+    src = "param-names-ast.c++",
     out = "api.ast.json.gz",
     target_compatible_with = select({
         "@platforms//os:windows": ["@platforms//:incompatible"],
         "//conditions:default": [],
     }),
     deps = [
-        ":api_encoder_lib",
-        ":compile_api_headers_only",
+        "//src/workerd/api:html-rewriter",
+        "//src/workerd/io",
+        "//src/workerd/jsg",
+        "//src/workerd/jsg:rtti",
+        "@capnp-cpp//src/capnp",
     ],
 )