From cd26dd4e66327a97bd9c7beea4ad8a829bd5c7cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Somhairle=20MacLe=C3=B2id?= Date: Thu, 18 Jul 2024 18:42:16 +0100 Subject: [PATCH] [types] Generate `@cloudflare/workers-types` with a Worker (#2405) --- .github/workflows/npm.yml | 28 ++- npm/scripts/build-shim-package.mjs | 7 +- src/workerd/api/rtti.c++ | 53 ++++- src/workerd/api/rtti.h | 2 + src/workerd/tools/BUILD.bazel | 121 +--------- src/workerd/tools/api-encoder.c++ | 261 --------------------- src/workerd/tools/param-names-ast.c++ | 28 +++ types/BUILD.bazel | 54 +++-- types/scripts/build-types.ts | 162 +++++++++++++ types/scripts/build-worker.ts | 145 ++++++++++++ types/scripts/config.capnp | 18 ++ types/src/generator/index.ts | 118 ---------- types/src/generator/parameter-names.ts | 83 ++----- types/src/index.ts | 232 +----------------- types/src/transforms/internal-namespace.ts | 2 +- types/src/worker/index.ts | 75 ++++++ types/src/worker/raw.d.ts | 4 + types/src/worker/rtti.d.ts | 7 + types/src/worker/tsconfig.json | 17 ++ types/src/worker/virtual.d.ts | 9 + types/test/index.spec.ts | 96 +------- types/tsconfig.json | 12 +- 22 files changed, 631 insertions(+), 903 deletions(-) delete mode 100644 src/workerd/tools/api-encoder.c++ create mode 100644 src/workerd/tools/param-names-ast.c++ create mode 100644 types/scripts/build-types.ts create mode 100644 types/scripts/build-worker.ts create mode 100644 types/scripts/config.capnp create mode 100644 types/src/worker/index.ts create mode 100644 types/src/worker/raw.d.ts create mode 100644 types/src/worker/rtti.d.ts create mode 100644 types/src/worker/tsconfig.json create mode 100644 types/src/worker/virtual.d.ts diff --git a/.github/workflows/npm.yml b/.github/workflows/npm.yml index 6c33efbdafd..cdd55611eb1 100644 --- a/.github/workflows/npm.yml +++ b/.github/workflows/npm.yml @@ -1,7 +1,7 @@ name: Publish to NPM on: -# Since we still need to manually upload binaries, use manual run -# Ideally this would trigger off `release` + # Since we still need to manually upload binaries, use manual run + # Ideally this would trigger off `release` workflow_dispatch: inputs: patch: @@ -57,7 +57,7 @@ jobs: fileName: workerd-${{ matrix.arch }}.gz tarBall: false zipBall: false - out-file-path: "release-downloads" + out-file-path: 'release-downloads' token: ${{ secrets.GITHUB_TOKEN }} # release-downloader does not support .gz files (unlike .tar.gz), decompress manually # Using the -N flag the right file name should be restored @@ -87,6 +87,28 @@ jobs: with: node-version: 18 + - name: Cache + id: cache + uses: actions/cache@v4 + # Use same cache and build configuration as release build, this allows us to keep download + # sizes small and generate types with optimization enabled, should be slightly faster. + with: + path: ~/bazel-disk-cache + key: bazel-disk-cache-release-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.bazelversion', '.bazelrc', 'WORKSPACE') }} + - name: Setup Linux + run: | + export DEBIAN_FRONTEND=noninteractive + wget https://apt.llvm.org/llvm.sh + sed -i '/apt-get install/d' llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 15 + sudo apt-get install -y --no-install-recommends clang-15 lld-15 libunwind-15 libc++abi1-15 libc++1-15 libc++-15-dev + echo "build:linux --action_env=CC=/usr/lib/llvm-15/bin/clang --action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc + echo "build:linux --host_action_env=CC=/usr/lib/llvm-15/bin/clang --host_action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc + - name: Build type generating Worker + run: | + bazel build --disk_cache=~/bazel-disk-cache --strip=always --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev --config=release_linux //types:types_worker + - name: Modify package.json version run: node npm/scripts/bump-version.mjs npm/workerd/package.json env: diff --git a/npm/scripts/build-shim-package.mjs b/npm/scripts/build-shim-package.mjs index d089915d688..94eba0a93fc 100644 --- a/npm/scripts/build-shim-package.mjs +++ b/npm/scripts/build-shim-package.mjs @@ -17,14 +17,17 @@ function buildNeutralLib() { '@cloudflare/workerd-darwin-64': process.env.WORKERD_VERSION, '@cloudflare/workerd-linux-arm64': process.env.WORKERD_VERSION, '@cloudflare/workerd-linux-64': process.env.WORKERD_VERSION, - '@cloudflare/workerd-windows-64': process.env.WORKERD_VERSION + '@cloudflare/workerd-windows-64': process.env.WORKERD_VERSION, }; fs.writeFileSync(pjPath, JSON.stringify(package_json, null, 2) + '\n'); const capnpPath = path.join('src', 'workerd', 'server', 'workerd.capnp'); - fs.copyFileSync(capnpPath, path.join('npm', 'workerd', 'workerd.capnp')) + fs.copyFileSync(capnpPath, path.join('npm', 'workerd', 'workerd.capnp')); + const typeWorkerPath = path.join('bazel-bin', 'types', 'dist', 'index.mjs'); + + fs.copyFileSync(typeWorkerPath, path.join('npm', 'workerd', 'worker.mjs')); } buildNeutralLib(); diff --git a/src/workerd/api/rtti.c++ b/src/workerd/api/rtti.c++ index 92fd03c9eff..bfff1eb5f32 100644 --- a/src/workerd/api/rtti.c++ +++ b/src/workerd/api/rtti.c++ @@ -29,6 +29,11 @@ #include #include #include +#include +#include +#include +#include +#include #include @@ -53,6 +58,10 @@ F("form-data", EW_FORMDATA_ISOLATE_TYPES) \ F("html-rewriter", EW_HTML_REWRITER_ISOLATE_TYPES) \ F("http", EW_HTTP_ISOLATE_TYPES) \ + F("hyperdrive", EW_HYPERDRIVE_ISOLATE_TYPES) \ + F("unsafe", EW_UNSAFE_ISOLATE_TYPES) \ + F("memory-cache", EW_MEMORY_CACHE_ISOLATE_TYPES) \ + F("pyodide", EW_PYODIDE_ISOLATE_TYPES) \ F("kv", EW_KV_ISOLATE_TYPES) \ F("queue", EW_QUEUE_ISOLATE_TYPES) \ F("r2-admin", EW_R2_PUBLIC_BETA_ADMIN_ISOLATE_TYPES) \ @@ -69,7 +78,8 @@ F("sockets", EW_SOCKETS_ISOLATE_TYPES) \ F("node", EW_NODE_ISOLATE_TYPES) \ F("rtti", EW_RTTI_ISOLATE_TYPES) \ - F("webgpu", EW_WEBGPU_ISOLATE_TYPES) + F("webgpu", EW_WEBGPU_ISOLATE_TYPES) \ + F("eventsource", EW_EVENTSOURCE_ISOLATE_TYPES) namespace workerd::api { @@ -148,14 +158,44 @@ CompatibilityFlags::Reader compileFlags(capnp::MessageBuilder &message, kj::Stri return kj::mv(reader); } +CompatibilityFlags::Reader compileAllFlags(capnp::MessageBuilder &message) { + auto output = message.initRoot(); + auto schema = capnp::Schema::from(); + auto dynamicOutput = capnp::toDynamic(output); + for (auto field: schema.getFields()) { + bool isNode = false; + + kj::StringPtr enableFlagName; + + for (auto annotation: field.getProto().getAnnotations()) { + if (annotation.getId() == COMPAT_ENABLE_FLAG_ANNOTATION_ID) { + enableFlagName = annotation.getValue().getText(); + // Exclude nodejs_compat, since the type generation scripts don't support node:* imports + // TODO: Figure out typing for node compat + isNode = enableFlagName == "nodejs_compat" || + enableFlagName == "nodejs_compat_v2"; + } + } + + dynamicOutput.set(field, !isNode); + } + auto reader = output.asReader(); + return kj::mv(reader); +} + struct TypesEncoder { public: + TypesEncoder(): compatFlags(kj::heapArray(0)) {} TypesEncoder(kj::String compatDate, kj::Array compatFlags): compatDate(kj::mv(compatDate)), compatFlags(kj::mv(compatFlags)) {} kj::Array encode() { capnp::MallocMessageBuilder flagsMessage; - CompatibilityFlags::Reader flags = compileFlags(flagsMessage, compatDate, false, compatFlags); - + CompatibilityFlags::Reader flags; + KJ_IF_SOME(date, compatDate) { + flags = compileFlags(flagsMessage, date, true, compatFlags); + } else { + flags = compileAllFlags(flagsMessage); + } capnp::MallocMessageBuilder message; auto root = message.initRoot(); @@ -221,7 +261,7 @@ private: KJ_ASSERT(structureIndex == structuresSize); } - kj::String compatDate; + kj::Maybe compatDate; kj::Array compatFlags; unsigned int groupsIndex = 0; @@ -235,4 +275,9 @@ kj::Array RTTIModule::exportTypes(kj::String compatDate, kj::Array RTTIModule::exportExperimentalTypes() { + TypesEncoder encoder; + return encoder.encode(); +} + } // namespace workerd::api diff --git a/src/workerd/api/rtti.h b/src/workerd/api/rtti.h index deaf52f233b..5b3e8ba934c 100644 --- a/src/workerd/api/rtti.h +++ b/src/workerd/api/rtti.h @@ -20,9 +20,11 @@ class RTTIModule final: public jsg::Object { RTTIModule(jsg::Lock&, const jsg::Url&) {} kj::Array exportTypes(kj::String compatDate, kj::Array compatFlags); + kj::Array exportExperimentalTypes(); JSG_RESOURCE_TYPE(RTTIModule) { JSG_METHOD(exportTypes); + JSG_METHOD(exportExperimentalTypes); } }; diff --git a/src/workerd/tools/BUILD.bazel b/src/workerd/tools/BUILD.bazel index 9fe370c5cb0..ffcdf812e98 100644 --- a/src/workerd/tools/BUILD.bazel +++ b/src/workerd/tools/BUILD.bazel @@ -5,133 +5,26 @@ load("//:build/run_binary_target.bzl", "run_binary_target") load("//:build/wd_cc_binary.bzl", "wd_cc_binary") load("//:build/wd_cc_library.bzl", "wd_cc_library") -# ======================================================================================== -# C++ deps for API extraction -# -# Both `api_encoder` and `param_extractor` need access to the API surface -# of `workerd`, so this target allows them to both have the same deps - -wd_cc_library( - name = "api_encoder_lib", - deps = [ - "//src/workerd/api:html-rewriter", - "//src/workerd/io", - "//src/workerd/jsg", - "//src/workerd/jsg:rtti", - "@capnp-cpp//src/capnp", - ], - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), -) - -api_encoder_src = "api-encoder.c++" - -# ======================================================================================== -# API Encoder -# -# Encodes runtime API type information into a capnp schema - -wd_cc_binary( - name = "api_encoder_bin", - srcs = [api_encoder_src], - deps = [":api_encoder_lib"], - # Use dynamic linkage where possible to reduce binary size – unlike the workerd binary, we - # shouldn't need to distribute the api encoder. - linkstatic = 0, - # The dependent targets are not Windows-compatible, no need to compile this. - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), -) - -# All compatibility dates that changed public facing types. -# Remember to update `npm/workers-types/README.md` when adding new dates here. -compat_dates = [ - # Oldest compatibility date, with no flags enabled - ("2021-01-01", "oldest"), - # https://developers.cloudflare.com/workers/platform/compatibility-dates/#formdata-parsing-supports-file - ("2021-11-03", "2021-11-03"), - # https://developers.cloudflare.com/workers/platform/compatibility-dates/#settersgetters-on-api-object-prototypes - ("2022-01-31", "2022-01-31"), - # https://developers.cloudflare.com/workers/platform/compatibility-dates/#global-navigator - ("2022-03-21", "2022-03-21"), - # https://developers.cloudflare.com/workers/platform/compatibility-dates/#r2-bucket-list-respects-the-include-option - ("2022-08-04", "2022-08-04"), - # https://developers.cloudflare.com/workers/platform/compatibility-dates/#new-url-parser-implementation - ("2022-10-31", "2022-10-31"), - # https://developers.cloudflare.com/workers/platform/compatibility-dates/#streams-constructors - # https://developers.cloudflare.com/workers/platform/compatibility-dates/#compliant-transformstream-constructor - ("2022-11-30", "2022-11-30"), - # https://github.com/cloudflare/workerd/blob/fcb6f33d10c71975cb2ce68dbf1924a1eeadbd8a/src/workerd/io/compatibility-date.capnp#L275-L280 (http_headers_getsetcookie) - ("2023-03-01", "2023-03-01"), - # https://github.com/cloudflare/workerd/blob/fcb6f33d10c71975cb2ce68dbf1924a1eeadbd8a/src/workerd/io/compatibility-date.capnp#L307-L312 (urlsearchparams_delete_has_value_arg) - ("2023-07-01", "2023-07-01"), - # Latest compatibility date (note these types should be the same as the previous entry) - (None, "experimental"), -] - -filegroup( - name = "api_encoder", - srcs = [ - "//src/workerd/tools:api_encoder_" + label - for (date, label) in compat_dates - ], - tags = ["no-arm64"], - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], -) - -[ - run_binary_target( - name = "api_encoder_" + label, - outs = [label + ".api.capnp.bin"], - args = [ - "--output", - "$(location " + label + ".api.capnp.bin)", - ] + ([ - "--compatibility-date", - date, - ] if date else []), - # Cross-compiling is not supported as this runs in target cfg. - tags = ["no-arm64"], - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), - tool = "api_encoder_bin", - visibility = ["//visibility:public"], - ) - for (date, label) in compat_dates -] - # ======================================================================================== # Parameter Name Extractor # # Extracts the parameter names of functions, methods, etc. of the runtime API, -# since they're not encoded in the type information generated by `api_encoder` - -cc_library( - name = "compile_api_headers_only", - defines = ["API_ENCODER_HDRS_ONLY=1"], -) +# since they're not encoded in the type information generated by the RTTI dump cc_ast_dump( name = "dump_api_ast", - src = api_encoder_src, + src = "param-names-ast.c++", out = "api.ast.json.gz", target_compatible_with = select({ "@platforms//os:windows": ["@platforms//:incompatible"], "//conditions:default": [], }), deps = [ - ":api_encoder_lib", - ":compile_api_headers_only", + "//src/workerd/api:html-rewriter", + "//src/workerd/io", + "//src/workerd/jsg", + "//src/workerd/jsg:rtti", + "@capnp-cpp//src/capnp", ], ) diff --git a/src/workerd/tools/api-encoder.c++ b/src/workerd/tools/api-encoder.c++ deleted file mode 100644 index 37eaab451de..00000000000 --- a/src/workerd/tools/api-encoder.c++ +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright (c) 2017-2022 Cloudflare, Inc. -// Licensed under the Apache 2.0 license found in the LICENSE file or at: -// https://opensource.org/licenses/Apache-2.0 - -// Encodes JSG RTTI for all APIs defined in `src/workerd/api` to a capnp binary -// for consumption by other tools (e.g. TypeScript type generation). - -// When creating type definitions, only include the API headers to reduce the clang AST dump size. -#if !API_ENCODER_HDRS_ONLY -#include -#include -#include -#include -#include -#include -#endif // !API_ENCODER_HDRS_ONLY - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef WORKERD_EXPERIMENTAL_ENABLE_WEBGPU -#include -#else -#define EW_WEBGPU_ISOLATE_TYPES -#endif - -#if !API_ENCODER_HDRS_ONLY - -#define EW_TYPE_GROUP_FOR_EACH(F) \ - F("dom-exception", jsg::DOMException) \ - F("global-scope", EW_GLOBAL_SCOPE_ISOLATE_TYPES) \ - F("durable-objects", EW_ACTOR_ISOLATE_TYPES) \ - F("durable-objects-state", EW_ACTOR_STATE_ISOLATE_TYPES) \ - F("analytics-engine", EW_ANALYTICS_ENGINE_ISOLATE_TYPES) \ - F("basics", EW_BASICS_ISOLATE_TYPES) \ - F("blob", EW_BLOB_ISOLATE_TYPES) \ - F("cache", EW_CACHE_ISOLATE_TYPES) \ - F("crypto", EW_CRYPTO_ISOLATE_TYPES) \ - F("encoding", EW_ENCODING_ISOLATE_TYPES) \ - F("events", EW_EVENTS_ISOLATE_TYPES) \ - F("form-data", EW_FORMDATA_ISOLATE_TYPES) \ - F("html-rewriter", EW_HTML_REWRITER_ISOLATE_TYPES) \ - F("http", EW_HTTP_ISOLATE_TYPES) \ - F("hyperdrive", EW_HYPERDRIVE_ISOLATE_TYPES) \ - F("kv", EW_KV_ISOLATE_TYPES) \ - F("queue", EW_QUEUE_ISOLATE_TYPES) \ - F("r2-admin", EW_R2_PUBLIC_BETA_ADMIN_ISOLATE_TYPES) \ - F("r2", EW_R2_PUBLIC_BETA_ISOLATE_TYPES) \ - F("worker-rpc", EW_WORKER_RPC_ISOLATE_TYPES) \ - F("scheduled", EW_SCHEDULED_ISOLATE_TYPES) \ - F("streams", EW_STREAMS_ISOLATE_TYPES) \ - F("trace", EW_TRACE_ISOLATE_TYPES) \ - F("url", EW_URL_ISOLATE_TYPES) \ - F("url-standard", EW_URL_STANDARD_ISOLATE_TYPES) \ - F("url-pattern", EW_URLPATTERN_ISOLATE_TYPES) \ - F("websocket", EW_WEBSOCKET_ISOLATE_TYPES) \ - F("sql", EW_SQL_ISOLATE_TYPES) \ - F("sockets", EW_SOCKETS_ISOLATE_TYPES) \ - F("node", EW_NODE_ISOLATE_TYPES) \ - F("webgpu", EW_WEBGPU_ISOLATE_TYPES) \ - F("eventsource", EW_EVENTSOURCE_ISOLATE_TYPES) - -namespace workerd::api { -namespace { - -using namespace jsg; - -struct ApiEncoderMain { - explicit ApiEncoderMain(kj::ProcessContext &context) : context(context) {} - - kj::MainFunc getMain() { - return kj::MainBuilder(context, "", "API Encoder") - .addOptionWithArg({"o", "output"}, KJ_BIND_METHOD(*this, setOutput), - "", "Output to ") - .addOptionWithArg( - {"c", "compatibility-date"}, - KJ_BIND_METHOD(*this, setCompatibilityDate), "", - "Set the compatibility date of the generated types to ") - .callAfterParsing(KJ_BIND_METHOD(*this, run)) - .build(); - } - - kj::MainBuilder::Validity setOutput(kj::StringPtr value) { - output = value; - return true; - } - - kj::MainBuilder::Validity setCompatibilityDate(kj::StringPtr value) { - compatibilityDate = value; - return true; - } - - CompatibilityFlags::Reader - compileFlags(capnp::MessageBuilder &message, kj::StringPtr compatDate, bool experimental, - kj::ArrayPtr compatFlags) { - // Based on src/workerd/io/compatibility-date-test.c++ - auto orphanage = message.getOrphanage(); - auto flagListOrphan = - orphanage.newOrphan>(compatFlags.size()); - auto flagList = flagListOrphan.get(); - for (auto i : kj::indices(compatFlags)) { - flagList.set(i, compatFlags.begin()[i]); - } - - auto output = message.initRoot(); - SimpleWorkerErrorReporter errorReporter; - - compileCompatibilityFlags(compatDate, flagList.asReader(), output, - errorReporter, experimental, - CompatibilityDateValidation::FUTURE_FOR_TEST); - - if (!errorReporter.errors.empty()) { - KJ_FAIL_ASSERT(kj::strArray(errorReporter.errors, "\n")); - } - - auto reader = output.asReader(); - return kj::mv(reader); - } - - void compileAllCompatibilityFlags(CompatibilityFlags::Builder output) { - - auto schema = capnp::Schema::from(); - auto dynamicOutput = capnp::toDynamic(output); - - for (auto field: schema.getFields()) { - bool isNode = false; - - kj::StringPtr enableFlagName; - - for (auto annotation: field.getProto().getAnnotations()) { - if (annotation.getId() == COMPAT_ENABLE_FLAG_ANNOTATION_ID) { - enableFlagName = annotation.getValue().getText(); - // Exclude nodejs_compat, since the type generation scripts don't support node:* imports - // TODO: Figure out typing for node compat - isNode = enableFlagName == "nodejs_compat" || - enableFlagName == "nodejs_compat_v2"; - } - } - - dynamicOutput.set(field, !isNode); - } - } - - CompatibilityFlags::Reader compileAllFlags(capnp::MessageBuilder &message) { - - auto output = message.initRoot(); - - compileAllCompatibilityFlags(output); - - auto reader = output.asReader(); - return kj::mv(reader); - } - - bool run() { - // Create RTTI builder with either: - // * All (non-experimental) compatibility flags as of a specific compatibility date - // (if one is specified) - // * All (including experimental, but excluding nodejs_compat) compatibility flags - // (if no compatibility date is provided) - - capnp::MallocMessageBuilder flagsMessage; - CompatibilityFlags::Reader flags; - KJ_IF_SOME (date, compatibilityDate) { - flags = compileFlags(flagsMessage, date, false, {}); - } else { - flags = compileAllFlags(flagsMessage); - } - auto builder = rtti::Builder(flags); - - // Build structure groups - capnp::MallocMessageBuilder message; - auto root = message.initRoot(); - -#define EW_TYPE_GROUP_COUNT(Name, Types) groupsSize++; -#define EW_TYPE_GROUP_WRITE(Name, Types) \ - writeGroup(groups, builder, Name); - - unsigned int groupsSize = 0; - EW_TYPE_GROUP_FOR_EACH(EW_TYPE_GROUP_COUNT) - auto groups = root.initGroups(groupsSize); - groupsIndex = 0; - EW_TYPE_GROUP_FOR_EACH(EW_TYPE_GROUP_WRITE) - KJ_ASSERT(groupsIndex == groupsSize); - -#undef EW_TYPE_GROUP_COUNT -#undef EW_TYPE_GROUP_WRITE - - // Write structure groups to a file or stdout if none specified - KJ_IF_SOME (value, output) { - auto fs = kj::newDiskFilesystem(); - auto path = kj::Path::parse(value); - auto writeMode = kj::WriteMode::CREATE | kj::WriteMode::MODIFY | - kj::WriteMode::CREATE_PARENT; - auto file = fs->getCurrent().openFile(path, writeMode); - auto words = capnp::messageToFlatArray(message); - auto bytes = words.asBytes(); - file->writeAll(bytes); - } else { - capnp::writeMessageToFd(1 /* stdout */, message); - } - - return true; - } - - template - void writeStructure(rtti::Builder &builder, - capnp::List::Builder structures) { - auto reader = builder.structure(); - structures.setWithCaveats(structureIndex++, reader); - } - - template - void writeGroup( - capnp::List::Builder &groups, - rtti::Builder &builder, kj::StringPtr name) { - auto group = groups[groupsIndex++]; - group.setName(name); - - unsigned int structuresSize = sizeof...(Types); - auto structures = group.initStructures(structuresSize); - structureIndex = 0; - (writeStructure(builder, structures), ...); - KJ_ASSERT(structureIndex == structuresSize); - } - -private: - kj::ProcessContext &context; - kj::Maybe output; - kj::Maybe compatibilityDate; - - unsigned int groupsIndex = 0; - unsigned int structureIndex = 0; -}; - -} // namespace -} // namespace workerd::api - -KJ_MAIN(workerd::api::ApiEncoderMain); - -#endif // !API_ENCODER_HDRS_ONLY - diff --git a/src/workerd/tools/param-names-ast.c++ b/src/workerd/tools/param-names-ast.c++ new file mode 100644 index 00000000000..d3d628ae1f9 --- /dev/null +++ b/src/workerd/tools/param-names-ast.c++ @@ -0,0 +1,28 @@ +// Copyright (c) 2017-2022 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// This file includes all Worker APIs, and is used to generate a Clang AST dump for later +// lookup of parameter names for inclusion in the TS types (since RTTI doesn't include this information) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/types/BUILD.bazel b/types/BUILD.bazel index ea99c9ebbb0..53832be6bc9 100644 --- a/types/BUILD.bazel +++ b/types/BUILD.bazel @@ -5,9 +5,10 @@ load("//:build/wd_ts_type_test.bzl", "wd_ts_type_test") wd_ts_project( name = "types_lib", - srcs = glob(["src/**/*"]), + srcs = glob(["src/**/*", "scripts/*.ts"], exclude = ["src/worker/**/*"]), deps = [ "//:node_modules/@types", + "//:node_modules/esbuild", "//:node_modules/@workerd/jsg", "//:node_modules/capnp-ts", "//:node_modules/prettier", @@ -16,32 +17,53 @@ wd_ts_project( ) js_binary( - name = "types_bin", + name = "build_types_bin", data = [ ":types_lib", ], - entry_point = "src/index.js", - node_options = ["--enable-source-maps"], + entry_point = "scripts/build-types.js", ) js_run_binary( name = "types", srcs = [ - "//src/workerd/tools:api_encoder", - "//src/workerd/tools:param_extractor", - ] + glob(["defines/**/*.ts"]), - args = [ - "--input-dir", - "src/workerd/tools", - "--defines-dir", - "types/defines", - "--output-dir", - "types/definitions", - "--format", + "scripts/config.capnp", + ":types_worker", + "//src/workerd/server:workerd", + "//:node_modules/prettier", + ], out_dirs = ["definitions"], silent_on_success = False, # Always enable logging for debugging - tool = ":types_bin", + tool = ":build_types_bin", +) + +js_binary( + name = "build_worker_bin", + data = [ + ":types_lib", + ], + entry_point = "scripts/build-worker.js", + node_options = ["--enable-source-maps"], +) + +js_run_binary( + name = "types_worker", + srcs = [ + "//:node_modules/esbuild", + "//:node_modules/@workerd/jsg", + "//:node_modules/capnp-ts", + "//:node_modules/typescript", + "//src/workerd/tools:param_extractor", + ] + glob( + [ + "src/**/*.ts", + "defines/**/*.ts", + ], + exclude = ["src/cli/**/*"], + ), + outs = ["dist/index.mjs"], + tool = ":build_worker_bin", ) [ diff --git a/types/scripts/build-types.ts b/types/scripts/build-types.ts new file mode 100644 index 00000000000..72610297725 --- /dev/null +++ b/types/scripts/build-types.ts @@ -0,0 +1,162 @@ +import assert from "node:assert"; +import childProcess from "node:child_process"; +import events from "node:events"; +import { readFileSync, readdirSync } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import prettier from "prettier"; +import ts from "typescript"; +import { SourcesMap, createMemoryProgram } from "../src/program.js"; + +const OUTPUT_PATH = "types/definitions"; +const ENTRYPOINTS = [ + { compatDate: "2021-01-01", name: "oldest" }, + // https://developers.cloudflare.com/workers/platform/compatibility-dates/#formdata-parsing-supports-file + { compatDate: "2021-11-03" }, + // https://developers.cloudflare.com/workers/platform/compatibility-dates/#settersgetters-on-api-object-prototypes + { compatDate: "2022-01-31" }, + // https://developers.cloudflare.com/workers/platform/compatibility-dates/#global-navigator + { compatDate: "2022-03-21" }, + // https://developers.cloudflare.com/workers/platform/compatibility-dates/#r2-bucket-list-respects-the-include-option + { compatDate: "2022-08-04" }, + // https://developers.cloudflare.com/workers/platform/compatibility-dates/#new-url-parser-implementation + { compatDate: "2022-10-31" }, + // https://developers.cloudflare.com/workers/platform/compatibility-dates/#streams-constructors + // https://developers.cloudflare.com/workers/platform/compatibility-dates/#compliant-transformstream-constructor + { compatDate: "2022-11-30" }, + // https://github.com/cloudflare/workerd/blob/fcb6f33d10c71975cb2ce68dbf1924a1eeadbd8a/src/workerd/io/compatibility-date.capnp#L275-L280 (http_headers_getsetcookie) + { compatDate: "2023-03-01" }, + // https://github.com/cloudflare/workerd/blob/fcb6f33d10c71975cb2ce68dbf1924a1eeadbd8a/src/workerd/io/compatibility-date.capnp#L307-L312 (urlsearchparams_delete_has_value_arg) + { compatDate: "2023-07-01" }, + // Latest compatibility date with experimental features + { compatDate: "experimental" }, +]; + +/** + * Copy all TS lib files into the memory filesystem. We only use lib.esnext + * but since TS lib files all reference each other to various extents it's + * easier to add them all and let TS figure out which ones it actually needs to load. + * This function uses the current local installation of TS as a source for lib files + */ +function loadLibFiles(): SourcesMap { + const libLocation = path.dirname(require.resolve("typescript")); + const libFiles = readdirSync(libLocation).filter( + (file) => file.startsWith("lib.") && file.endsWith(".d.ts") + ); + const lib: SourcesMap = new Map(); + for (const file of libFiles) { + lib.set( + `/node_modules/typescript/lib/${file}`, + readFileSync(path.join(libLocation, file), "utf-8") + ); + } + return lib; +} + +function checkDiagnostics(sources: SourcesMap) { + const program = createMemoryProgram( + sources, + undefined, + { + noEmit: true, + lib: ["lib.esnext.d.ts"], + types: [], + }, + loadLibFiles() + ); + + const emitResult = program.emit(); + + const allDiagnostics = ts + .getPreEmitDiagnostics(program) + .concat(emitResult.diagnostics); + + allDiagnostics.forEach((diagnostic) => { + if (diagnostic.file) { + const { line, character } = ts.getLineAndCharacterOfPosition( + diagnostic.file, + diagnostic.start! + ); + const message = ts.flattenDiagnosticMessageText( + diagnostic.messageText, + "\n" + ); + console.log( + `${diagnostic.file.fileName}:${line + 1}:${character + 1} : ${message}` + ); + } else { + console.log( + ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n") + ); + } + }); + + assert(allDiagnostics.length === 0, "TypeScript failed to compile!"); +} + +function spawnWorkerd( + configPath: string +): Promise<{ url: URL; kill: () => Promise }> { + return new Promise((resolve) => { + const workerdProcess = childProcess.spawn( + "./src/workerd/server/workerd", + ["serve", "--verbose", "--experimental", "--control-fd=3", configPath], + { stdio: ["inherit", "inherit", "inherit", "pipe"] } + ); + const exitPromise = events.once(workerdProcess, "exit"); + workerdProcess.stdio?.[3]?.on("data", (chunk) => { + const message = JSON.parse(chunk.toString().trim()); + assert.strictEqual(message.event, "listen"); + resolve({ + url: new URL(`http://127.0.0.1:${message.port}`), + async kill() { + workerdProcess.kill("SIGINT"); + await exitPromise; + }, + }); + }); + }); +} + +async function buildEntrypoint( + entrypoint: (typeof ENTRYPOINTS)[number], + workerUrl: URL +) { + const url = new URL(`/${entrypoint.compatDate}.bundle`, workerUrl); + const response = await fetch(url); + if (!response.ok) throw new Error(await response.text()); + const bundle = await response.formData(); + + const name = entrypoint.name ?? entrypoint.compatDate; + const entrypointPath = path.join(OUTPUT_PATH, name); + await fs.mkdir(entrypointPath, { recursive: true }); + for (const [name, definitions] of bundle) { + assert(typeof definitions === "string"); + const prettierIgnoreRegexp = /^\s*\/\/\s*prettier-ignore\s*\n/gm; + let typings = definitions.replaceAll(prettierIgnoreRegexp, ""); + + typings = await prettier.format(typings, { + parser: "typescript", + }); + + checkDiagnostics(new SourcesMap([["/$virtual/source.ts", typings]])); + + await fs.writeFile(path.join(entrypointPath, name), typings); + } +} + +async function buildAllEntrypoints(workerUrl: URL) { + for (const entrypoint of ENTRYPOINTS) + await buildEntrypoint(entrypoint, workerUrl); +} +export async function main() { + const worker = await spawnWorkerd("./types/scripts/config.capnp"); + try { + await buildAllEntrypoints(worker.url); + } finally { + await worker.kill(); + } +} + +// Outputting to a CommonJS module so can't use top-level await +if (require.main === module) void main(); diff --git a/types/scripts/build-worker.ts b/types/scripts/build-worker.ts new file mode 100644 index 00000000000..8ec8a9cd7e3 --- /dev/null +++ b/types/scripts/build-worker.ts @@ -0,0 +1,145 @@ +import assert from "node:assert"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { build } from "esbuild"; +import { CommentsData } from "src/transforms"; +import cloudflareComments from "../src/cloudflare"; +import { collateStandardComments } from "../src/standards"; + +async function readPath(rootPath: string): Promise { + try { + return await fs.readFile(rootPath, "utf8"); + } catch (e) { + if (!(e && typeof e === "object" && "code" in e && e.code === "EISDIR")) + throw e; + const fileNames = await fs.readdir(rootPath); + const contentsPromises = fileNames.map((fileName) => { + const filePath = path.join(rootPath, fileName); + return readPath(filePath); + }); + const contents = await Promise.all(contentsPromises); + return contents.join("\n"); + } +} + +async function readParamNames() { + // Support methods defined in parent classes + const additionalClassNames: [string, string][] = [ + ["DurableObjectStorageOperations", "DurableObjectStorage"], + ["DurableObjectStorageOperations", "DurableObjectTransaction"], + ]; + + const data = await fs.readFile("src/workerd/tools/param-names.json", "utf8"); + const recordArray = JSON.parse(data) as { + fully_qualified_parent_name: string[]; + function_like_name: string; + index: number; + name: string; + }[]; + + function registerApi( + structureName: string, + record: (typeof recordArray)[number] + ) { + let functionName: string = record.function_like_name; + if (functionName.endsWith("_")) functionName = functionName.slice(0, -1); + // `constructor` is a reserved property name + if (functionName === "constructor") functionName = `$${functionName}`; + + result[structureName] ??= {}; + const structureRecord = result[structureName]; + + structureRecord[functionName] ??= []; + const functionArray = structureRecord[functionName]; + functionArray[record.index] = record.name; + } + + const result: Record> = {}; + for (const record of recordArray) { + const structureName: string = record.fully_qualified_parent_name + .filter(Boolean) + .join("::"); + + registerApi(structureName, record); + + for (const [className, renamedClass] of additionalClassNames) { + if (structureName.includes(className)) { + registerApi(structureName.replace(className, renamedClass), record); + } + } + } + return result; +} + +export async function readComments(): Promise { + const comments = collateStandardComments( + path.join( + path.dirname(require.resolve("typescript")), + "lib.webworker.d.ts" + ), + path.join( + path.dirname(require.resolve("typescript")), + "lib.webworker.iterable.d.ts" + ) + ); + + // We want to deep merge here so that our comment overrides can be very targeted + for (const [name, members] of Object.entries(cloudflareComments)) { + comments[name] ??= {}; + for (const [member, comment] of Object.entries(members)) { + const apiEntry = comments[name]; + assert(apiEntry !== undefined); + apiEntry[member] = comment; + } + } + return comments; +} + +if (require.main === module) + void build({ + logLevel: "info", + format: "esm", + target: "esnext", + external: ["node:*", "workerd:*"], + bundle: true, + minify: true, + outdir: "types/dist", + outExtension: { ".js": ".mjs" }, + entryPoints: ["types/src/worker/index.ts"], + plugins: [ + { + name: "raw", + setup(build) { + build.onResolve({ filter: /^raw:/ }, async (args) => { + const resolved = path.resolve(args.resolveDir, args.path.slice(4)); + return { namespace: "raw", path: resolved }; + }); + build.onLoad({ namespace: "raw", filter: /.*/ }, async (args) => { + const contents = await readPath(args.path); + return { contents, loader: "text" }; + }); + }, + }, + { + name: "virtual", + setup(build) { + build.onResolve({ filter: /^virtual:/ }, (args) => { + return { + namespace: "virtual", + path: args.path.substring("virtual:".length), + }; + }); + build.onLoad({ namespace: "virtual", filter: /.*/ }, async (args) => { + if (args.path === "param-names.json") { + const contents = await readParamNames(); + return { contents: JSON.stringify(contents), loader: "json" }; + } + if (args.path === "comments.json") { + const comments = await readComments(); + return { contents: JSON.stringify(comments), loader: "json" }; + } + }); + }, + }, + ], + }); diff --git a/types/scripts/config.capnp b/types/scripts/config.capnp new file mode 100644 index 00000000000..15f33b1afd3 --- /dev/null +++ b/types/scripts/config.capnp @@ -0,0 +1,18 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + ( name = "main", worker = .worker ), + ], + sockets = [ + ( name = "http", address = "127.0.0.1:0", http = (), service = "main" ), + ] +); + +const worker :Workerd.Worker = ( + compatibilityDate = "2024-01-01", + compatibilityFlags = ["nodejs_compat", "rtti_api"], + modules = [ + ( name = "./index.mjs", esModule = embed "../dist/index.mjs" ) + ], +); diff --git a/types/src/generator/index.ts b/types/src/generator/index.ts index fe29943dece..c4a44255d96 100644 --- a/types/src/generator/index.ts +++ b/types/src/generator/index.ts @@ -152,38 +152,6 @@ function collectClasses(map: StructureMap): Set { return classes; } -// Builds a map mapping structure names that are top-level nested types of -// module structures to the names of those modules. Essentially, a map of which -// modules export which types (e.g. "workerd::api::node::AsyncLocalStorage" => -// "node-internal:async_hooks"). We use this to make sure we don't include -// duplicate definitions if an internal module references a type from another -// internal module. In this case, we'll include the definition in the one that -// exported it. -function collectModuleTypeExports( - root: StructureGroups, - map: StructureMap -): Map { - const typeExports = new Map(); - root.getModules().forEach((module) => { - if (!module.isStructureName()) return; - - // Get module root type - const specifier = module.getSpecifier(); - const moduleRootName = module.getStructureName(); - const moduleRoot = map.get(moduleRootName); - assert(moduleRoot !== undefined); - - // Add all nested types in module root - moduleRoot.getMembers().forEach((member) => { - if (!member.isNested()) return; - const nested = member.getNested(); - typeExports.set(nested.getStructure().getFullyQualifiedName(), specifier); - }); - }); - - return typeExports; -} - export function generateDefinitions(root: StructureGroups): { nodes: ts.Statement[]; structureMap: StructureMap; @@ -206,92 +174,6 @@ export function generateDefinitions(root: StructureGroups): { }); const flatNodes = nodes.flat(); - const typeExports = collectModuleTypeExports(root, structureMap); - root.getModules().forEach((module) => { - if (!module.isStructureName()) return; - - // Get module root type - const specifier = module.getSpecifier(); - const moduleRootName = module.getStructureName(); - const moduleRoot = structureMap.get(moduleRootName); - assert(moduleRoot !== undefined); - - // Build a set of nested types exported by this module. These will always - // be included in the module, even if they're referenced globally. - const nestedTypeNames = new Set(); - moduleRoot.getMembers().forEach((member) => { - if (member.isNested()) { - const nested = member.getNested(); - nestedTypeNames.add(nested.getStructure().getFullyQualifiedName()); - } - }); - - // Add all types required by this module, but not the top level or another - // internal module. - const moduleIncluded = collectIncluded(structureMap, moduleRootName); - const statements: ts.Statement[] = []; - - let nextImportId = 1; - for (const name of moduleIncluded) { - // If this structure was already included globally, ignore it, - // unless it's explicitly declared a nested type of this module - if (globalIncluded.has(name) && !nestedTypeNames.has(name)) continue; - - // If this structure was exported by another module, import it. Note we - // don't need to check whether we've already imported the type as - // `moduleIncluded` is a `Set`. - const maybeOwningModule = typeExports.get(name); - if (maybeOwningModule !== undefined && maybeOwningModule !== specifier) { - // Internal modules only have default exports, so we generate something - // that looks like this: - // ``` - // import _internal1 from "node-internal:async_hooks"; - // import AsyncLocalStorage = _internal1.AsyncLocalStorage; // (type & value alias) - // ``` - const identifier = f.createIdentifier(`_internal${nextImportId++}`); - const importClause = f.createImportClause( - false, - /* name */ identifier, - /* namedBindings */ undefined - ); - const importDeclaration = f.createImportDeclaration( - /* modifiers */ undefined, - importClause, - f.createStringLiteral(maybeOwningModule) - ); - const typeName = getTypeName(name); - const importEqualsDeclaration = f.createImportEqualsDeclaration( - /* modifiers */ undefined, - /* isTypeOnly */ false, - typeName, - f.createQualifiedName(identifier, typeName) - ); - statements.unshift(importDeclaration, importEqualsDeclaration); - - continue; - } - - // Otherwise, just include the structure in the module - const structure = structureMap.get(name); - assert(structure !== undefined); - const asClass = classes.has(name); - const statement = createStructureNode(structure, { - asClass, - ambientContext: true, - // nameOverride: nestedNameOverrides.get(name), // TODO: remove - }); - statements.push(statement); - } - - const moduleBody = f.createModuleBlock(statements); - const moduleDeclaration = f.createModuleDeclaration( - [f.createToken(ts.SyntaxKind.DeclareKeyword)], - f.createStringLiteral(specifier), - moduleBody - ); - flatNodes.push(moduleDeclaration); - }); - return { nodes: flatNodes, structureMap }; } diff --git a/types/src/generator/parameter-names.ts b/types/src/generator/parameter-names.ts index aaf80564814..008f8577263 100644 --- a/types/src/generator/parameter-names.ts +++ b/types/src/generator/parameter-names.ts @@ -2,76 +2,27 @@ // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 -import fs from "node:fs"; +export type ParameterNamesData = Record< + /* fullyQualifiedParentName */ string, + Record | undefined +>; -let paramNameData: Map = new Map(); - -interface Parameter { - fullyQualifiedParentName: string; - methodName: string; - parameterIndex: number; - name: string; +let data: ParameterNamesData | undefined; +export function installParameterNames(newData: ParameterNamesData) { + data = newData; } -// Support methods defined in parent classes -const additionalClassNames: Record = { - DurableObjectStorageOperations: [ - "DurableObjectStorage", - "DurableObjectTransaction", - ], -}; +const reservedKeywords = ["function", "number", "string"]; + export function getParameterName( fullyQualifiedParentName: string, - methodName: string, - parameterIndex: number + functionName: string, + index: number ): string { - const path = `${fullyQualifiedParentName}.${methodName}[${parameterIndex}]`; - - const name = paramNameData.get(path)?.name; - // Some parameter names are reserved TypeScript words, which break later down the pipeline - if (name === "function" || name === "number" || name === "string") { - return `$${name}`; - } - if (name && name !== "undefined") { - return name; - } - return `param${parameterIndex}`; -} - -export function parseApiAstDump(astDumpPath: string) { - paramNameData = new Map( - JSON.parse(fs.readFileSync(astDumpPath, { encoding: "utf-8" })) - .flatMap((p: any) => { - const shouldRename = p.fully_qualified_parent_name.find( - (n: string) => !!additionalClassNames[n] - ); - const renameOptions = additionalClassNames[shouldRename] ?? []; - const methodName = p.function_like_name.endsWith("_") - ? p.function_like_name.slice(0, -1) - : p.function_like_name; - return [ - ...renameOptions.map((r) => ({ - fullyQualifiedParentName: p.fully_qualified_parent_name - .filter((n: any) => !!n) - .map((n: string) => (n === shouldRename ? r : n)) - .join("::"), - methodName, - parameterIndex: p.index, - name: p.name, - })), - { - fullyQualifiedParentName: p.fully_qualified_parent_name - .filter((n: any) => !!n) - .join("::"), - methodName, - parameterIndex: p.index, - name: p.name, - }, - ]; - }) - .map((p: Parameter) => [ - `${p.fullyQualifiedParentName}.${p.methodName}[${p.parameterIndex}]`, - p, - ]) as [string, Parameter][] - ); + // `constructor` is a reserved property name + if (functionName === "constructor") functionName = `$${functionName}`; + const name = data?.[fullyQualifiedParentName]?.[functionName]?.[index]; + if (name === undefined) return `param${index}`; + if (reservedKeywords.includes(name)) return `$${name}`; + return name; } diff --git a/types/src/index.ts b/types/src/index.ts index d73b393969f..1e8cd08038d 100644 --- a/types/src/index.ts +++ b/types/src/index.ts @@ -1,19 +1,9 @@ -#!/usr/bin/env node import assert from "node:assert"; -import { readFileSync, readdirSync } from "node:fs"; -import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; -import path from "node:path"; -import util from "node:util"; import { StructureGroups } from "@workerd/jsg/rtti.capnp.js"; -import { Message } from "capnp-ts"; -import prettier from "prettier"; import ts from "typescript"; -import cloudflareComments from "./cloudflare"; -import { collectTypeScriptModules, generateDefinitions } from "./generator"; -import { parseApiAstDump } from "./generator/parameter-names"; +import { generateDefinitions } from "./generator"; import { printNodeList, printer } from "./print"; import { SourcesMap, createMemoryProgram } from "./program"; -import { collateStandardComments } from "./standards"; import { CommentsData, compileOverridesDefines, @@ -22,7 +12,6 @@ import { createGlobalScopeTransformer, createImportResolveTransformer, createImportableTransformer, - createInternalNamespaceTransformer, createIteratorTransformer, createOverrideDefineTransformer, } from "./transforms"; @@ -45,86 +34,6 @@ and limitations under the License. // noinspection JSUnusedGlobalSymbols `; -async function* walkDir(root: string): AsyncGenerator { - const entries = await readdir(root, { withFileTypes: true }); - for (const entry of entries) { - const entryPath = path.join(root, entry.name); - if (entry.isDirectory()) yield* walkDir(entryPath); - else yield entryPath; - } -} - -async function collateExtraDefinitions(definitionsDir?: string) { - if (definitionsDir === undefined) return ""; - const files: Promise[] = []; - for await (const filePath of walkDir(path.resolve(definitionsDir))) { - files.push(readFile(filePath, "utf8")); - } - return (await Promise.all(files)).join("\n"); -} - -/** - * Copy all TS lib files into the memory filesystem. We only use lib.esnext - * but since TS lib files all reference each other to various extents it's - * easier to add them all and let TS figure out which ones it actually needs to load. - * This function uses the current local installation of TS as a source for lib files - */ -function loadLibFiles(): SourcesMap { - const libLocation = path.dirname(require.resolve("typescript")); - const libFiles = readdirSync(libLocation).filter( - (file) => file.startsWith("lib.") && file.endsWith(".d.ts") - ); - const lib: SourcesMap = new Map(); - for (const file of libFiles) { - lib.set( - `/node_modules/typescript/lib/${file}`, - readFileSync(path.join(libLocation, file), "utf-8") - ); - } - return lib; -} - -function checkDiagnostics(sources: SourcesMap) { - const program = createMemoryProgram( - sources, - undefined, - { - noEmit: true, - lib: ["lib.esnext.d.ts"], - types: [], - }, - loadLibFiles() - ); - - const emitResult = program.emit(); - - const allDiagnostics = ts - .getPreEmitDiagnostics(program) - .concat(emitResult.diagnostics); - - allDiagnostics.forEach((diagnostic) => { - if (diagnostic.file) { - const { line, character } = ts.getLineAndCharacterOfPosition( - diagnostic.file, - diagnostic.start! - ); - const message = ts.flattenDiagnosticMessageText( - diagnostic.messageText, - "\n" - ); - console.log( - `${diagnostic.file.fileName}:${line + 1}:${character + 1} : ${message}` - ); - } else { - console.log( - ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n") - ); - } - }); - - assert(allDiagnostics.length === 0, "TypeScript failed to compile!"); -} - function transform( sources: SourcesMap, sourcePath: string, @@ -142,13 +51,13 @@ function transform( return printer.printFile(result.transformed[0]); } -function printDefinitions( +export function printDefinitions( root: StructureGroups, commentData: CommentsData, extraDefinitions: string ): { ambient: string; importable: string } { // Generate TypeScript nodes from capnp request - const { nodes, structureMap } = generateDefinitions(root); + const { nodes } = generateDefinitions(root); // Assemble partial overrides and defines to valid TypeScript source files const [sources, replacements] = compileOverridesDefines(root); @@ -166,11 +75,14 @@ function printDefinitions( // Run global scope transformer after overrides so members added in // overrides are extracted createGlobalScopeTransformer(checker), - createInternalNamespaceTransformer(root, structureMap), + // TODO: enable this once we've figured out how not to expose internal modules + // createInternalNamespaceTransformer(root, structureMap), createCommentsTransformer(commentData), ]); - source += collectTypeScriptModules(root) + extraDefinitions; + // TODO: enable this once we've figured out how not to expose internal modules + // source += collectTypeScriptModules(root) + extraDefinitions; + source += extraDefinitions; // We need the type checker to respect our updated definitions after applying // overrides (e.g. to find the correct nodes when traversing heritage), so @@ -181,143 +93,15 @@ function printDefinitions( createAmbientTransformer(), ]); - checkDiagnostics(new SourcesMap([[sourcePath, source]])); - const importable = transform( new SourcesMap([[sourcePath, source]]), sourcePath, () => [createImportableTransformer()] ); - checkDiagnostics(new SourcesMap([[sourcePath, importable]])); - // Print program to string return { ambient: definitionsHeader + source, importable: definitionsHeader + importable, }; } - -// Generates TypeScript types from a binary Cap’n Proto file containing encoded -// JSG RTTI. See src/workerd/tools/api-encoder.c++ for a script that generates -// input expected by this tool. -// -// To generate types using default options, run `bazel build //types:types`. -// -// Usage: types [options] [input] -// -// Options: -// -d, --defines -// Directory containing extra TypeScript definitions, not associated with C++ -// files, to concatenate to the output -// -o, --output -// Directory to write types to, in folders based on compat date -// -f, --format -// Formats generated types with Prettier -// -// Input: -// Directory containing binary Cap’n Proto file paths, in the format