From e6c3c60f67e252fdad4fab68d13ec96b0c55934e Mon Sep 17 00:00:00 2001 From: Pete Bacon Darwin Date: Fri, 29 Apr 2022 12:09:23 +0100 Subject: [PATCH] refactor: use esbuild "file" loaders to add runtime files to the published package When esbuild builds Wrangler, it creates a new directory "wrangler-dist", which contains the bundled output, but then any `__dirname` or similar Node.js constant used in the code, is relative to this output directory. During testing, Jest does not bundle the code, and so these `__dirname` constants are now relative to the directory of the original source file. This is a refactor that ensures consistency in the access of runtime files between production and test environments, by implementing build/test time transforms. In esbuild we use a "file" loader that will not inline the content of the imported file into the bundle, but instead copy the content to the output directory, and then return the path to this file in the code. We can then use this value to load the content at runtime. Similarly, in Jest testing, we have a custom transform that will return the location of the original file (since there is no real output directory to copy the file to). The result of this change, is that we can just ensure that the paths in the source code are relative to the source file and not worry about where the generated code will be output. Further we are able to remove a bunch of files from the package that we publish to npm. --- .changeset/afraid-poets-return.md | 26 +++++++++ .../miniflare-config-stubs/.env.empty | 0 .../miniflare-config-stubs/package.empty.json | 1 - .../wrangler.empty.toml | 0 packages/wrangler/package.json | 6 +- .../wrangler/pages/functions/buildPlugin.ts | 6 +- .../wrangler/pages/functions/buildWorker.ts | 8 ++- ...{template-plugin.ts => plugin.template.ts} | 0 ...{template-worker.ts => worker.template.ts} | 0 packages/wrangler/scripts/bundle.ts | 8 +++ .../wrangler/scripts/file-loader-transform.js | 58 +++++++++++++++++++ packages/wrangler/src/bundle.ts | 9 +-- packages/wrangler/src/index.tsx | 13 ++++- .../{new-worker.js => new-worker.template.js} | 0 .../{new-worker.ts => new-worker.template.ts} | 0 ...ade.js => static-asset-facade.template.js} | 0 .../{tsconfig.json => tsconfig.template.json} | 0 packages/wrangler/tsconfig.json | 8 +-- 18 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 .changeset/afraid-poets-return.md delete mode 100644 packages/wrangler/miniflare-config-stubs/.env.empty delete mode 100644 packages/wrangler/miniflare-config-stubs/package.empty.json delete mode 100644 packages/wrangler/miniflare-config-stubs/wrangler.empty.toml rename packages/wrangler/pages/functions/{template-plugin.ts => plugin.template.ts} (100%) rename packages/wrangler/pages/functions/{template-worker.ts => worker.template.ts} (100%) create mode 100644 packages/wrangler/scripts/file-loader-transform.js rename packages/wrangler/templates/{new-worker.js => new-worker.template.js} (100%) rename packages/wrangler/templates/{new-worker.ts => new-worker.template.ts} (100%) rename packages/wrangler/templates/{static-asset-facade.js => static-asset-facade.template.js} (100%) rename packages/wrangler/templates/{tsconfig.json => tsconfig.template.json} (100%) diff --git a/.changeset/afraid-poets-return.md b/.changeset/afraid-poets-return.md new file mode 100644 index 000000000000..88d10f1c99e4 --- /dev/null +++ b/.changeset/afraid-poets-return.md @@ -0,0 +1,26 @@ +--- +"wrangler": patch +--- + +refactor: use esbuild "file" loaders to add runtime files to the published package + +When esbuild builds Wrangler, it creates a new directory "wrangler-dist", which +contains the bundled output, but then any `__dirname` or similar Node.js constant +used in the code, is relative to this output directory. + +During testing, Jest does not bundle the code, and so these `__dirname` constants +are now relative to the directory of the original source file. + +This is a refactor that ensures consistency in the access of runtime files +between production and test environments, by implementing build/test time transforms. +In esbuild we use a "file" loader that will not inline the content of the imported file +into the bundle, but instead copy the content to the output directory, and then return +the path to this file in the code. We can then use this value to load the content at +runtime. + +Similarly, in Jest testing, we have a custom transform that will return the location of the +original file (since there is no real output directory to copy the file to). + +The result of this change, is that we can just ensure that the paths in the source code are +relative to the source file and not worry about where the generated code will be output. +Further we are able to remove a bunch of files from the package that we publish to npm. diff --git a/packages/wrangler/miniflare-config-stubs/.env.empty b/packages/wrangler/miniflare-config-stubs/.env.empty deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/packages/wrangler/miniflare-config-stubs/package.empty.json b/packages/wrangler/miniflare-config-stubs/package.empty.json deleted file mode 100644 index 0967ef424bce..000000000000 --- a/packages/wrangler/miniflare-config-stubs/package.empty.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/packages/wrangler/miniflare-config-stubs/wrangler.empty.toml b/packages/wrangler/miniflare-config-stubs/wrangler.empty.toml deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/packages/wrangler/package.json b/packages/wrangler/package.json index e956ba4089f1..253bf05ab666 100644 --- a/packages/wrangler/package.json +++ b/packages/wrangler/package.json @@ -101,11 +101,8 @@ "files": [ "src", "bin", - "pages", - "miniflare-config-stubs", "miniflare-dist", "wrangler-dist", - "templates", "vendor", "import_meta_url.js" ], @@ -134,6 +131,9 @@ "miniflare/cli": "/../../node_modules/miniflare/dist/src/cli.js" }, "transform": { + "\\.template\\.[jt]s(on)?$": [ + "./scripts/file-loader-transform" + ], "^.+\\.c?(t|j)sx?$": [ "esbuild-jest", { diff --git a/packages/wrangler/pages/functions/buildPlugin.ts b/packages/wrangler/pages/functions/buildPlugin.ts index fca2b03ed1e7..cad605af2e71 100644 --- a/packages/wrangler/pages/functions/buildPlugin.ts +++ b/packages/wrangler/pages/functions/buildPlugin.ts @@ -1,6 +1,10 @@ import { resolve } from "node:path"; import { build } from "esbuild"; +// See scripts/file-loader-transform.ts to understand what is happening here. +// eslint-disable-next-line @typescript-eslint/no-var-requires +const templatePluginPath = require("./plugin.template.ts"); + type Options = { routesModule: string; outfile: string; @@ -19,7 +23,7 @@ export function buildPlugin({ onEnd = () => {}, }: Options) { return build({ - entryPoints: [resolve(__dirname, "../pages/functions/template-plugin.ts")], + entryPoints: [resolve(__dirname, templatePluginPath)], inject: [routesModule], bundle: true, format: "esm", diff --git a/packages/wrangler/pages/functions/buildWorker.ts b/packages/wrangler/pages/functions/buildWorker.ts index a362c940461e..ad784047524f 100644 --- a/packages/wrangler/pages/functions/buildWorker.ts +++ b/packages/wrangler/pages/functions/buildWorker.ts @@ -1,6 +1,10 @@ import path from "node:path"; import { build } from "esbuild"; +// See scripts/file-loader-transform.ts to understand what is happening here. +// eslint-disable-next-line @typescript-eslint/no-var-requires +const templateWorkerPath = require("./worker.template.ts"); + type Options = { routesModule: string; outfile: string; @@ -21,9 +25,7 @@ export function buildWorker({ onEnd = () => {}, }: Options) { return build({ - entryPoints: [ - path.resolve(__dirname, "../pages/functions/template-worker.ts"), - ], + entryPoints: [path.resolve(__dirname, templateWorkerPath)], inject: [routesModule], bundle: true, format: "esm", diff --git a/packages/wrangler/pages/functions/template-plugin.ts b/packages/wrangler/pages/functions/plugin.template.ts similarity index 100% rename from packages/wrangler/pages/functions/template-plugin.ts rename to packages/wrangler/pages/functions/plugin.template.ts diff --git a/packages/wrangler/pages/functions/template-worker.ts b/packages/wrangler/pages/functions/worker.template.ts similarity index 100% rename from packages/wrangler/pages/functions/template-worker.ts rename to packages/wrangler/pages/functions/worker.template.ts diff --git a/packages/wrangler/scripts/bundle.ts b/packages/wrangler/scripts/bundle.ts index 35433f89aa2d..e535b5751386 100644 --- a/packages/wrangler/scripts/bundle.ts +++ b/packages/wrangler/scripts/bundle.ts @@ -25,6 +25,14 @@ async function run() { define: { "import.meta.url": "import_meta_url", }, + // `.template.*` files are external source files that will be loaded at runtime to be included in generated Workers. + // So we mark them as type "file" so that esbuild copies them into the build directory and replaces the import + // in the code with the path to the copied file. + loader: { + ".template.js": "file", + ".template.ts": "file", + ".template.json": "file", + }, }); // custom miniflare cli diff --git a/packages/wrangler/scripts/file-loader-transform.js b/packages/wrangler/scripts/file-loader-transform.js new file mode 100644 index 000000000000..e6a2b8471723 --- /dev/null +++ b/packages/wrangler/scripts/file-loader-transform.js @@ -0,0 +1,58 @@ +/** + * A jest transform that returns the path to the resolved file as its only export. + * + * This transform is a partner to the esbuild "file" loader, which does the same at build time. + * We use the "file" loader when we have a asset that we wish to load at runtime. + * The loader will ensure that the file is copied into the distributable directory and then the + * path to this new file is returned from the `import` (or `require()` call). + * + * Since this esbuild "file" loader does not work in the esbuild-jest transform (since it runs in memory), + * this transform is used to ensure that the correct path is returned as the contents of the imported/required + * file, so that the behaviour works as expected in Jest tests. + * + * The Jest config, in package.json, has a section like the following: + * + * ```json + * "transform": { + * "\\.template\\.[jt]s(on)?$": [ + * "./scripts/file-loader-transform" + * ], + * "^.+\\.c?(t|j)sx?$": [ + * "esbuild-jest", + * { + * "sourcemap": true + * } + * ] + * }, + * ``` + * + * And the esbuild config, in scripts/build.ts has a section like the following: + * + * ```ts + * loader: { + * ".template.js": "file", + * ".template.ts": "file", + * ".template.json": "file", + * }, + * ``` + * + * To use such a runtime file in the source code add a `require()` statement that will return the path to the file. + * The path passed to the `require()` call should be relative to source file containing the call. + * The return value of the `require()` call will be the path to the runtime file. + * For example: + * + * ```ts + * // eslint-disable-next-line @typescript-eslint/no-var-requires + * const templatePluginPath = require("./plugin.template.ts"); + * ``` + */ +module.exports = { + process(_src, filename) { + if (filename.endsWith(".json")) { + // Jest tries to parse JSON files so we cannot return a JS module. + return `${JSON.stringify(filename)}`; + } else { + return `module.exports = ${JSON.stringify(filename)};`; + } + }, +}; diff --git a/packages/wrangler/src/bundle.ts b/packages/wrangler/src/bundle.ts index e815dd9451ac..ea4e11944128 100644 --- a/packages/wrangler/src/bundle.ts +++ b/packages/wrangler/src/bundle.ts @@ -7,6 +7,10 @@ import type { Config } from "./config"; import type { Entry } from "./entry"; import type { CfModule } from "./worker"; +// See scripts/file-loader-transform.ts to understand what is happening here. +// eslint-disable-next-line @typescript-eslint/no-var-requires +const staticAssetFacadePath = require("../templates/static-asset-facade.template.js"); + type BundleResult = { modules: CfModule[]; resolvedEntryPointPath: string; @@ -132,10 +136,7 @@ function getEntryPoint( return { stdin: { contents: fs - .readFileSync( - path.join(__dirname, "../templates/static-asset-facade.js"), - "utf8" - ) + .readFileSync(path.join(__dirname, staticAssetFacadePath), "utf8") .replace("__ENTRY_POINT__", entryFile), sourcefile: "static-asset-facade.js", resolveDir: path.dirname(entryFile), diff --git a/packages/wrangler/src/index.tsx b/packages/wrangler/src/index.tsx index ce94387f55e0..137c4c7ef227 100644 --- a/packages/wrangler/src/index.tsx +++ b/packages/wrangler/src/index.tsx @@ -67,6 +67,13 @@ import type { RawData } from "ws"; import type { CommandModule } from "yargs"; import type Yargs from "yargs"; +// eslint-disable-next-line @typescript-eslint/no-var-requires +const newWorkerJSTemplatePath = require("../templates/new-worker.template.js"); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const newWorkerTSTemplatePath = require("../templates/new-worker.template.ts"); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const tsconfigTemplatePath = require("../templates/tsconfig.template.json"); + type ConfigPath = string | undefined; const resetColor = "\x1b[0m"; @@ -419,7 +426,7 @@ export async function main(argv: string[]): Promise { isTypescriptProject = true; await writeFile( path.join(creationDirectory, "./tsconfig.json"), - readFileSync(path.join(__dirname, "../templates/tsconfig.json")) + readFileSync(path.resolve(__dirname, tsconfigTemplatePath)) ); await packageManager.addDevDeps( "@cloudflare/workers-types", @@ -551,7 +558,7 @@ export async function main(argv: string[]): Promise { }); await writeFile( path.join(creationDirectory, "./src/index.ts"), - readFileSync(path.join(__dirname, "../templates/new-worker.ts")) + readFileSync(path.resolve(__dirname, newWorkerTSTemplatePath)) ); logger.log(`✨ Created src/index.ts`); @@ -575,7 +582,7 @@ export async function main(argv: string[]): Promise { }); await writeFile( path.join(path.join(creationDirectory, "./src/index.js")), - readFileSync(path.join(__dirname, "../templates/new-worker.js")) + readFileSync(path.resolve(__dirname, newWorkerJSTemplatePath)) ); logger.log(`✨ Created src/index.js`); diff --git a/packages/wrangler/templates/new-worker.js b/packages/wrangler/templates/new-worker.template.js similarity index 100% rename from packages/wrangler/templates/new-worker.js rename to packages/wrangler/templates/new-worker.template.js diff --git a/packages/wrangler/templates/new-worker.ts b/packages/wrangler/templates/new-worker.template.ts similarity index 100% rename from packages/wrangler/templates/new-worker.ts rename to packages/wrangler/templates/new-worker.template.ts diff --git a/packages/wrangler/templates/static-asset-facade.js b/packages/wrangler/templates/static-asset-facade.template.js similarity index 100% rename from packages/wrangler/templates/static-asset-facade.js rename to packages/wrangler/templates/static-asset-facade.template.js diff --git a/packages/wrangler/templates/tsconfig.json b/packages/wrangler/templates/tsconfig.template.json similarity index 100% rename from packages/wrangler/templates/tsconfig.json rename to packages/wrangler/templates/tsconfig.template.json diff --git a/packages/wrangler/tsconfig.json b/packages/wrangler/tsconfig.json index bfcac2ec72b8..8437ae0d5636 100644 --- a/packages/wrangler/tsconfig.json +++ b/packages/wrangler/tsconfig.json @@ -6,11 +6,5 @@ "resolveJsonModule": true, "jsx": "react" }, - "exclude": [ - "vendor", - "*-dist", - "pages/functions/template-worker.ts", - "pages/functions/template-plugin.ts", - "templates" - ] + "exclude": ["vendor", "*-dist", "**/*.template.ts"] }