refactor: use esbuild "file" loaders to add runtime files to the published assets

.changeset/afraid-poets-return.md

@@ -0,0 +1,26 @@
+---
+"wrangler": patch
+---
+
+refactor: use esbuild "file" loaders to add runtime files to the published package
+
+When esbuild builds Wrangler, it creates a new directory "wrangler-dist", which
+contains the bundled output, but then any `__dirname` or similar Node.js constant
+used in the code, is relative to this output directory.
+
+During testing, Jest does not bundle the code, and so these `__dirname` constants
+are now relative to the directory of the original source file.
+
+This is a refactor that ensures consistency in the access of runtime files
+between production and test environments, by implementing build/test time transforms.
+In esbuild we use a "file" loader that will not inline the content of the imported file
+into the bundle, but instead copy the content to the output directory, and then return
+the path to this file in the code. We can then use this value to load the content at
+runtime.
+
+Similarly, in Jest testing, we have a custom transform that will return the location of the
+original file (since there is no real output directory to copy the file to).
+
+The result of this change, is that we can just ensure that the paths in the source code are
+relative to the source file and not worry about where the generated code will be output.
+Further we are able to remove a bunch of files from the package that we publish to npm.

packages/wrangler/package.json

@@ -101,11 +101,8 @@
   "files": [
     "src",
     "bin",
-    "pages",
-    "miniflare-config-stubs",
     "miniflare-dist",
     "wrangler-dist",
-    "templates",
     "vendor",
     "import_meta_url.js"
   ],
@@ -134,6 +131,9 @@
       "miniflare/cli": "<rootDir>/../../node_modules/miniflare/dist/src/cli.js"
     },
     "transform": {
+      "\\.template\\.[jt]s(on)?$": [
+        "./scripts/file-loader-transform"
+      ],
       "^.+\\.c?(t|j)sx?$": [
         "esbuild-jest",
         {

packages/wrangler/pages/functions/buildPlugin.ts

@@ -1,6 +1,10 @@
 import { resolve } from "node:path";
 import { build } from "esbuild";
 
+// See scripts/file-loader-transform.ts to understand what is happening here.
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const templatePluginPath = require("./plugin.template.ts");
+
 type Options = {
   routesModule: string;
   outfile: string;
@@ -19,7 +23,7 @@ export function buildPlugin({
   onEnd = () => {},
 }: Options) {
   return build({
-    entryPoints: [resolve(__dirname, "../pages/functions/template-plugin.ts")],
+    entryPoints: [resolve(__dirname, templatePluginPath)],
     inject: [routesModule],
     bundle: true,
     format: "esm",

packages/wrangler/pages/functions/buildWorker.ts

@@ -1,6 +1,10 @@
 import path from "node:path";
 import { build } from "esbuild";
 
+// See scripts/file-loader-transform.ts to understand what is happening here.
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const templateWorkerPath = require("./worker.template.ts");
+
 type Options = {
   routesModule: string;
   outfile: string;
@@ -21,9 +25,7 @@ export function buildWorker({
   onEnd = () => {},
 }: Options) {
   return build({
-    entryPoints: [
-      path.resolve(__dirname, "../pages/functions/template-worker.ts"),
-    ],
+    entryPoints: [path.resolve(__dirname, templateWorkerPath)],
     inject: [routesModule],
     bundle: true,
     format: "esm",

packages/wrangler/scripts/bundle.ts

@@ -25,6 +25,14 @@ async function run() {
     define: {
       "import.meta.url": "import_meta_url",
     },
+    // `.template.*` files are external source files that will be loaded at runtime to be included in generated Workers.
+    // So we mark them as type "file" so that esbuild copies them into the build directory and replaces the import
+    // in the code with the path to the copied file.
+    loader: {
+      ".template.js": "file",
+      ".template.ts": "file",
+      ".template.json": "file",
+    },
   });
 
   // custom miniflare cli

packages/wrangler/scripts/file-loader-transform.js

@@ -0,0 +1,58 @@
+/**
+ * A jest transform that returns the path to the resolved file as its only export.
+ *
+ * This transform is a partner to the esbuild "file" loader, which does the same at build time.
+ * We use the "file" loader when we have a asset that we wish to load at runtime.
+ * The loader will ensure that the file is copied into the distributable directory and then the
+ * path to this new file is returned from the `import` (or `require()` call).
+ *
+ * Since this esbuild "file" loader does not work in the esbuild-jest transform (since it runs in memory),
+ * this transform is used to ensure that the correct path is returned as the contents of the imported/required
+ * file, so that the behaviour works as expected in Jest tests.
+ *
+ * The Jest config, in package.json, has a section like the following:
+ *
+ * ```json
+ * "transform": {
+ *   "\\.template\\.[jt]s(on)?$": [
+ *     "./scripts/file-loader-transform"
+ *   ],
+ *   "^.+\\.c?(t|j)sx?$": [
+ *     "esbuild-jest",
+ *     {
+ *       "sourcemap": true
+ *     }
+ *   ]
+ * },
+ * ```
+ *
+ * And the esbuild config, in scripts/build.ts has a section like the following:
+ *
+ * ```ts
+ * loader: {
+ *   ".template.js": "file",
+ *   ".template.ts": "file",
+ *   ".template.json": "file",
+ * },
+ * ```
+ *
+ * To use such a runtime file in the source code add a `require()` statement that will return the path to the file.
+ * The path passed to the `require()` call should be relative to source file containing the call.
+ * The return value of the `require()` call will be the path to the runtime file.
+ * For example:
+ *
+ * ```ts
+ * // eslint-disable-next-line @typescript-eslint/no-var-requires
+ * const templatePluginPath = require("./plugin.template.ts");
+ * ```
+ */
+module.exports = {
+  process(_src, filename) {
+    if (filename.endsWith(".json")) {
+      // Jest tries to parse JSON files so we cannot return a JS module.
+      return `${JSON.stringify(filename)}`;
+    } else {
+      return `module.exports = ${JSON.stringify(filename)};`;
+    }
+  },
+};

packages/wrangler/src/bundle.ts

@@ -7,6 +7,10 @@ import type { Config } from "./config";
 import type { Entry } from "./entry";
 import type { CfModule } from "./worker";
 
+// See scripts/file-loader-transform.ts to understand what is happening here.
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const staticAssetFacadePath = require("../templates/static-asset-facade.template.js");
+
 type BundleResult = {
   modules: CfModule[];
   resolvedEntryPointPath: string;
@@ -132,10 +136,7 @@ function getEntryPoint(
     return {
       stdin: {
         contents: fs
-          .readFileSync(
-            path.join(__dirname, "../templates/static-asset-facade.js"),
-            "utf8"
-          )
+          .readFileSync(path.join(__dirname, staticAssetFacadePath), "utf8")
           .replace("__ENTRY_POINT__", entryFile),
         sourcefile: "static-asset-facade.js",
         resolveDir: path.dirname(entryFile),

packages/wrangler/src/index.tsx

@@ -67,6 +67,13 @@ import type { RawData } from "ws";
 import type { CommandModule } from "yargs";
 import type Yargs from "yargs";
 
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const newWorkerJSTemplatePath = require("../templates/new-worker.template.js");
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const newWorkerTSTemplatePath = require("../templates/new-worker.template.ts");
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const tsconfigTemplatePath = require("../templates/tsconfig.template.json");
+
 type ConfigPath = string | undefined;
 
 const resetColor = "\x1b[0m";
@@ -419,7 +426,7 @@ export async function main(argv: string[]): Promise<void> {
           isTypescriptProject = true;
           await writeFile(
             path.join(creationDirectory, "./tsconfig.json"),
-            readFileSync(path.join(__dirname, "../templates/tsconfig.json"))
+            readFileSync(path.resolve(__dirname, tsconfigTemplatePath))
           );
           await packageManager.addDevDeps(
             "@cloudflare/workers-types",
@@ -551,7 +558,7 @@ export async function main(argv: string[]): Promise<void> {
             });
             await writeFile(
               path.join(creationDirectory, "./src/index.ts"),
-              readFileSync(path.join(__dirname, "../templates/new-worker.ts"))
+              readFileSync(path.resolve(__dirname, newWorkerTSTemplatePath))
             );
 
             logger.log(`✨ Created src/index.ts`);
@@ -575,7 +582,7 @@ export async function main(argv: string[]): Promise<void> {
             });
             await writeFile(
               path.join(path.join(creationDirectory, "./src/index.js")),
-              readFileSync(path.join(__dirname, "../templates/new-worker.js"))
+              readFileSync(path.resolve(__dirname, newWorkerJSTemplatePath))
             );
 
             logger.log(`✨ Created src/index.js`);

packages/wrangler/tsconfig.json

@@ -6,11 +6,5 @@
     "resolveJsonModule": true,
     "jsx": "react"
   },
-  "exclude": [
-    "vendor",
-    "*-dist",
-    "pages/functions/template-worker.ts",
-    "pages/functions/template-plugin.ts",
-    "templates"
-  ]
+  "exclude": ["vendor", "*-dist", "**/*.template.ts"]
 }