From d1fdf6b3a98d0b77912fa679d0de97bef6caae7e Mon Sep 17 00:00:00 2001 From: MrBBot Date: Wed, 16 Aug 2023 10:26:29 +0100 Subject: [PATCH] Serve linked source maps from loopback server (#660) With cloudflare/workerd#710, `workerd` supports breakpoint debugging! Support for this in Miniflare just worked, assuming you were using a plain JavaScript worker, or you had inline source maps. `workerd` doesn't know where workers are located on disk, it just knows files' locations relative to each other. This means it's unable to resolve locations of corresponding linked `.map` files in `sourceMappingURL` comments. Miniflare _does_ have this information though. This change detects linked source maps and rewrites `sourceMappingURL` comments to `http` URLs pointing to Miniflare's loopback server. This then looks for the source map relative to the known on-disk source location. Source maps' `sourceRoot` attributes are updated to ensure correct locations are displayed in DevTools. **This enables breakpoint debugging for compiled TypeScript with linked source maps!** :tada: Closes DEVX-872 --- packages/miniflare/package.json | 1 + packages/miniflare/src/index.ts | 10 + packages/miniflare/src/plugins/core/index.ts | 16 +- .../miniflare/src/plugins/core/modules.ts | 17 +- .../miniflare/src/plugins/shared/index.ts | 3 + .../miniflare/src/plugins/shared/registry.ts | 105 ++++++++++ .../test/plugins/core/errors/index.spec.ts | 188 ++++++++++++++---- 7 files changed, 295 insertions(+), 45 deletions(-) create mode 100644 packages/miniflare/src/plugins/shared/registry.ts diff --git a/packages/miniflare/package.json b/packages/miniflare/package.json index 66f07d3cd58c..d1ea56d49ae1 100644 --- a/packages/miniflare/package.json +++ b/packages/miniflare/package.json @@ -56,6 +56,7 @@ "@types/stoppable": "^1.1.1", "@types/ws": "^8.5.3", "devalue": "^4.3.0", + "devtools-protocol": "^0.0.1182435", "semiver": "^1.1.0" }, "engines": { diff --git a/packages/miniflare/src/index.ts b/packages/miniflare/src/index.ts index 5ff84c558f8b..6906939e3403 100644 --- a/packages/miniflare/src/index.ts +++ b/packages/miniflare/src/index.ts @@ -38,6 +38,7 @@ import { QueueConsumers, QueuesError, SharedOptions, + SourceMapRegistry, WorkerOptions, getGlobalServices, maybeGetSitesManifestModule, @@ -403,6 +404,7 @@ export class Miniflare { #runtime?: Runtime; #removeRuntimeExitHook?: () => void; #runtimeEntryURL?: URL; + #sourceMapRegistry?: SourceMapRegistry; // Path to temporary directory for use as scratch space/"in-memory" Durable // Object storage. Note this may not exist, it's up to the consumers to @@ -664,6 +666,8 @@ export class Miniflare { this.#log.debug(`Error parsing response log: ${String(e)}`); } response = new Response(null, { status: 204 }); + } else if (url.pathname.startsWith(SourceMapRegistry.PATHNAME_PREFIX)) { + response = await this.#sourceMapRegistry?.get(url); } else { // TODO: check for proxying/outbound fetch header first (with plans for fetch mocking) response = await this.#handleLoopbackPlugins(request, url); @@ -771,6 +775,7 @@ export class Miniflare { sharedOpts.core.cf = await setupCf(this.#log, sharedOpts.core.cf); + const sourceMapRegistry = new SourceMapRegistry(this.#log, loopbackPort); const durableObjectClassNames = getDurableObjectClassNames(allWorkerOpts); const queueConsumers = getQueueConsumers(allWorkerOpts); const allWorkerRoutes = getWorkerRoutes(allWorkerOpts); @@ -823,6 +828,7 @@ export class Miniflare { workerIndex: i, additionalModules, tmpPath: this.#tmpPath, + sourceMapRegistry, durableObjectClassNames, queueConsumers, }; @@ -845,6 +851,10 @@ export class Miniflare { } } + // Once we've assembled the config, and are about to restart the runtime, + // update the source map registry. + this.#sourceMapRegistry = sourceMapRegistry; + return { services: Array.from(services.values()), sockets }; } diff --git a/packages/miniflare/src/plugins/core/index.ts b/packages/miniflare/src/plugins/core/index.ts index c4304c8afe96..faddaa3a99cd 100644 --- a/packages/miniflare/src/plugins/core/index.ts +++ b/packages/miniflare/src/plugins/core/index.ts @@ -30,6 +30,7 @@ import { HEADER_CF_BLOB, Plugin, SERVICE_LOOPBACK, + SourceMapRegistry, WORKER_BINDING_SERVICE_LOOPBACK, parseRoutes, } from "../shared"; @@ -320,9 +321,14 @@ export const CORE_PLUGIN: Plugin< workerIndex, durableObjectClassNames, additionalModules, + sourceMapRegistry, }) { // Define regular user worker - const workerScript = getWorkerScript(options, workerIndex); + const workerScript = getWorkerScript( + sourceMapRegistry, + options, + workerIndex + ); // Add additional modules (e.g. "__STATIC_CONTENT_MANIFEST") if any if ("modules" in workerScript) { workerScript.modules.push(...additionalModules); @@ -480,6 +486,7 @@ export function getGlobalServices({ } function getWorkerScript( + sourceMapRegistry: SourceMapRegistry, options: SourceOptions, workerIndex: number ): { serviceWorkerScript: string } | { modules: Worker_Module[] } { @@ -489,7 +496,7 @@ function getWorkerScript( ("modulesRoot" in options ? options.modulesRoot : undefined) ?? ""; return { modules: options.modules.map((module) => - convertModuleDefinition(modulesRoot, module) + convertModuleDefinition(sourceMapRegistry, modulesRoot, module) ), }; } @@ -509,7 +516,7 @@ function getWorkerScript( if (options.modules) { // If `modules` is `true`, automatically collect modules... - const locator = new ModuleLocator(options.modulesRules); + const locator = new ModuleLocator(sourceMapRegistry, options.modulesRules); // If `script` and `scriptPath` are set, resolve modules in `script` // against `scriptPath`. locator.visitEntrypoint( @@ -520,6 +527,9 @@ function getWorkerScript( } else { // ...otherwise, `modules` will either be `false` or `undefined`, so treat // `code` as a service worker + if ("scriptPath" in options && options.scriptPath !== undefined) { + code = sourceMapRegistry.register(code, options.scriptPath); + } return { serviceWorkerScript: code }; } } diff --git a/packages/miniflare/src/plugins/core/modules.ts b/packages/miniflare/src/plugins/core/modules.ts index 857852921c35..a42e53a0dff3 100644 --- a/packages/miniflare/src/plugins/core/modules.ts +++ b/packages/miniflare/src/plugins/core/modules.ts @@ -15,6 +15,7 @@ import { globsToRegExps, testRegExps, } from "../../shared"; +import { SourceMapRegistry } from "../shared"; const SUGGEST_BUNDLE = "If you're trying to import an npm package, you'll need to bundle your Worker first."; @@ -122,11 +123,13 @@ function getResolveErrorPrefix(referencingPath: string): string { } export class ModuleLocator { + readonly #sourceMapRegistry: SourceMapRegistry; readonly #compiledRules: CompiledModuleRule[]; readonly #visitedPaths = new Set(); readonly modules: Worker_Module[] = []; - constructor(rules?: ModuleRule[]) { + constructor(sourceMapRegistry: SourceMapRegistry, rules?: ModuleRule[]) { + this.#sourceMapRegistry = sourceMapRegistry; this.#compiledRules = compileModuleRules(rules); } @@ -144,6 +147,7 @@ export class ModuleLocator { #visitJavaScriptModule(code: string, modulePath: string, esModule = true) { // Register module const name = path.relative("", modulePath); + code = this.#sourceMapRegistry.register(code, modulePath); this.modules.push( esModule ? { name, esModule: code } : { name, commonJsModule: code } ); @@ -305,18 +309,23 @@ function contentsToArray(contents: string | Uint8Array): Uint8Array { return typeof contents === "string" ? encoder.encode(contents) : contents; } export function convertModuleDefinition( + sourceMapRegistry: SourceMapRegistry, modulesRoot: string, def: ModuleDefinition ): Worker_Module { // The runtime requires module identifiers to be relative paths let name = path.relative(modulesRoot, def.path); if (path.sep === "\\") name = name.replaceAll("\\", "/"); - const contents = def.contents ?? readFileSync(def.path); + let contents = def.contents ?? readFileSync(def.path); switch (def.type) { case "ESModule": - return { name, esModule: contentsToString(contents) }; + contents = contentsToString(contents); + contents = sourceMapRegistry.register(contents, def.path); + return { name, esModule: contents }; case "CommonJS": - return { name, commonJsModule: contentsToString(contents) }; + contents = contentsToString(contents); + contents = sourceMapRegistry.register(contents, def.path); + return { name, commonJsModule: contents }; case "Text": return { name, text: contentsToString(contents) }; case "Data": diff --git a/packages/miniflare/src/plugins/shared/index.ts b/packages/miniflare/src/plugins/shared/index.ts index b1f1afaefe31..0ef4baf050e6 100644 --- a/packages/miniflare/src/plugins/shared/index.ts +++ b/packages/miniflare/src/plugins/shared/index.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { Service, Worker_Binding, Worker_Module } from "../../runtime"; import { Awaitable, Log, OptionalZodTypeOf } from "../../shared"; import { GatewayConstructor } from "./gateway"; +import { SourceMapRegistry } from "./registry"; import { RouterConstructor } from "./router"; // Maps **service** names to the Durable Object class names exported by them @@ -41,6 +42,7 @@ export interface PluginServicesOptions< workerIndex: number; additionalModules: Worker_Module[]; tmpPath: string; + sourceMapRegistry: SourceMapRegistry; // ~~Leaky abstractions~~ "Plugin specific options" :) durableObjectClassNames: DurableObjectClassNames; @@ -91,5 +93,6 @@ export function namespaceEntries( export * from "./constants"; export * from "./gateway"; export * from "./range"; +export * from "./registry"; export * from "./router"; export * from "./routing"; diff --git a/packages/miniflare/src/plugins/shared/registry.ts b/packages/miniflare/src/plugins/shared/registry.ts new file mode 100644 index 000000000000..063e8867f877 --- /dev/null +++ b/packages/miniflare/src/plugins/shared/registry.ts @@ -0,0 +1,105 @@ +import crypto from "crypto"; +import fs from "fs/promises"; +import path from "path"; +import { fileURLToPath, pathToFileURL } from "url"; +import type { RawSourceMap } from "source-map"; +import { Response } from "../../http"; +import { Log } from "../../shared"; + +function maybeParseURL(url: string): URL | undefined { + if (path.isAbsolute(url)) return; + try { + return new URL(url); + } catch {} +} + +export class SourceMapRegistry { + static PATHNAME_PREFIX = "/core/source-map/"; + + constructor( + private readonly log: Log, + private readonly loopbackPort: number + ) {} + + readonly #map = new Map(); + + register(script: string, scriptPath: string): string /* newScript */ { + // Try to find the last source mapping URL in the file, if none could be + // found, return the script as is + const mappingURLIndex = script.lastIndexOf("//# sourceMappingURL="); + if (mappingURLIndex === -1) return script; + + // `pathToFileURL()` will resolve `scriptPath` relative to the current + // working directory if needed + const scriptURL = pathToFileURL(scriptPath); + + const sourceSegment = script.substring(0, mappingURLIndex); + const mappingURLSegment = script + .substring(mappingURLIndex) + .replace(/^\/\/# sourceMappingURL=(.+)/, (substring, mappingURL) => { + // If the mapping URL is already a URL (e.g. `data:`), return it as is + if (maybeParseURL(mappingURL) !== undefined) return substring; + + // Otherwise, resolve it relative to the script, and register it + const resolvedMappingURL = new URL(mappingURL, scriptURL); + const resolvedMappingPath = fileURLToPath(resolvedMappingURL); + + // We intentionally register source maps in a map to prevent arbitrary + // file access via the loopback server. + const id = crypto.randomUUID(); + this.#map.set(id, resolvedMappingPath); + mappingURL = `http://localhost:${this.loopbackPort}${SourceMapRegistry.PATHNAME_PREFIX}${id}`; + + this.log.verbose( + `Registered source map ${JSON.stringify( + resolvedMappingPath + )} at ${mappingURL}` + ); + + return `//# sourceMappingURL=${mappingURL}`; + }); + + return sourceSegment + mappingURLSegment; + } + + async get(url: URL): Promise { + // Try to get source map from registry + const id = url.pathname.substring(SourceMapRegistry.PATHNAME_PREFIX.length); + const sourceMapPath = this.#map.get(id); + if (sourceMapPath === undefined) return; + + // Try to load and parse source map from disk + let contents: string; + try { + contents = await fs.readFile(sourceMapPath, "utf8"); + } catch (e) { + this.log.warn( + `Error reading source map ${JSON.stringify(sourceMapPath)}: ${e}` + ); + return; + } + let map: RawSourceMap; + try { + map = JSON.parse(contents); + } catch (e) { + this.log.warn( + `Error parsing source map ${JSON.stringify(sourceMapPath)}: ${e}` + ); + return; + } + + // Modify the `sourceRoot` so source files get the correct paths. Note, + // `sourceMapPath` will always be an absolute path. + const sourceMapDir = path.dirname(sourceMapPath); + map.sourceRoot = + map.sourceRoot === undefined + ? sourceMapDir + : path.resolve(sourceMapDir, map.sourceRoot); + + return Response.json(map, { + // This source map will be served from the loopback server to DevTools, + // which will likely be on a different origin. + headers: { "Access-Control-Allow-Origin": "*" }, + }); + } +} diff --git a/packages/miniflare/test/plugins/core/errors/index.spec.ts b/packages/miniflare/test/plugins/core/errors/index.spec.ts index a9843f1cc515..c90cf56317b0 100644 --- a/packages/miniflare/test/plugins/core/errors/index.spec.ts +++ b/packages/miniflare/test/plugins/core/errors/index.spec.ts @@ -1,8 +1,15 @@ +import assert from "assert"; import fs from "fs/promises"; +import http from "http"; +import { AddressInfo } from "net"; import path from "path"; import test from "ava"; +import Protocol from "devtools-protocol"; import esbuild from "esbuild"; -import { Miniflare } from "miniflare"; +import { DeferredPromise, Miniflare } from "miniflare"; +import { RawSourceMap } from "source-map"; +import { fetch } from "undici"; +import NodeWebSocket from "ws"; import { escapeRegexp, useTmp } from "../../../test-shared"; const FIXTURES_PATH = path.resolve( @@ -19,6 +26,7 @@ const FIXTURES_PATH = path.resolve( const SERVICE_WORKER_ENTRY_PATH = path.join(FIXTURES_PATH, "service-worker.ts"); const MODULES_ENTRY_PATH = path.join(FIXTURES_PATH, "modules.ts"); const DEP_ENTRY_PATH = path.join(FIXTURES_PATH, "nested/dep.ts"); +const REDUCE_PATH = path.join(FIXTURES_PATH, "reduce.ts"); test("source maps workers", async (t) => { // Build fixtures @@ -40,8 +48,18 @@ test("source maps workers", async (t) => { const serviceWorkerContent = await fs.readFile(serviceWorkerPath, "utf8"); const modulesContent = await fs.readFile(modulesPath, "utf8"); - // Check service-workers source mapped + // The OS should assign random ports in sequential order, meaning + // `inspectorPort` is unlikely to be immediately chosen as a random port again + const server = http.createServer(); + const inspectorPort = await new Promise((resolve, reject) => { + server.listen(0, () => { + const port = (server.address() as AddressInfo).port; + server.close((err) => (err ? reject(err) : resolve(port))); + }); + }); + const mf = new Miniflare({ + inspectorPort, workers: [ { bindings: { MESSAGE: "unnamed" }, @@ -54,60 +72,45 @@ test("source maps workers", async (t) => { script: serviceWorkerContent, scriptPath: serviceWorkerPath, }, - ], - }); - let error = await t.throwsAsync(mf.dispatchFetch("http://localhost"), { - message: "unnamed", - }); - const serviceWorkerEntryRegexp = escapeRegexp( - `${SERVICE_WORKER_ENTRY_PATH}:6:17` - ); - t.regex(String(error?.stack), serviceWorkerEntryRegexp); - error = await t.throwsAsync(mf.dispatchFetch("http://localhost/a"), { - message: "a", - }); - t.regex(String(error?.stack), serviceWorkerEntryRegexp); - - // Check modules workers source mapped - await mf.setOptions({ - workers: [ { + name: "b", + routes: ["*/b"], modules: true, scriptPath: modulesPath, - bindings: { MESSAGE: "unnamed" }, + bindings: { MESSAGE: "b" }, }, { - name: "a", - routes: ["*/a"], - bindings: { MESSAGE: "a" }, + name: "c", + routes: ["*/c"], + bindings: { MESSAGE: "c" }, modules: true, script: modulesContent, scriptPath: modulesPath, }, { - name: "b", - routes: ["*/b"], - bindings: { MESSAGE: "b" }, + name: "d", + routes: ["*/d"], + bindings: { MESSAGE: "d" }, modules: [{ type: "ESModule", path: modulesPath }], }, { - name: "c", - routes: ["*/c"], - bindings: { MESSAGE: "c" }, + name: "e", + routes: ["*/e"], + bindings: { MESSAGE: "e" }, modules: [ { type: "ESModule", path: modulesPath, contents: modulesContent }, ], }, { - name: "d", - routes: ["*/d"], - bindings: { MESSAGE: "d" }, + name: "f", + routes: ["*/f"], + bindings: { MESSAGE: "f" }, modulesRoot: tmp, modules: [{ type: "ESModule", path: modulesPath }], }, { - name: "e", - routes: ["*/e"], + name: "g", + routes: ["*/g"], modules: [ // Check importing module with source map (e.g. Wrangler no bundle with built dependencies) { @@ -118,20 +121,37 @@ test("source maps workers", async (t) => { { type: "ESModule", path: depPath }, ], }, + { + name: "h", + // Generated with `esbuild --sourcemap=inline --sources-content=false worker.ts` + script: `"use strict"; +addEventListener("fetch", (event) => { + event.respondWith(new Response("body")); +}); +//# sourceMappingURL=data:application/json;base64,ewogICJ2ZXJzaW9uIjogMywKICAic291cmNlcyI6IFsid29ya2VyLnRzIl0sCiAgIm1hcHBpbmdzIjogIjtBQUFBLGlCQUFpQixTQUFTLENBQUMsVUFBVTtBQUNuQyxRQUFNLFlBQVksSUFBSSxTQUFTLE1BQU0sQ0FBQztBQUN4QyxDQUFDOyIsCiAgIm5hbWVzIjogW10KfQo= +`, + }, ], }); - error = await t.throwsAsync(mf.dispatchFetch("http://localhost"), { + + // Check service-workers source mapped + let error = await t.throwsAsync(mf.dispatchFetch("http://localhost"), { message: "unnamed", }); - const modulesEntryRegexp = escapeRegexp(`${MODULES_ENTRY_PATH}:5:19`); - t.regex(String(error?.stack), modulesEntryRegexp); + const serviceWorkerEntryRegexp = escapeRegexp( + `${SERVICE_WORKER_ENTRY_PATH}:6:17` + ); + t.regex(String(error?.stack), serviceWorkerEntryRegexp); error = await t.throwsAsync(mf.dispatchFetch("http://localhost/a"), { message: "a", }); - t.regex(String(error?.stack), modulesEntryRegexp); + t.regex(String(error?.stack), serviceWorkerEntryRegexp); + + // Check modules workers source mapped error = await t.throwsAsync(mf.dispatchFetch("http://localhost/b"), { message: "b", }); + const modulesEntryRegexp = escapeRegexp(`${MODULES_ENTRY_PATH}:5:19`); t.regex(String(error?.stack), modulesEntryRegexp); error = await t.throwsAsync(mf.dispatchFetch("http://localhost/c"), { message: "c", @@ -142,9 +162,101 @@ test("source maps workers", async (t) => { }); t.regex(String(error?.stack), modulesEntryRegexp); error = await t.throwsAsync(mf.dispatchFetch("http://localhost/e"), { + message: "e", + }); + t.regex(String(error?.stack), modulesEntryRegexp); + error = await t.throwsAsync(mf.dispatchFetch("http://localhost/f"), { + message: "f", + }); + t.regex(String(error?.stack), modulesEntryRegexp); + error = await t.throwsAsync(mf.dispatchFetch("http://localhost/g"), { instanceOf: TypeError, message: "Dependency error", }); const nestedRegexp = escapeRegexp(`${DEP_ENTRY_PATH}:4:17`); t.regex(String(error?.stack), nestedRegexp); + + // Check source mapping URLs rewritten + let sources = await getSources(inspectorPort, "core:user:"); + t.deepEqual(sources, [REDUCE_PATH, SERVICE_WORKER_ENTRY_PATH]); + sources = await getSources(inspectorPort, "core:user:a"); + t.deepEqual(sources, [REDUCE_PATH, SERVICE_WORKER_ENTRY_PATH]); + sources = await getSources(inspectorPort, "core:user:b"); + t.deepEqual(sources, [MODULES_ENTRY_PATH, REDUCE_PATH]); + sources = await getSources(inspectorPort, "core:user:c"); + t.deepEqual(sources, [MODULES_ENTRY_PATH, REDUCE_PATH]); + sources = await getSources(inspectorPort, "core:user:d"); + t.deepEqual(sources, [MODULES_ENTRY_PATH, REDUCE_PATH]); + sources = await getSources(inspectorPort, "core:user:e"); + t.deepEqual(sources, [MODULES_ENTRY_PATH, REDUCE_PATH]); + sources = await getSources(inspectorPort, "core:user:f"); + t.deepEqual(sources, [MODULES_ENTRY_PATH, REDUCE_PATH]); + sources = await getSources(inspectorPort, "core:user:g"); + t.deepEqual(sources, [DEP_ENTRY_PATH, REDUCE_PATH]); // (entry point script overridden) + + // Check respects map's existing `sourceRoot` + const sourceRoot = "a/b/c/d/e"; + const serviceWorkerMapPath = serviceWorkerPath + ".map"; + const serviceWorkerMap: RawSourceMap = JSON.parse( + await fs.readFile(serviceWorkerMapPath, "utf8") + ); + serviceWorkerMap.sourceRoot = sourceRoot; + await fs.writeFile(serviceWorkerMapPath, JSON.stringify(serviceWorkerMap)); + t.deepEqual(await getSources(inspectorPort, "core:user:"), [ + path.resolve(tmp, sourceRoot, path.relative(tmp, REDUCE_PATH)), + path.resolve( + tmp, + sourceRoot, + path.relative(tmp, SERVICE_WORKER_ENTRY_PATH) + ), + ]); + + // Check does nothing with URL source mapping URLs + const sourceMapURL = await getSourceMapURL(inspectorPort, "core:user:h"); + t.regex(sourceMapURL, /^data:application\/json;base64/); }); + +function getSourceMapURL( + inspectorPort: number, + serviceName: string +): Promise { + let sourceMapURL: string | undefined; + const promise = new DeferredPromise(); + const inspectorUrl = `ws://127.0.0.1:${inspectorPort}/${serviceName}`; + const ws = new NodeWebSocket(inspectorUrl); + ws.on("message", async (raw) => { + try { + const message = JSON.parse(raw.toString("utf8")); + if (message.method === "Debugger.scriptParsed") { + const params: Protocol.Debugger.ScriptParsedEvent = message.params; + if (params.sourceMapURL === undefined || params.sourceMapURL === "") { + return; + } + sourceMapURL = params.sourceMapURL; + ws.close(); + } + } catch (e) { + promise.reject(e); + } + }); + ws.on("open", () => { + ws.send(JSON.stringify({ id: 0, method: "Debugger.enable", params: {} })); + }); + ws.on("close", () => { + assert(sourceMapURL !== undefined, "Expected `sourceMapURL`"); + promise.resolve(sourceMapURL); + }); + return promise; +} + +async function getSources(inspectorPort: number, serviceName: string) { + const sourceMapURL = await getSourceMapURL(inspectorPort, serviceName); + // The loopback server will be listening on `127.0.0.1`, which + // `localhost` should resolve to, but `undici` only looks at the first + // DNS entry, which will be `::1` on Node 17+. + // noinspection JSObjectNullOrUndefined + const res = await fetch(sourceMapURL.replace("localhost", "127.0.0.1")); + const { sourceRoot, sources } = (await res.json()) as RawSourceMap; + assert(sourceRoot !== undefined); + return sources.map((source) => path.resolve(sourceRoot, source)).sort(); +}