From c843b57a0a9bb227f91b0154662215749cfc02c8 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Mon, 16 Sep 2024 13:57:35 -0700 Subject: [PATCH] Base JS runtime for builds Currently we ship dev runtimes (with things like HMR logic) along with code for loading chunks. This separates them and allows us to include a minimal runtime for builds. Test Plan: `TURBOPACK=1 TURBOPACK_BUILD=1 pnpm build` on an app with a `middleware.ts` and verified it loads when started. --- .../src/ecmascript/evaluate/chunk.rs | 2 + .../js/package.json | 3 + .../js/src/browser/runtime/base/dummy.ts | 23 + .../src/browser/runtime/base/runtime-base.ts | 511 ++++++++++++++++++ .../js/src/browser/runtime/base/tsconfig.json | 8 + .../runtime/dom/runtime-backend-dom.ts | 297 ++++++++++ .../js/src/browser/runtime/dom/tsconfig.json | 8 + .../runtime/edge/runtime-backend-edge.ts | 186 +++++++ .../js/src/browser/runtime/edge/tsconfig.json | 8 + .../js/src/nodejs/runtime.ts | 2 +- .../js/src/shared/runtime-types.d.ts | 2 +- .../js/src/shared/runtime-utils.ts | 2 + .../src/browser_runtime.rs | 40 +- 13 files changed, 1082 insertions(+), 10 deletions(-) create mode 100644 turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/dummy.ts create mode 100644 turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/runtime-base.ts create mode 100644 turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/tsconfig.json create mode 100644 turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/dom/runtime-backend-dom.ts create mode 100644 turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/dom/tsconfig.json create mode 100644 turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/edge/runtime-backend-edge.ts create mode 100644 turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/edge/tsconfig.json diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/evaluate/chunk.rs b/turbopack/crates/turbopack-browser/src/ecmascript/evaluate/chunk.rs index 57b5bc6390f8a..313e75aa028c5 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/evaluate/chunk.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/evaluate/chunk.rs @@ -147,6 +147,7 @@ impl EcmascriptDevEvaluateChunk { let runtime_code = turbopack_ecmascript_runtime::get_browser_runtime_code( environment, chunking_context.chunk_base_path(), + Value::new(chunking_context.runtime_type()), Vc::cell(output_root.to_string().into()), ); code.push_code(&*runtime_code.await?); @@ -155,6 +156,7 @@ impl EcmascriptDevEvaluateChunk { let runtime_code = turbopack_ecmascript_runtime::get_browser_runtime_code( environment, chunking_context.chunk_base_path(), + Value::new(chunking_context.runtime_type()), Vc::cell(output_root.to_string().into()), ); code.push_code(&*runtime_code.await?); diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/package.json b/turbopack/crates/turbopack-ecmascript-runtime/js/package.json index e9d1421f77411..852ad7369c267 100644 --- a/turbopack/crates/turbopack-ecmascript-runtime/js/package.json +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/package.json @@ -8,6 +8,9 @@ "check": "run-p check:*", "check:nodejs": "tsc -p src/nodejs", "check:browser-dev-client": "tsc -p src/browser/dev/hmr-client", + "check:browser-runtime-base": "tsc -p src/browser/runtime/base", + "check:browser-runtime-dom": "tsc -p src/browser/runtime/dom", + "check:browser-runtime-edge": "tsc -p src/browser/runtime/edge", "check:browser-dev-runtime-base": "tsc -p src/browser/dev/runtime/base", "check:browser-dev-runtime-dom": "tsc -p src/browser/dev/runtime/dom", "check:browser-dev-runtime-edge": "tsc -p src/browser/dev/runtime/edge" diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/dummy.ts b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/dummy.ts new file mode 100644 index 0000000000000..fd5e03937a0f3 --- /dev/null +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/dummy.ts @@ -0,0 +1,23 @@ +/** + * This file acts as a dummy implementor for the interface that + * `runtime-base.ts` expects to be available in the global scope. + * + * This interface will be implemented by runtime backends. + */ + +/* eslint-disable @typescript-eslint/no-unused-vars */ + +/// +/// + +declare var BACKEND: RuntimeBackend; +declare var loadWebAssembly: ( + source: SourceInfo, + wasmChunkPath: ChunkPath, + imports: WebAssembly.Imports +) => Exports; +declare var loadWebAssemblyModule: ( + source: SourceInfo, + wasmChunkPath: ChunkPath +) => WebAssembly.Module; +declare var relativeURL: (inputUrl: string) => void; diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/runtime-base.ts b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/runtime-base.ts new file mode 100644 index 0000000000000..849d33ba75606 --- /dev/null +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/runtime-base.ts @@ -0,0 +1,511 @@ +/** + * This file contains runtime types and functions that are shared between all + * Turbopack *development* ECMAScript runtimes. + * + * It will be appended to the runtime code of each runtime right after the + * shared runtime utils. + */ + +/* eslint-disable @typescript-eslint/no-unused-vars */ + +/// + +declare var TURBOPACK_WORKER_LOCATION: string; +declare var CHUNK_BASE_PATH: string; + +type ModuleFactory = ( + this: Module["exports"], + context: TurbopackBaseContext +) => undefined; + +type DevRuntimeParams = { + otherChunks: ChunkData[]; + runtimeModuleIds: ModuleId[]; +}; + +type ChunkRegistration = [ + chunkPath: ChunkPath, + chunkModules: ModuleFactories, + params: DevRuntimeParams | undefined +]; +type ChunkList = { + path: ChunkPath; + chunks: ChunkData[]; + source: "entry" | "dynamic"; +}; + +enum SourceType { + /** + * The module was instantiated because it was included in an evaluated chunk's + * runtime. + */ + Runtime = 0, + /** + * The module was instantiated because a parent module imported it. + */ + Parent = 1, + /** + * The module was instantiated because it was included in a chunk's hot module + * update. + */ + Update = 2, +} + +type SourceInfo = + | { + type: SourceType.Runtime; + chunkPath: ChunkPath; + } + | { + type: SourceType.Parent; + parentId: ModuleId; + } + | { + type: SourceType.Update; + parents?: ModuleId[]; + }; + +interface RuntimeBackend { + registerChunk: (chunkPath: ChunkPath, params?: DevRuntimeParams) => void; + loadChunk: (chunkPath: ChunkPath, source: SourceInfo) => Promise; + reloadChunk?: (chunkPath: ChunkPath) => Promise; + unloadChunk?: (chunkPath: ChunkPath) => void; + + restart: () => void; +} + +const moduleFactories: ModuleFactories = Object.create(null); +const moduleCache: ModuleCache = Object.create(null); +/** + * Module IDs that are instantiated as part of the runtime of a chunk. + */ +const runtimeModules: Set = new Set(); +/** + * Map from module ID to the chunks that contain this module. + * + * In HMR, we need to keep track of which modules are contained in which so + * chunks. This is so we don't eagerly dispose of a module when it is removed + * from chunk A, but still exists in chunk B. + */ +const moduleChunksMap: Map> = new Map(); +/** + * Map from a chunk path to all modules it contains. + */ +const chunkModulesMap: Map> = new Map(); +/** + * Chunk lists that contain a runtime. When these chunk lists receive an update + * that can't be reconciled with the current state of the page, we need to + * reload the runtime entirely. + */ +const runtimeChunkLists: Set = new Set(); +/** + * Map from a chunk list to the chunk paths it contains. + */ +const chunkListChunksMap: Map> = new Map(); +/** + * Map from a chunk path to the chunk lists it belongs to. + */ +const chunkChunkListsMap: Map> = new Map(); + +const availableModules: Map | true> = new Map(); + +const availableModuleChunks: Map | true> = new Map(); + +async function loadChunk( + source: SourceInfo, + chunkData: ChunkData +): Promise { + if (typeof chunkData === "string") { + return loadChunkPath(source, chunkData); + } + + const includedList = chunkData.included || []; + const modulesPromises = includedList.map((included) => { + if (moduleFactories[included]) return true; + return availableModules.get(included); + }); + if (modulesPromises.length > 0 && modulesPromises.every((p) => p)) { + // When all included items are already loaded or loading, we can skip loading ourselves + return Promise.all(modulesPromises); + } + + const includedModuleChunksList = chunkData.moduleChunks || []; + const moduleChunksPromises = includedModuleChunksList + .map((included) => { + // TODO(alexkirsz) Do we need this check? + // if (moduleFactories[included]) return true; + return availableModuleChunks.get(included); + }) + .filter((p) => p); + + let promise; + if (moduleChunksPromises.length > 0) { + // Some module chunks are already loaded or loading. + + if (moduleChunksPromises.length === includedModuleChunksList.length) { + // When all included module chunks are already loaded or loading, we can skip loading ourselves + return Promise.all(moduleChunksPromises); + } + + const moduleChunksToLoad: Set = new Set(); + for (const moduleChunk of includedModuleChunksList) { + if (!availableModuleChunks.has(moduleChunk)) { + moduleChunksToLoad.add(moduleChunk); + } + } + + for (const moduleChunkToLoad of moduleChunksToLoad) { + const promise = loadChunkPath(source, moduleChunkToLoad); + + availableModuleChunks.set(moduleChunkToLoad, promise); + + moduleChunksPromises.push(promise); + } + + promise = Promise.all(moduleChunksPromises); + } else { + promise = loadChunkPath(source, chunkData.path); + + // Mark all included module chunks as loading if they are not already loaded or loading. + for (const includedModuleChunk of includedModuleChunksList) { + if (!availableModuleChunks.has(includedModuleChunk)) { + availableModuleChunks.set(includedModuleChunk, promise); + } + } + } + + for (const included of includedList) { + if (!availableModules.has(included)) { + // It might be better to race old and new promises, but it's rare that the new promise will be faster than a request started earlier. + // In production it's even more rare, because the chunk optimization tries to deduplicate modules anyway. + availableModules.set(included, promise); + } + } + + return promise; +} + +async function loadChunkPath( + source: SourceInfo, + chunkPath: ChunkPath +): Promise { + try { + await BACKEND.loadChunk(chunkPath, source); + } catch (error) { + let loadReason; + switch (source.type) { + case SourceType.Runtime: + loadReason = `as a runtime dependency of chunk ${source.chunkPath}`; + break; + case SourceType.Parent: + loadReason = `from module ${source.parentId}`; + break; + case SourceType.Update: + loadReason = "from an HMR update"; + break; + default: + invariant(source, (source) => `Unknown source type: ${source?.type}`); + } + throw new Error( + `Failed to load chunk ${chunkPath} ${loadReason}${ + error ? `: ${error}` : "" + }`, + error + ? { + cause: error, + } + : undefined + ); + } +} + +/** + * Returns an absolute url to an asset. + */ +function createResolvePathFromModule( + resolver: (moduleId: string) => Exports +): (moduleId: string) => string { + return function resolvePathFromModule(moduleId: string): string { + const exported = resolver(moduleId); + return exported?.default ?? exported; + }; +} + +function instantiateModule(id: ModuleId, source: SourceInfo): Module { + const moduleFactory = moduleFactories[id]; + if (typeof moduleFactory !== "function") { + // This can happen if modules incorrectly handle HMR disposes/updates, + // e.g. when they keep a `setTimeout` around which still executes old code + // and contains e.g. a `require("something")` call. + let instantiationReason; + switch (source.type) { + case SourceType.Runtime: + instantiationReason = `as a runtime entry of chunk ${source.chunkPath}`; + break; + case SourceType.Parent: + instantiationReason = `because it was required from module ${source.parentId}`; + break; + case SourceType.Update: + instantiationReason = "because of an HMR update"; + break; + default: + invariant(source, (source) => `Unknown source type: ${source?.type}`); + } + throw new Error( + `Module ${id} was instantiated ${instantiationReason}, but the module factory is not available. It might have been deleted in an HMR update.` + ); + } + + let parents: ModuleId[]; + switch (source.type) { + case SourceType.Runtime: + runtimeModules.add(id); + parents = []; + break; + case SourceType.Parent: + // No need to add this module as a child of the parent module here, this + // has already been taken care of in `getOrInstantiateModuleFromParent`. + parents = [source.parentId]; + break; + case SourceType.Update: + parents = source.parents || []; + break; + default: + invariant(source, (source) => `Unknown source type: ${source?.type}`); + } + + const module: Module = { + exports: {}, + error: undefined, + loaded: false, + id, + parents, + children: [], + namespaceObject: undefined, + }; + + moduleCache[id] = module; + + // NOTE(alexkirsz) This can fail when the module encounters a runtime error. + try { + const sourceInfo: SourceInfo = { type: SourceType.Parent, parentId: id }; + + const r = commonJsRequire.bind(null, module); + moduleFactory.call( + module.exports, + { + a: asyncModule.bind(null, module), + e: module.exports, + r: commonJsRequire.bind(null, module), + t: runtimeRequire, + f: moduleContext, + i: esmImport.bind(null, module), + s: esmExport.bind(null, module, module.exports), + j: dynamicExport.bind(null, module, module.exports), + v: exportValue.bind(null, module), + n: exportNamespace.bind(null, module), + m: module, + c: moduleCache, + M: moduleFactories, + l: loadChunk.bind(null, sourceInfo), + w: loadWebAssembly.bind(null, sourceInfo), + u: loadWebAssemblyModule.bind(null, sourceInfo), + g: globalThis, + P: resolveAbsolutePath, + U: relativeURL, + R: createResolvePathFromModule(r), + b: getWorkerBlobURL, + __dirname: typeof module.id === "string" ? module.id.replace(/(^|\/)\/+$/, "") : module.id + } + ); + } catch (error) { + module.error = error as any; + throw error; + } + + module.loaded = true; + if (module.namespaceObject && module.exports !== module.namespaceObject) { + // in case of a circular dependency: cjs1 -> esm2 -> cjs1 + interopEsm(module.exports, module.namespaceObject); + } + + return module; +} + +/** + * no-op for browser + * @param modulePath + */ +function resolveAbsolutePath(modulePath?: string): string { + return `/ROOT/${modulePath ?? ""}`; +} + +function getWorkerBlobURL(chunks: ChunkPath[]): string { + let bootstrap = `TURBOPACK_WORKER_LOCATION = ${JSON.stringify(location.origin)};importScripts(${chunks.map(c => (`TURBOPACK_WORKER_LOCATION + ${JSON.stringify(getChunkRelativeUrl(c))}`)).join(", ")});`; + let blob = new Blob([bootstrap], { type: "text/javascript" }); + return URL.createObjectURL(blob); +} + +/** + * Retrieves a module from the cache, or instantiate it if it is not cached. + */ +const getOrInstantiateModuleFromParent: GetOrInstantiateModuleFromParent = ( + id, + sourceModule +) => { + const module = moduleCache[id]; + + if (sourceModule.children.indexOf(id) === -1) { + sourceModule.children.push(id); + } + + if (module) { + if (module.parents.indexOf(sourceModule.id) === -1) { + module.parents.push(sourceModule.id); + } + + return module; + } + + return instantiateModule(id, { + type: SourceType.Parent, + parentId: sourceModule.id, + }); +}; + +/** + * Adds a module to a chunk. + */ +function addModuleToChunk(moduleId: ModuleId, chunkPath: ChunkPath) { + let moduleChunks = moduleChunksMap.get(moduleId); + if (!moduleChunks) { + moduleChunks = new Set([chunkPath]); + moduleChunksMap.set(moduleId, moduleChunks); + } else { + moduleChunks.add(chunkPath); + } + + let chunkModules = chunkModulesMap.get(chunkPath); + if (!chunkModules) { + chunkModules = new Set([moduleId]); + chunkModulesMap.set(chunkPath, chunkModules); + } else { + chunkModules.add(moduleId); + } +} + +/** + * Returns the first chunk that included a module. + * This is used by the Node.js backend, hence why it's marked as unused in this + * file. + */ +function getFirstModuleChunk(moduleId: ModuleId) { + const moduleChunkPaths = moduleChunksMap.get(moduleId); + if (moduleChunkPaths == null) { + return null; + } + + return moduleChunkPaths.values().next().value; +} + +/** + * Removes a module from a chunk. + * Returns `true` if there are no remaining chunks including this module. + */ +function removeModuleFromChunk( + moduleId: ModuleId, + chunkPath: ChunkPath +): boolean { + const moduleChunks = moduleChunksMap.get(moduleId)!; + moduleChunks.delete(chunkPath); + + const chunkModules = chunkModulesMap.get(chunkPath)!; + chunkModules.delete(moduleId); + + const noRemainingModules = chunkModules.size === 0; + if (noRemainingModules) { + chunkModulesMap.delete(chunkPath); + } + + const noRemainingChunks = moduleChunks.size === 0; + if (noRemainingChunks) { + moduleChunksMap.delete(moduleId); + } + + return noRemainingChunks; +} + +/** + * Instantiates a runtime module. + */ +function instantiateRuntimeModule( + moduleId: ModuleId, + chunkPath: ChunkPath +): Module { + return instantiateModule(moduleId, { type: SourceType.Runtime, chunkPath }); +} + +/** + * Gets or instantiates a runtime module. + */ +function getOrInstantiateRuntimeModule( + moduleId: ModuleId, + chunkPath: ChunkPath +): Module { + const module = moduleCache[moduleId]; + if (module) { + if (module.error) { + throw module.error; + } + return module; + } + + return instantiateModule(moduleId, { type: SourceType.Runtime, chunkPath }); +} + +/** + * Returns the URL relative to the origin where a chunk can be fetched from. + */ +function getChunkRelativeUrl(chunkPath: ChunkPath): string { + return `${CHUNK_BASE_PATH}${chunkPath + .split("/") + .map((p) => encodeURIComponent(p)) + .join("/")}`; +} + +/** + * Marks a chunk list as a runtime chunk list. There can be more than one + * runtime chunk list. For instance, integration tests can have multiple chunk + * groups loaded at runtime, each with its own chunk list. + */ +function markChunkListAsRuntime(chunkListPath: ChunkPath) { + runtimeChunkLists.add(chunkListPath); +} + +function registerChunk([ + chunkPath, + chunkModules, + runtimeParams, +]: ChunkRegistration) { + for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) { + if (!moduleFactories[moduleId]) { + moduleFactories[moduleId] = moduleFactory; + } + addModuleToChunk(moduleId, chunkPath); + } + + return BACKEND.registerChunk(chunkPath, runtimeParams); +} + +// const chunkListsToRegister = globalThis.TURBOPACK_CHUNK_LISTS; +// if (Array.isArray(chunkListsToRegister)) { +// for (const chunkList of chunkListsToRegister) { +// registerChunkList(globalThis.TURBOPACK_CHUNK_UPDATE_LISTENERS, chunkList); +// } +// } + +// globalThis.TURBOPACK_CHUNK_LISTS = { +// push: (chunkList) => { +// registerChunkList(globalThis.TURBOPACK_CHUNK_UPDATE_LISTENERS!, chunkList); +// }, +// } satisfies ChunkListProvider; diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/tsconfig.json b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/tsconfig.json new file mode 100644 index 0000000000000..5aa7968522993 --- /dev/null +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + // environment, we need WebWorker for WebAssembly types + "lib": ["ESNext", "WebWorker"] + }, + "include": ["runtime-base.ts", "dummy.ts"] +} diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/dom/runtime-backend-dom.ts b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/dom/runtime-backend-dom.ts new file mode 100644 index 0000000000000..0988c10e55df6 --- /dev/null +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/dom/runtime-backend-dom.ts @@ -0,0 +1,297 @@ +/** + * This file contains the runtime code specific to the Turbopack development + * ECMAScript DOM runtime. + * + * It will be appended to the base development runtime code. + */ + +/* eslint-disable @typescript-eslint/no-unused-vars */ + +/// +/// + +type ChunkResolver = { + resolved: boolean; + resolve: () => void; + reject: (error?: Error) => void; + promise: Promise; +}; + +let BACKEND: RuntimeBackend; + +function fetchWebAssembly(wasmChunkPath: ChunkPath) { + return fetch(getChunkRelativeUrl(wasmChunkPath)); +} + +async function loadWebAssembly( + _source: unknown, + wasmChunkPath: ChunkPath, + importsObj: WebAssembly.Imports +): Promise { + const req = fetchWebAssembly(wasmChunkPath); + + const { instance } = await WebAssembly.instantiateStreaming(req, importsObj); + + return instance.exports; +} + +async function loadWebAssemblyModule( + _source: unknown, + wasmChunkPath: ChunkPath +): Promise { + const req = fetchWebAssembly(wasmChunkPath); + + return await WebAssembly.compileStreaming(req); +} + +(() => { + BACKEND = { + async registerChunk(chunkPath, params) { + const resolver = getOrCreateResolver(chunkPath); + resolver.resolve(); + + if (params == null) { + return; + } + + for (const otherChunkData of params.otherChunks) { + const otherChunkPath = getChunkPath(otherChunkData); + // Chunk might have started loading, so we want to avoid triggering another load. + getOrCreateResolver(otherChunkPath); + } + + // This waits for chunks to be loaded, but also marks included items as available. + await Promise.all( + params.otherChunks.map((otherChunkData) => + loadChunk({ type: SourceType.Runtime, chunkPath }, otherChunkData) + ) + ); + + if (params.runtimeModuleIds.length > 0) { + for (const moduleId of params.runtimeModuleIds) { + getOrInstantiateRuntimeModule(moduleId, chunkPath); + } + } + }, + + loadChunk(chunkPath, source) { + return doLoadChunk(chunkPath, source); + }, + + unloadChunk(chunkPath) { + deleteResolver(chunkPath); + + const chunkUrl = getChunkRelativeUrl(chunkPath); + // TODO(PACK-2140): remove this once all filenames are guaranteed to be escaped. + const decodedChunkUrl = decodeURI(chunkUrl); + + if (chunkPath.endsWith(".css")) { + const links = document.querySelectorAll( + `link[href="${chunkUrl}"],link[href^="${chunkUrl}?"],link[href="${decodedChunkUrl}"],link[href^="${decodedChunkUrl}?"]` + ); + for (const link of Array.from(links)) { + link.remove(); + } + } else if (chunkPath.endsWith(".js")) { + // Unloading a JS chunk would have no effect, as it lives in the JS + // runtime once evaluated. + // However, we still want to remove the script tag from the DOM to keep + // the HTML somewhat consistent from the user's perspective. + const scripts = document.querySelectorAll( + `script[src="${chunkUrl}"],script[src^="${chunkUrl}?"],script[src="${decodedChunkUrl}"],script[src^="${decodedChunkUrl}?"]` + ); + for (const script of Array.from(scripts)) { + script.remove(); + } + } else { + throw new Error(`can't infer type of chunk from path ${chunkPath}`); + } + }, + + reloadChunk(chunkPath) { + return new Promise((resolve, reject) => { + if (!chunkPath.endsWith(".css")) { + reject(new Error("The DOM backend can only reload CSS chunks")); + return; + } + + const chunkUrl = getChunkRelativeUrl(chunkPath); + const decodedChunkUrl = decodeURI(chunkUrl); + + const previousLinks = document.querySelectorAll( + `link[rel=stylesheet][href="${chunkUrl}"],link[rel=stylesheet][href^="${chunkUrl}?"],link[rel=stylesheet][href="${decodedChunkUrl}"],link[rel=stylesheet][href^="${decodedChunkUrl}?"]` + ); + + if (previousLinks.length === 0) { + reject(new Error(`No link element found for chunk ${chunkPath}`)); + return; + } + + const link = document.createElement("link"); + link.rel = "stylesheet"; + + if (navigator.userAgent.includes("Firefox")) { + // Firefox won't reload CSS files that were previously loaded on the current page, + // we need to add a query param to make sure CSS is actually reloaded from the server. + // + // I believe this is this issue: https://bugzilla.mozilla.org/show_bug.cgi?id=1037506 + // + // Safari has a similar issue, but only if you have a `` tag + // pointing to the same URL as the stylesheet: https://bugs.webkit.org/show_bug.cgi?id=187726 + link.href = `${chunkUrl}?ts=${Date.now()}`; + } else { + link.href = chunkUrl; + } + + link.onerror = () => { + reject(); + }; + link.onload = () => { + // First load the new CSS, then remove the old ones. This prevents visible + // flickering that would happen in-between removing the previous CSS and + // loading the new one. + for (const previousLink of Array.from(previousLinks)) + previousLink.remove(); + + // CSS chunks do not register themselves, and as such must be marked as + // loaded instantly. + resolve(); + }; + + // Make sure to insert the new CSS right after the previous one, so that + // its precedence is higher. + previousLinks[0].parentElement!.insertBefore( + link, + previousLinks[0].nextSibling + ); + }); + }, + + restart: () => self.location.reload(), + }; + + /** + * Maps chunk paths to the corresponding resolver. + */ + const chunkResolvers: Map = new Map(); + + function getOrCreateResolver(chunkPath: ChunkPath): ChunkResolver { + let resolver = chunkResolvers.get(chunkPath); + if (!resolver) { + let resolve: () => void; + let reject: (error?: Error) => void; + const promise = new Promise((innerResolve, innerReject) => { + resolve = innerResolve; + reject = innerReject; + }); + resolver = { + resolved: false, + promise, + resolve: () => { + resolver!.resolved = true; + resolve(); + }, + reject: reject!, + }; + chunkResolvers.set(chunkPath, resolver); + } + return resolver; + } + + function deleteResolver(chunkPath: ChunkPath) { + chunkResolvers.delete(chunkPath); + } + + /** + * Loads the given chunk, and returns a promise that resolves once the chunk + * has been loaded. + */ + async function doLoadChunk(chunkPath: ChunkPath, source: SourceInfo) { + const resolver = getOrCreateResolver(chunkPath); + if (resolver.resolved) { + return resolver.promise; + } + + if (source.type === SourceType.Runtime) { + // We don't need to load chunks references from runtime code, as they're already + // present in the DOM. + + if (chunkPath.endsWith(".css")) { + // CSS chunks do not register themselves, and as such must be marked as + // loaded instantly. + resolver.resolve(); + } + + // We need to wait for JS chunks to register themselves within `registerChunk` + // before we can start instantiating runtime modules, hence the absence of + // `resolver.resolve()` in this branch. + + return resolver.promise; + } + + const chunkUrl = getChunkRelativeUrl(chunkPath); + const decodedChunkUrl = decodeURI(chunkUrl); + + if (typeof importScripts === "function") { + // We're in a web worker + if (chunkPath.endsWith(".css")) { + // ignore + } else if (chunkPath.endsWith(".js")) { + importScripts(TURBOPACK_WORKER_LOCATION + chunkUrl); + } else { + throw new Error(`can't infer type of chunk from path ${chunkPath} in worker`); + } + } else { + if (chunkPath.endsWith(".css")) { + const previousLinks = document.querySelectorAll( + `link[rel=stylesheet][href="${chunkUrl}"],link[rel=stylesheet][href^="${chunkUrl}?"],link[rel=stylesheet][href="${decodedChunkUrl}"],link[rel=stylesheet][href^="${decodedChunkUrl}?"]` + ); + if (previousLinks.length > 0) { + // CSS chunks do not register themselves, and as such must be marked as + // loaded instantly. + resolver.resolve(); + } else { + const link = document.createElement("link"); + link.rel = "stylesheet"; + link.href = chunkUrl; + link.onerror = () => { + resolver.reject(); + }; + link.onload = () => { + // CSS chunks do not register themselves, and as such must be marked as + // loaded instantly. + resolver.resolve(); + }; + document.body.appendChild(link); + } + } else if (chunkPath.endsWith(".js")) { + const previousScripts = document.querySelectorAll( + `script[src="${chunkUrl}"],script[src^="${chunkUrl}?"],script[src="${decodedChunkUrl}"],script[src^="${decodedChunkUrl}?"]` + ); + if (previousScripts.length > 0) { + // There is this edge where the script already failed loading, but we + // can't detect that. The Promise will never resolve in this case. + for (const script of Array.from(previousScripts)) { + script.addEventListener("error", () => { + resolver.reject(); + }); + } + } else { + const script = document.createElement("script"); + script.src = chunkUrl; + // We'll only mark the chunk as loaded once the script has been executed, + // which happens in `registerChunk`. Hence the absence of `resolve()` in + // this branch. + script.onerror = () => { + resolver.reject(); + }; + document.body.appendChild(script); + } + } else { + throw new Error(`can't infer type of chunk from path ${chunkPath}`); + } + } + + return resolver.promise; + } +})(); diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/dom/tsconfig.json b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/dom/tsconfig.json new file mode 100644 index 0000000000000..4908beba8e3ea --- /dev/null +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/dom/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + // environment + "lib": ["ESNext", "DOM", "WebWorker.ImportScripts"] + }, + "include": ["*.ts"] +} diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/edge/runtime-backend-edge.ts b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/edge/runtime-backend-edge.ts new file mode 100644 index 0000000000000..1974bdec118ea --- /dev/null +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/edge/runtime-backend-edge.ts @@ -0,0 +1,186 @@ +/** + * This file contains the runtime code specific to the Turbopack development + * ECMAScript "None" runtime (e.g. for Edge). + * + * It will be appended to the base development runtime code. + */ + +/* eslint-disable @typescript-eslint/no-unused-vars */ + +/// + +type ChunkRunner = { + requiredChunks: Set; + chunkPath: ChunkPath; + runtimeModuleIds: ModuleId[]; +}; + +let BACKEND: RuntimeBackend; + +type ExternalRequire = ( + id: ModuleId, + esm?: boolean +) => Exports | EsmNamespaceObject; +type ExternalImport = (id: ModuleId) => Promise; + +async function loadWebAssembly( + source: SourceInfo, + chunkPath: ChunkPath, + imports: WebAssembly.Imports +): Promise { + const module = await loadWebAssemblyModule(source, chunkPath); + + return await WebAssembly.instantiate(module, imports); +} + +function getFileStem(path: string): string { + const fileName = path.split("/").pop()!; + + const stem = fileName.split(".").shift()!; + + if (stem === "") { + return fileName; + } + + return stem; +} + +type GlobalWithInjectedWebAssembly = typeof globalThis & { + [key: `wasm_${string}`]: WebAssembly.Module; +}; + +async function loadWebAssemblyModule( + _source: SourceInfo, + chunkPath: ChunkPath +): Promise { + const stem = getFileStem(chunkPath); + + // very simple escaping just replacing unsupported characters with `_` + const escaped = stem.replace(/[^a-zA-Z0-9$_]/gi, "_"); + + const identifier: `wasm_${string}` = `wasm_${escaped}`; + + const module = (globalThis as GlobalWithInjectedWebAssembly)[identifier]; + + if (!module) { + throw new Error( + `dynamically loading WebAssembly is not supported in this runtime and global \`${identifier}\` was not injected` + ); + } + + return module; +} + +(() => { + BACKEND = { + // The "none" runtime expects all chunks within the same chunk group to be + // registered before any of them are instantiated. + // Furthermore, modules must be instantiated synchronously, hence we don't + // use promises here. + registerChunk(chunkPath, params) { + registeredChunks.add(chunkPath); + instantiateDependentChunks(chunkPath); + + if (params == null) { + return; + } + + if (params.otherChunks.length === 0) { + // The current chunk does not depend on any other chunks, it can be + // instantiated immediately. + instantiateRuntimeModules(params.runtimeModuleIds, chunkPath); + } else { + // The current chunk depends on other chunks, so we need to wait for + // those chunks to be registered before instantiating the runtime + // modules. + registerChunkRunner( + chunkPath, + params.otherChunks.filter((chunk) => + // The none runtime can only handle JS chunks, so we only wait for these + getChunkPath(chunk).endsWith(".js") + ), + params.runtimeModuleIds + ); + } + }, + + loadChunk(_chunkPath, _fromChunkPath) { + throw new Error("chunk loading is not supported"); + }, + + restart: () => { + throw new Error("restart is not supported"); + }, + }; + + const registeredChunks: Set = new Set(); + const runners: Map> = new Map(); + + /** + * Registers a chunk runner that will be instantiated once all of the + * dependencies of the chunk have been registered. + */ + function registerChunkRunner( + chunkPath: ChunkPath, + otherChunks: ChunkData[], + runtimeModuleIds: ModuleId[] + ) { + const requiredChunks: Set = new Set(); + const runner = { + runtimeModuleIds, + chunkPath, + requiredChunks, + }; + + for (const otherChunkData of otherChunks) { + const otherChunkPath = getChunkPath(otherChunkData); + if (registeredChunks.has(otherChunkPath)) { + continue; + } + + requiredChunks.add(otherChunkPath); + let runnersForChunk = runners.get(otherChunkPath); + if (runnersForChunk == null) { + runnersForChunk = new Set(); + runners.set(otherChunkPath, runnersForChunk); + } + runnersForChunk.add(runner); + } + // When all chunks are already registered, we can instantiate the runtime module + if (runner.requiredChunks.size === 0) { + instantiateRuntimeModules(runner.runtimeModuleIds, runner.chunkPath); + } + } + + /** + * Instantiates any chunk runners that were waiting for the given chunk to be + * registered. + */ + function instantiateDependentChunks(chunkPath: ChunkPath) { + // Run any chunk runners that were waiting for this chunk to be + // registered. + const runnersForChunk = runners.get(chunkPath); + if (runnersForChunk != null) { + for (const runner of runnersForChunk) { + runner.requiredChunks.delete(chunkPath); + + if (runner.requiredChunks.size === 0) { + instantiateRuntimeModules(runner.runtimeModuleIds, runner.chunkPath); + } + } + runners.delete(chunkPath); + } + } + + /** + * Instantiates the runtime modules for the given chunk. + */ + function instantiateRuntimeModules( + runtimeModuleIds: ModuleId[], + chunkPath: ChunkPath + ) { + for (const moduleId of runtimeModuleIds) { + getOrInstantiateRuntimeModule(moduleId, chunkPath); + } + } +})(); diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/edge/tsconfig.json b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/edge/tsconfig.json new file mode 100644 index 0000000000000..5722142375561 --- /dev/null +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/edge/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + // environment, we need WebWorker for WebAssembly types + "lib": ["ESNext", "WebWorker"] + }, + "include": ["*.ts"] +} diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/nodejs/runtime.ts b/turbopack/crates/turbopack-ecmascript-runtime/js/src/nodejs/runtime.ts index 034488a5bdb7c..5d0f88da10e1d 100644 --- a/turbopack/crates/turbopack-ecmascript-runtime/js/src/nodejs/runtime.ts +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/nodejs/runtime.ts @@ -177,7 +177,7 @@ function loadWebAssemblyModule(chunkPath: ChunkPath) { return compileWebAssemblyFromPath(resolved); } -function getWorkerBlobURL(_chunks: ChunkPath[]) { +function getWorkerBlobURL(_chunks: ChunkPath[]): string { throw new Error("Worker blobs are not implemented yet for Node.js"); } diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/shared/runtime-types.d.ts b/turbopack/crates/turbopack-ecmascript-runtime/js/src/shared/runtime-types.d.ts index 53b0ffaded23c..2dea43a5a0303 100644 --- a/turbopack/crates/turbopack-ecmascript-runtime/js/src/shared/runtime-types.d.ts +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/shared/runtime-types.d.ts @@ -38,7 +38,7 @@ type LoadWebAssembly = ( type LoadWebAssemblyModule = (wasmChunkPath: ChunkPath) => WebAssembly.Module; type ModuleCache = Record; -type ModuleFactories = Record; +type ModuleFactories = Record; type RelativeURL = (inputUrl: string) => void; type ResolvePathFromModule = (moduleId: string) => string; diff --git a/turbopack/crates/turbopack-ecmascript-runtime/js/src/shared/runtime-utils.ts b/turbopack/crates/turbopack-ecmascript-runtime/js/src/shared/runtime-utils.ts index e74d2f08302ad..3ff080d9797a4 100644 --- a/turbopack/crates/turbopack-ecmascript-runtime/js/src/shared/runtime-utils.ts +++ b/turbopack/crates/turbopack-ecmascript-runtime/js/src/shared/runtime-utils.ts @@ -243,7 +243,9 @@ function esmImport( // Add a simple runtime require so that environments without one can still pass // `typeof require` CommonJS checks so that exports are correctly registered. const runtimeRequire = + // @ts-ignore typeof require === "function" + // @ts-ignore ? require : function require() { throw new Error("Unexpected use of runtime require"); diff --git a/turbopack/crates/turbopack-ecmascript-runtime/src/browser_runtime.rs b/turbopack/crates/turbopack-ecmascript-runtime/src/browser_runtime.rs index 3ef6e5f8f01e5..46437c1057153 100644 --- a/turbopack/crates/turbopack-ecmascript-runtime/src/browser_runtime.rs +++ b/turbopack/crates/turbopack-ecmascript-runtime/src/browser_runtime.rs @@ -2,7 +2,7 @@ use std::io::Write; use anyhow::Result; use indoc::writedoc; -use turbo_tasks::{RcStr, Vc}; +use turbo_tasks::{RcStr, Value, Vc}; use turbopack_core::{ code_builder::{Code, CodeBuilder}, context::AssetContext, @@ -10,13 +10,14 @@ use turbopack_core::{ }; use turbopack_ecmascript::utils::StringifyJs; -use crate::{asset_context::get_runtime_asset_context, embed_js::embed_static_code}; +use crate::{asset_context::get_runtime_asset_context, embed_js::embed_static_code, RuntimeType}; -/// Returns the code for the development ECMAScript runtime. +/// Returns the code for the ECMAScript runtime. #[turbo_tasks::function] pub async fn get_browser_runtime_code( environment: Vc, chunk_base_path: Vc>, + runtime_type: Value, output_root: Vc, ) -> Result> { let asset_context = get_runtime_asset_context(environment); @@ -25,7 +26,11 @@ pub async fn get_browser_runtime_code( embed_static_code(asset_context, "shared/runtime-utils.ts".into()); let runtime_base_code = embed_static_code( asset_context, - "browser/dev/runtime/base/runtime-base.ts".into(), + if *runtime_type == RuntimeType::Development { + "browser/dev/runtime/base/runtime-base.ts".into() + } else { + "browser/runtime/base/runtime-base.ts".into() + }, ); let chunk_loading = &*asset_context @@ -34,15 +39,34 @@ pub async fn get_browser_runtime_code( .chunk_loading() .await?; + dbg!(runtime_type); + let runtime_backend_code = embed_static_code( asset_context, - match chunk_loading { - ChunkLoading::Edge => "browser/dev/runtime/edge/runtime-backend-edge.ts".into(), + match (chunk_loading, *runtime_type) { + (ChunkLoading::Edge, RuntimeType::Development) => { + "browser/dev/runtime/edge/runtime-backend-edge.ts".into() + } + (ChunkLoading::Edge, RuntimeType::Production) => { + // TODO + "browser/runtime/edge/runtime-backend-edge.ts".into() + } // This case should never be hit. - ChunkLoading::NodeJs => { + (ChunkLoading::NodeJs, _) => { panic!("Node.js runtime is not supported in the browser runtime!") } - ChunkLoading::Dom => "browser/dev/runtime/dom/runtime-backend-dom.ts".into(), + (ChunkLoading::Dom, RuntimeType::Development) => { + "browser/dev/runtime/dom/runtime-backend-dom.ts".into() + } + (ChunkLoading::Dom, RuntimeType::Production) => { + // TODO + "browser/runtime/dom/runtime-backend-dom.ts".into() + } + + #[cfg(feature = "test")] + (_, RuntimeType::Dummy) => { + panic!("This configuration is not supported in the browser runtime") + } }, );