From e97ac1ebf429a8b50453d3b44622a84d01abcd4b Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 15 Apr 2024 15:02:18 +0200 Subject: [PATCH] fix: correct soft invalidation --- .../vite/src/module-runner/moduleCache.ts | 9 +- packages/vite/src/module-runner/runner.ts | 121 +++++++++--------- packages/vite/src/module-runner/types.ts | 3 +- packages/vite/src/module-runner/utils.ts | 27 ++++ packages/vite/src/node/server/hmr.ts | 27 ---- .../vite/src/node/server/transformRequest.ts | 97 +++++++------- .../__tests__/server-source-maps.spec.ts | 2 +- .../src/node/ssr/runtime/__tests__/utils.ts | 5 +- packages/vite/src/node/ssr/ssrTransform.ts | 51 ++++++++ 9 files changed, 201 insertions(+), 141 deletions(-) diff --git a/packages/vite/src/module-runner/moduleCache.ts b/packages/vite/src/module-runner/moduleCache.ts index 3e7029274d38a4..ed94cc7bcbcbe2 100644 --- a/packages/vite/src/module-runner/moduleCache.ts +++ b/packages/vite/src/module-runner/moduleCache.ts @@ -46,6 +46,7 @@ export class ModuleCacheMap extends Map { Object.assign(mod, { imports: new Set(), importers: new Set(), + timestamp: 0, }) } return mod @@ -63,8 +64,12 @@ export class ModuleCacheMap extends Map { return this.deleteByModuleId(this.normalize(fsPath)) } - invalidate(id: string): void { + invalidateUrl(id: string): void { const module = this.get(id) + this.invalidateModule(module) + } + + invalidateModule(module: ModuleCache): void { module.evaluated = false module.meta = undefined module.map = undefined @@ -90,7 +95,7 @@ export class ModuleCacheMap extends Map { invalidated.add(id) const mod = super.get(id) if (mod?.importers) this.invalidateDepTree(mod.importers, invalidated) - this.invalidate(id) + this.invalidateUrl(id) } return invalidated } diff --git a/packages/vite/src/module-runner/runner.ts b/packages/vite/src/module-runner/runner.ts index 92565380014374..8e95e1907febb3 100644 --- a/packages/vite/src/module-runner/runner.ts +++ b/packages/vite/src/module-runner/runner.ts @@ -14,7 +14,6 @@ import { } from '../shared/ssrTransform' import { ModuleCacheMap } from './moduleCache' import type { - FetchResult, ModuleCache, ModuleEvaluator, ModuleRunnerContext, @@ -24,6 +23,7 @@ import type { SSRImportMetadata, } from './types' import { + parseUrl, posixDirname, posixPathToFileHref, posixResolve, @@ -53,19 +53,19 @@ export class ModuleRunner { public moduleCache: ModuleCacheMap public hmrClient?: HMRClient - private idToUrlMap = new Map() - private fileToIdMap = new Map() - private envProxy = new Proxy({} as any, { + private readonly urlToIdMap = new Map() + private readonly fileToIdMap = new Map() + private readonly envProxy = new Proxy({} as any, { get(_, p) { throw new Error( `[module runner] Dynamic access of "import.meta.env" is not supported. Please, use "import.meta.env.${String(p)}" instead.`, ) }, }) - private transport: RunnerTransport + private readonly transport: RunnerTransport + private readonly resetSourceMapSupport?: () => void - private _destroyed = false - private _resetSourceMapSupport?: () => void + private destroyed = false constructor( public options: ModuleRunnerOptions, @@ -80,18 +80,20 @@ export class ModuleRunner { ? silentConsole : options.hmr.logger || console, options.hmr.connection, - ({ acceptedPath, invalidates }) => { - this.moduleCache.invalidate(acceptedPath) - if (invalidates) { - this.invalidateFiles(invalidates) - } - return this.import(acceptedPath) + ({ acceptedPath, explicitImportRequired, timestamp }) => { + const [acceptedPathWithoutQuery, query] = acceptedPath.split(`?`) + const url = + acceptedPathWithoutQuery + + `?${explicitImportRequired ? 'import&' : ''}t=${timestamp}${ + query ? `&${query}` : '' + }` + return this.import(url) }, ) options.hmr.connection.onUpdate(createHMRHandler(this)) } if (options.sourcemapInterceptor !== false) { - this._resetSourceMapSupport = enableSourceMapSupport(this) + this.resetSourceMapSupport = enableSourceMapSupport(this) } } @@ -109,7 +111,7 @@ export class ModuleRunner { */ public clearCache(): void { this.moduleCache.clear() - this.idToUrlMap.clear() + this.urlToIdMap.clear() this.hmrClient?.clear() } @@ -118,27 +120,17 @@ export class ModuleRunner { * This method doesn't stop the HMR connection. */ public async destroy(): Promise { - this._resetSourceMapSupport?.() + this.resetSourceMapSupport?.() this.clearCache() this.hmrClient = undefined - this._destroyed = true + this.destroyed = true } /** * Returns `true` if the runtime has been destroyed by calling `destroy()` method. */ public isDestroyed(): boolean { - return this._destroyed - } - - // map files to modules and invalidate them - private invalidateFiles(files: string[]) { - files.forEach((file) => { - const ids = this.fileToIdMap.get(file) - if (ids) { - ids.forEach((id) => this.moduleCache.invalidate(id)) - } - }) + return this.destroyed } // we don't use moduleCache.normalize because this URL doesn't have to follow the same rules @@ -187,13 +179,12 @@ export class ModuleRunner { private async cachedRequest( id: string, - fetchedModule: ResolvedResult, + mod: ModuleCache, callstack: string[] = [], metadata?: SSRImportMetadata, ): Promise { - const moduleId = fetchedModule.id - - const mod = this.moduleCache.getByModuleId(moduleId) + const meta = mod.meta! + const moduleId = meta.id const { imports, importers } = mod as Required @@ -206,8 +197,7 @@ export class ModuleRunner { callstack.includes(moduleId) || Array.from(imports.values()).some((i) => importers.has(i)) ) { - if (mod.exports) - return this.processImport(mod.exports, fetchedModule, metadata) + if (mod.exports) return this.processImport(mod.exports, meta, metadata) } let debugTimer: any @@ -228,12 +218,12 @@ export class ModuleRunner { try { // cached module if (mod.promise) - return this.processImport(await mod.promise, fetchedModule, metadata) + return this.processImport(await mod.promise, meta, metadata) - const promise = this.directRequest(id, fetchedModule, callstack) + const promise = this.directRequest(id, mod, callstack) mod.promise = promise mod.evaluated = false - return this.processImport(await promise, fetchedModule, metadata) + return this.processImport(await promise, meta, metadata) } finally { mod.evaluated = true if (debugTimer) clearTimeout(debugTimer) @@ -241,35 +231,46 @@ export class ModuleRunner { } private async cachedModule( - id: string, + url: string, importer?: string, - ): Promise { - if (this._destroyed) { + ): Promise { + if (this.destroyed) { throw new Error(`[vite] Vite runtime has been destroyed.`) } - const normalized = this.idToUrlMap.get(id) + const normalized = this.urlToIdMap.get(url) if (normalized) { const mod = this.moduleCache.getByModuleId(normalized) if (mod.meta) { - return mod.meta as ResolvedResult + return mod } } - this.debug?.('[module runner] fetching', id) + + this.debug?.('[module runner] fetching', url) // fast return for established externalized patterns - const fetchedModule = id.startsWith('data:') - ? ({ externalize: id, type: 'builtin' } satisfies FetchResult) - : await this.transport.fetchModule(id, importer) + const fetchedModule = ( + url.startsWith('data:') + ? { externalize: url, type: 'builtin' } + : await this.transport.fetchModule(url, importer) + ) as ResolvedResult + // base moduleId on "file" and not on id // if `import(variable)` is called it's possible that it doesn't have an extension for example - // if we used id for that, it's possible to have a duplicated module - const idQuery = id.split('?')[1] - const query = idQuery ? `?${idQuery}` : '' + // if we used id for that, then a module will be duplicated + const { query, timestamp } = parseUrl(url) const file = 'file' in fetchedModule ? fetchedModule.file : undefined - const fullFile = file ? `${file}${query}` : id - const moduleId = this.moduleCache.normalize(fullFile) + const fileId = file ? `${file}${query}` : url + const moduleId = this.moduleCache.normalize(fileId) const mod = this.moduleCache.getByModuleId(moduleId) - ;(fetchedModule as ResolvedResult).id = moduleId + + // if URL has a ?t= query, it might've been invalidated due to HMR + // checking if we should also invalidate the module + if (mod.timestamp != null && timestamp > 0 && mod.timestamp < timestamp) { + this.moduleCache.invalidateModule(mod) + } + + fetchedModule.id = moduleId mod.meta = fetchedModule + mod.timestamp = timestamp if (file) { const fileModules = this.fileToIdMap.get(file) || [] @@ -277,27 +278,27 @@ export class ModuleRunner { this.fileToIdMap.set(file, fileModules) } - this.idToUrlMap.set(id, moduleId) - this.idToUrlMap.set(unwrapId(id), moduleId) - return fetchedModule as ResolvedResult + this.urlToIdMap.set(url, moduleId) + this.urlToIdMap.set(unwrapId(url), moduleId) + return mod } // override is allowed, consider this a public API protected async directRequest( id: string, - fetchResult: ResolvedResult, + mod: ModuleCache, _callstack: string[], ): Promise { + const fetchResult = mod.meta! const moduleId = fetchResult.id const callstack = [..._callstack, moduleId] - const mod = this.moduleCache.getByModuleId(moduleId) - const request = async (dep: string, metadata?: SSRImportMetadata) => { const fetchedModule = await this.cachedModule(dep, moduleId) - const depMod = this.moduleCache.getByModuleId(fetchedModule.id) + const resolvedId = fetchedModule.meta!.id + const depMod = this.moduleCache.getByModuleId(resolvedId) depMod.importers!.add(moduleId) - mod.imports!.add(fetchedModule.id) + mod.imports!.add(resolvedId) return this.cachedRequest(dep, fetchedModule, callstack, metadata) } diff --git a/packages/vite/src/module-runner/types.ts b/packages/vite/src/module-runner/types.ts index 578a1b6baee75d..165a593fa31654 100644 --- a/packages/vite/src/module-runner/types.ts +++ b/packages/vite/src/module-runner/types.ts @@ -69,7 +69,8 @@ export interface ModuleCache { exports?: any evaluated?: boolean map?: DecodedMap - meta?: FetchResult + meta?: ResolvedResult + timestamp?: number /** * Module ids that imports this module */ diff --git a/packages/vite/src/module-runner/utils.ts b/packages/vite/src/module-runner/utils.ts index 12e06a3ebb1882..60070fc22531a2 100644 --- a/packages/vite/src/module-runner/utils.ts +++ b/packages/vite/src/module-runner/utils.ts @@ -16,6 +16,33 @@ const carriageReturnRegEx = /\r/g const tabRegEx = /\t/g const questionRegex = /\?/g const hashRegex = /#/g +const timestampRegex = /[?&]t=(\d{13})(&?)/ + +interface ParsedPath { + query: string + timestamp: number +} + +export function parseUrl(url: string): ParsedPath { + const idQuery = url.split('?')[1] + let timestamp = 0 + // for performance, we avoid using URL constructor and parsing twice + // it's not really needed, but it's a micro-optimization that we can do for free + const query = idQuery + ? ('?' + idQuery).replace( + timestampRegex, + (substring, tsString, nextItem) => { + timestamp = Number(tsString) + // remove the "?t=" query since it's only used for invalidation + return substring[0] === '?' && nextItem === '&' ? '?' : '' + }, + ) + : '' + return { + query, + timestamp, + } +} function encodePathChars(filepath: string) { if (filepath.indexOf('%') !== -1) diff --git a/packages/vite/src/node/server/hmr.ts b/packages/vite/src/node/server/hmr.ts index 0924f4b387e602..d1fe86317be25b 100644 --- a/packages/vite/src/node/server/hmr.ts +++ b/packages/vite/src/node/server/hmr.ts @@ -397,7 +397,6 @@ export function updateModules( ? isExplicitImportRequired(acceptedVia.url) : false, isWithinCircularImport, - invalidates: getInvalidatedImporters(acceptedVia), }), ), ) @@ -435,32 +434,6 @@ export function updateModules( }) } -function populateImporters( - module: EnvironmentModuleNode, - timestamp: number, - seen: Set = new Set(), -) { - module.importedModules.forEach((importer) => { - if (seen.has(importer)) { - return - } - if ( - importer.lastHMRTimestamp === timestamp || - importer.lastInvalidationTimestamp === timestamp - ) { - seen.add(importer) - populateImporters(importer, timestamp, seen) - } - }) - return seen -} - -function getInvalidatedImporters(module: EnvironmentModuleNode) { - return [...populateImporters(module, module.lastHMRTimestamp)].map( - (m) => m.file!, - ) -} - function areAllImportsAccepted( importedBindings: Set, acceptedExports: Set, diff --git a/packages/vite/src/node/server/transformRequest.ts b/packages/vite/src/node/server/transformRequest.ts index 6e67a0e1ff829b..916f82da16f3f4 100644 --- a/packages/vite/src/node/server/transformRequest.ts +++ b/packages/vite/src/node/server/transformRequest.ts @@ -20,6 +20,7 @@ import { } from '../utils' import { checkPublicFile } from '../publicDir' import { cleanUrl, unwrapId } from '../../shared/utils' +import { ssrParseImports } from '../ssr/ssrTransform' import { applySourcemapIgnoreList, extractSourcemapFromFile, @@ -39,6 +40,7 @@ const debugCache = createDebugger('vite:cache') export interface TransformResult { code: string map: SourceMap | { mappings: '' } | null + ssr?: boolean etag?: string deps?: string[] dynamicDeps?: string[] @@ -466,62 +468,59 @@ async function handleModuleSoftInvalidation( } let result: TransformResult - // For SSR soft-invalidation, no transformation is needed - if (environment.name !== 'client') { - result = transformResult - } - // For client soft-invalidation, we need to transform each imports with new timestamps if available - else { - await init - const source = transformResult.code - const s = new MagicString(source) - const [imports] = parseImports(source, mod.id || undefined) - - for (const imp of imports) { - let rawUrl = source.slice(imp.s, imp.e) - if (rawUrl === 'import.meta') continue - - const hasQuotes = rawUrl[0] === '"' || rawUrl[0] === "'" - if (hasQuotes) { - rawUrl = rawUrl.slice(1, -1) - } - - const urlWithoutTimestamp = removeTimestampQuery(rawUrl) - // hmrUrl must be derived the same way as importAnalysis - const hmrUrl = unwrapId( - stripBase(removeImportQuery(urlWithoutTimestamp), server.config.base), - ) - for (const importedMod of mod.importedModules) { - if (importedMod.url !== hmrUrl) continue - if (importedMod.lastHMRTimestamp > 0) { - const replacedUrl = injectQuery( - urlWithoutTimestamp, - `t=${importedMod.lastHMRTimestamp}`, - ) - const start = hasQuotes ? imp.s + 1 : imp.s - const end = hasQuotes ? imp.e - 1 : imp.e - s.overwrite(start, end, replacedUrl) - } + const source = transformResult.code + const s = new MagicString(source) + const imports = transformResult.ssr + ? await ssrParseImports(mod.url, source) + : await (async () => { + await init + return parseImports(source, mod.id || undefined)[0] + })() + + for (const imp of imports) { + let rawUrl = source.slice(imp.s, imp.e) + if (rawUrl === 'import.meta') continue + + const hasQuotes = rawUrl[0] === '"' || rawUrl[0] === "'" + if (hasQuotes) { + rawUrl = rawUrl.slice(1, -1) + } - if (imp.d === -1 && server.config.server.preTransformRequests) { - // pre-transform known direct imports - environment.warmupRequest(hmrUrl) - } + const urlWithoutTimestamp = removeTimestampQuery(rawUrl) + // hmrUrl must be derived the same way as importAnalysis + const hmrUrl = unwrapId( + stripBase(removeImportQuery(urlWithoutTimestamp), server.config.base), + ) + for (const importedMod of mod.importedModules) { + if (importedMod.url !== hmrUrl) continue + if (importedMod.lastHMRTimestamp > 0) { + const replacedUrl = injectQuery( + urlWithoutTimestamp, + `t=${importedMod.lastHMRTimestamp}`, + ) + const start = hasQuotes ? imp.s + 1 : imp.s + const end = hasQuotes ? imp.e - 1 : imp.e + s.overwrite(start, end, replacedUrl) + } - break + if (imp.d === -1 && server.config.server.preTransformRequests) { + // pre-transform known direct imports + environment.warmupRequest(hmrUrl) } - } - // Update `transformResult` with new code. We don't have to update the sourcemap - // as the timestamp changes doesn't affect the code lines (stable). - const code = s.toString() - result = { - ...transformResult, - code, - etag: getEtag(code, { weak: true }), + break } } + // Update `transformResult` with new code. We don't have to update the sourcemap + // as the timestamp changes doesn't affect the code lines (stable). + const code = s.toString() + result = { + ...transformResult, + code, + etag: getEtag(code, { weak: true }), + } + // Only cache the result if the module wasn't invalidated while it was // being processed, so it is re-processed next time if it is stale if (timestamp > mod.lastInvalidationTimestamp) diff --git a/packages/vite/src/node/ssr/runtime/__tests__/server-source-maps.spec.ts b/packages/vite/src/node/ssr/runtime/__tests__/server-source-maps.spec.ts index d1ec332aadc342..cc97a44cc494ef 100644 --- a/packages/vite/src/node/ssr/runtime/__tests__/server-source-maps.spec.ts +++ b/packages/vite/src/node/ssr/runtime/__tests__/server-source-maps.spec.ts @@ -21,7 +21,7 @@ describe('module runner initialization', async () => { const serializeStack = (runner: ModuleRunner, err: Error) => { return err.stack!.split('\n')[1].replace(runner.options.root, '') } - const serializeStackDeep = (runtime: ViteRuntime, err: Error) => { + const serializeStackDeep = (runtime: ModuleRunner, err: Error) => { return err .stack!.split('\n') .map((s) => s.replace(runtime.options.root, '')) diff --git a/packages/vite/src/node/ssr/runtime/__tests__/utils.ts b/packages/vite/src/node/ssr/runtime/__tests__/utils.ts index cc764c3a91b120..9d52ffa0c7cbd6 100644 --- a/packages/vite/src/node/ssr/runtime/__tests__/utils.ts +++ b/packages/vite/src/node/ssr/runtime/__tests__/utils.ts @@ -9,10 +9,12 @@ import type { ViteDevServer } from '../../../server' import type { InlineConfig } from '../../../config' import { createServer } from '../../../server' import { createServerModuleRunner } from '../serverModuleRunner' +import type { DevEnvironment } from '../../../server/environment' interface TestClient { server: ViteDevServer runner: ModuleRunner + environment: DevEnvironment } export async function createModuleRunnerTester( @@ -73,7 +75,8 @@ export async function createModuleRunnerTester( ], ...config, }) - t.runner = await createServerModuleRunner(t.server.environments.ssr, { + t.environment = t.server.environments.ssr + t.runner = await createServerModuleRunner(t.environment, { hmr: { logger: false, }, diff --git a/packages/vite/src/node/ssr/ssrTransform.ts b/packages/vite/src/node/ssr/ssrTransform.ts index c3800a48a8d138..8e9c477fb587aa 100644 --- a/packages/vite/src/node/ssr/ssrTransform.ts +++ b/packages/vite/src/node/ssr/ssrTransform.ts @@ -2,8 +2,10 @@ import path from 'node:path' import MagicString from 'magic-string' import type { SourceMap } from 'rollup' import type { + CallExpression, Function as FunctionNode, Identifier, + Literal, Pattern, Property, VariableDeclaration, @@ -13,6 +15,7 @@ import { extract_names as extractNames } from 'periscopic' import { walk as eswalk } from 'estree-walker' import type { RawSourceMap } from '@ampproject/remapping' import { parseAstAsync as rollupParseAstAsync } from 'rollup/parseAst' +import type { ImportSpecifier } from 'es-module-lexer' import type { TransformResult } from '../server/transformRequest' import { combineSourcemaps, isDefined } from '../utils' import { isJSONRequest } from '../plugins/json' @@ -59,6 +62,7 @@ async function ssrTransformJSON( map: inMap, deps: [], dynamicDeps: [], + ssr: true, } } @@ -322,11 +326,58 @@ async function ssrTransformScript( return { code: s.toString(), map, + ssr: true, deps: [...deps], dynamicDeps: [...dynamicDeps], } } +export async function ssrParseImports( + url: string, + code: string, +): Promise { + let ast: any + try { + ast = await rollupParseAstAsync(code) + } catch (err) { + if (!err.loc || !err.loc.line) throw err + const line = err.loc.line + throw new Error( + `Parse failure: ${ + err.message + }\nAt file: ${url}\nContents of line ${line}: ${ + code.split('\n')[line - 1] + }`, + ) + } + const imports: ImportSpecifier[] = [] + eswalk(ast, { + enter(_n, parent) { + if (_n.type !== 'Identifier') return + const node = _n as Node & Identifier + const isStaticImport = node.name === ssrImportKey + const isDynamicImport = node.name === ssrDynamicImportKey + if (isStaticImport || isDynamicImport) { + // this is a standardised output, so we can safely assume the parent and arguments + const importExpression = parent as Node & CallExpression + const importLiteral = importExpression.arguments[0] as Node & Literal + + imports.push({ + n: importLiteral.value as string | undefined, + s: importLiteral.start, + e: importLiteral.end, + se: importExpression.start, + ss: importExpression.end, + t: isStaticImport ? 2 : 1, + d: isDynamicImport ? importLiteral.start : -1, + a: -1, // not used + }) + } + }, + }) + return imports +} + interface Visitors { onIdentifier: ( node: Identifier & {