Skip to content

Commit

Permalink
feat: add an option to enable Vite optimizer (#2912)
Browse files Browse the repository at this point in the history
  • Loading branch information
sheremet-va authored Feb 25, 2023
1 parent 610b1d4 commit af8de36
Show file tree
Hide file tree
Showing 17 changed files with 178 additions and 61 deletions.
22 changes: 22 additions & 0 deletions docs/config/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,28 @@ Files to exclude from the test run, using glob pattern.

Handling for dependencies resolution.

#### deps.experimentalOptimizer

- **Type:** `DepOptimizationConfig & { enabled: boolean }`
- **Version:** Vitets 0.29.0
- **See also:** [Dep Optimization Options](https://vitejs.dev/config/dep-optimization-options.html)

Enable dependency optimization. If you have a lot of tests, this might improve their performance.

For `jsdom` and `happy-dom` environments, when Vitest will encounter the external library, it will be bundled into a single file using esbuild and imported as a whole module. This is good for several reasons:

- Importing packages with a lot of imports is expensive. By bundling them into one file we can save a lot of time
- Importing UI libraries is expensive because they are not meant to run inside Node.js
- Your `alias` configuration is now respected inside bundled packages

You can opt-out of this behavior for certain packages with `exclude` option. You can read more about available options in [Vite](https://vitejs.dev/config/dep-optimization-options.html) docs.

This options also inherits your `optimizeDeps` configuration. If you redefine `include`/`exclude`/`entries` option in `deps.experimentalOptimizer` it will overwrite your `optimizeDeps` when running tests.

:::note
You will not be able to edit your `node_modules` code for debugging, since the code is actually located in your `cacheDir` or `test.cache.dir` directory. If you want to debug with `console.log` statements, edit it directly or force rebundling with `deps.experimentalOptimizer.force` option.
:::

#### deps.external

- **Type:** `(string | RegExp)[]`
Expand Down
8 changes: 4 additions & 4 deletions packages/vite-node/src/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -204,18 +204,18 @@ export class ViteNodeRunner {
return !isInternalRequest(id) && !isNodeBuiltin(id)
}

private async _resolveUrl(id: string, importee?: string): Promise<[url: string, fsPath: string]> {
private async _resolveUrl(id: string, importer?: string): Promise<[url: string, fsPath: string]> {
// we don't pass down importee here, because otherwise Vite doesn't resolve it correctly
// should be checked before normalization, because it removes this prefix
if (importee && id.startsWith(VALID_ID_PREFIX))
importee = undefined
if (importer && id.startsWith(VALID_ID_PREFIX))
importer = undefined
id = normalizeRequestId(id, this.options.base)
if (!this.shouldResolveId(id))
return [id, id]
const { path, exists } = toFilePath(id, this.root)
if (!this.options.resolveId || exists)
return [id, path]
const resolved = await this.options.resolveId(id, importee)
const resolved = await this.options.resolveId(id, importer)
const resolvedId = resolved
? normalizeRequestId(resolved.id, this.options.base)
: id
Expand Down
4 changes: 4 additions & 0 deletions packages/vite-node/src/externalize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,10 @@ async function _shouldExternalize(

id = patchWindowsImportPath(id)

// always externalize Vite deps, they are too big to inline
if (options?.cacheDir && id.includes(options.cacheDir))
return id

if (matchExternalizePattern(id, options?.inline))
return false
if (matchExternalizePattern(id, options?.external))
Expand Down
57 changes: 46 additions & 11 deletions packages/vite-node/src/server.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { performance } from 'node:perf_hooks'
import { resolve } from 'pathe'
import { existsSync } from 'node:fs'
import { join, relative, resolve } from 'pathe'
import type { TransformResult, ViteDevServer } from 'vite'
import createDebug from 'debug'
import type { EncodedSourceMap } from '@jridgewell/trace-mapping'
Expand All @@ -17,6 +18,8 @@ export class ViteNodeServer {
private fetchPromiseMap = new Map<string, Promise<FetchResult>>()
private transformPromiseMap = new Map<string, Promise<TransformResult | null | undefined>>()

private existingOptimizedDeps = new Set<string>()

fetchCache = new Map<string, {
duration?: number
timestamp: number
Expand All @@ -34,9 +37,12 @@ export class ViteNodeServer {
// eslint-disable-next-line @typescript-eslint/prefer-ts-expect-error
// @ts-ignore ssr is not typed in Vite 2, but defined in Vite 3, so we can't use expect-error
const ssrOptions = server.config.ssr
if (ssrOptions) {
options.deps ??= {}

options.deps ??= {}

options.deps.cacheDir = relative(server.config.root, server.config.cacheDir)

if (ssrOptions) {
// we don't externalize ssr, because it has different semantics in Vite
// if (ssrOptions.external) {
// options.deps.external ??= []
Expand Down Expand Up @@ -65,10 +71,26 @@ export class ViteNodeServer {
return shouldExternalize(id, this.options.deps, this.externalizeCache)
}

async resolveId(id: string, importer?: string): Promise<ViteNodeResolveId | null> {
private async ensureExists(id: string): Promise<boolean> {
if (this.existingOptimizedDeps.has(id))
return true
if (existsSync(id)) {
this.existingOptimizedDeps.add(id)
return true
}
return new Promise<boolean>((resolve) => {
setTimeout(() => {
this.ensureExists(id).then(() => {
resolve(true)
})
})
})
}

async resolveId(id: string, importer?: string, transformMode?: 'web' | 'ssr'): Promise<ViteNodeResolveId | null> {
if (importer && !importer.startsWith(this.server.config.root))
importer = resolve(this.server.config.root, importer)
const mode = (importer && this.getTransformMode(importer)) || 'ssr'
const mode = transformMode ?? ((importer && this.getTransformMode(importer)) || 'ssr')
return this.server.pluginContainer.resolveId(id, importer, { ssr: mode === 'ssr' })
}

Expand All @@ -80,12 +102,12 @@ export class ViteNodeServer {
return (ssrTransformResult?.map || null) as unknown as EncodedSourceMap | null
}

async fetchModule(id: string): Promise<FetchResult> {
async fetchModule(id: string, transformMode?: 'web' | 'ssr'): Promise<FetchResult> {
id = normalizeModuleId(id)
// reuse transform for concurrent requests
if (!this.fetchPromiseMap.has(id)) {
this.fetchPromiseMap.set(id,
this._fetchModule(id)
this._fetchModule(id, transformMode)
.then((r) => {
return this.options.sourcemap !== true ? { ...r, map: undefined } : r
})
Expand Down Expand Up @@ -123,9 +145,20 @@ export class ViteNodeServer {
return 'web'
}

private async _fetchModule(id: string): Promise<FetchResult> {
private async _fetchModule(id: string, transformMode?: 'web' | 'ssr'): Promise<FetchResult> {
let result: FetchResult

const cacheDir = this.options.deps?.cacheDir

if (cacheDir && id.includes(cacheDir) && !id.includes(this.server.config.root)) {
id = join(this.server.config.root, id)
const timeout = setTimeout(() => {
throw new Error(`ViteNodeServer: ${id} not found. This is a bug, please report it.`)
}, 5000) // CI can be quite slow
await this.ensureExists(id)
clearTimeout(timeout)
}

const { path: filePath } = toFilePath(id, this.server.config.root)

const module = this.server.moduleGraph.getModuleById(id)
Expand All @@ -143,7 +176,7 @@ export class ViteNodeServer {
}
else {
const start = performance.now()
const r = await this._transformRequest(id)
const r = await this._transformRequest(id, transformMode)
duration = performance.now() - start
result = { code: r?.code, map: r?.map as any }
}
Expand All @@ -157,7 +190,7 @@ export class ViteNodeServer {
return result
}

private async _transformRequest(id: string) {
private async _transformRequest(id: string, customTransformMode?: 'web' | 'ssr') {
debugRequest(id)

let result: TransformResult | null = null
Expand All @@ -168,7 +201,9 @@ export class ViteNodeServer {
return result
}

if (this.getTransformMode(id) === 'web') {
const transformMode = customTransformMode ?? this.getTransformMode(id)

if (transformMode === 'web') {
// for components like Vue, we want to use the client side
// plugins but then convert the code to be consumed by the server
result = await this.server.transformRequest(id)
Expand Down
1 change: 1 addition & 0 deletions packages/vite-node/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export type Arrayable<T> = T | Array<T>
export interface DepsHandlingOptions {
external?: (string | RegExp)[]
inline?: (string | RegExp)[] | true
cacheDir?: string
/**
* Try to guess the CJS version of a package when it's invalid ESM
* @default false
Expand Down
49 changes: 33 additions & 16 deletions packages/vitest/src/node/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,9 @@ export class Vitest {
}

async typecheck(filters: string[] = []) {
const { dir, root } = this.config
const { include, exclude } = this.config.typecheck
const testsFilesList = await this.globFiles(filters, include, exclude)
const testsFilesList = this.filterFiles(await this.globFiles(include, exclude, dir || root), filters)
const checker = new Typechecker(this, testsFilesList)
this.typechecker = checker
checker.onParseEnd(async ({ files, sourceErrors }) => {
Expand Down Expand Up @@ -606,32 +607,26 @@ export class Vitest {
)))
}

async globFiles(filters: string[], include: string[], exclude: string[]) {
async globFiles(include: string[], exclude: string[], cwd: string) {
const globOptions: fg.Options = {
absolute: true,
dot: true,
cwd: this.config.dir || this.config.root,
cwd,
ignore: exclude,
}

let testFiles = await fg(include, globOptions)

if (filters.length && process.platform === 'win32')
filters = filters.map(f => toNamespacedPath(f))

if (filters.length)
testFiles = testFiles.filter(i => filters.some(f => i.includes(f)))

return testFiles
return fg(include, globOptions)
}

async globTestFiles(filters: string[] = []) {
const { include, exclude, includeSource } = this.config
private _allTestsCache: string[] | null = null

async globAllTestFiles(config: ResolvedConfig, cwd: string) {
const { include, exclude, includeSource } = config

const testFiles = await this.globFiles(filters, include, exclude)
const testFiles = await this.globFiles(include, exclude, cwd)

if (includeSource) {
const files = await this.globFiles(filters, includeSource, exclude)
const files = await this.globFiles(includeSource, exclude, cwd)

await Promise.all(files.map(async (file) => {
try {
Expand All @@ -645,9 +640,31 @@ export class Vitest {
}))
}

this._allTestsCache = testFiles

return testFiles
}

filterFiles(testFiles: string[], filters: string[] = []) {
if (filters.length && process.platform === 'win32')
filters = filters.map(f => toNamespacedPath(f))

if (filters.length)
return testFiles.filter(i => filters.some(f => i.includes(f)))

return testFiles
}

async globTestFiles(filters: string[] = []) {
const { dir, root } = this.config

const testFiles = this._allTestsCache ?? await this.globAllTestFiles(this.config, dir || root)

this._allTestsCache = null

return this.filterFiles(testFiles, filters)
}

async isTargetFile(id: string, source?: string): Promise<boolean> {
const relativeId = relative(this.config.dir || this.config.root, id)
if (mm.isMatch(relativeId, this.config.exclude))
Expand Down
3 changes: 2 additions & 1 deletion packages/vitest/src/node/create.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ export async function createVitest(mode: VitestRunMode, options: UserConfig, vit

const server = await createServer(mergeConfig(config, mergeConfig(viteOverrides, { root: options.root })))

if (ctx.config.api?.port)
// optimizer needs .listen() to be called
if (ctx.config.api?.port || ctx.config.deps?.experimentalOptimizer?.enabled)
await server.listen()
else
await server.pluginContainer.buildStart({})
Expand Down
36 changes: 29 additions & 7 deletions packages/vitest/src/node/plugins/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest('t
options() {
this.meta.watchMode = false
},
config(viteConfig: any) {
async config(viteConfig: any) {
// preliminary merge of options to be able to create server options for vite
// however to allow vitest plugins to modify vitest config values
// this is repeated in configResolved where the config is final
Expand Down Expand Up @@ -131,15 +131,37 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest('t
}

if (!options.browser) {
// disable deps optimization
Object.assign(config, {
cacheDir: undefined,
optimizeDeps: {
const optimizeConfig: Partial<ViteConfig> = {}
const optimizer = preOptions.deps?.experimentalOptimizer
if (!optimizer?.enabled) {
optimizeConfig.cacheDir = undefined
optimizeConfig.optimizeDeps = {
// experimental in Vite >2.9.2, entries remains to help with older versions
disabled: true,
entries: [],
},
})
}
}
else {
const entries = await ctx.globAllTestFiles(preOptions as ResolvedConfig, preOptions.dir || getRoot())
optimizeConfig.cacheDir = preOptions.cache?.dir ?? 'node_modules/.vitest'
optimizeConfig.optimizeDeps = {
...viteConfig.optimizeDeps,
...optimizer,
disabled: false,
entries: [...(optimizer.entries || viteConfig.optimizeDeps?.entries || []), ...entries],
exclude: ['vitest', ...(optimizer.exclude || viteConfig.optimizeDeps?.exclude || [])],
include: (optimizer.include || viteConfig.optimizeDeps?.include || []).filter((n: string) => n !== 'vitest'),
}
// Vite throws an error that it cannot rename "deps_temp", but optimization still works
// let's not show this error to users
const { error: logError } = console
console.error = (...args) => {
if (typeof args[0] === 'string' && args[0].includes('/deps_temp'))
return
return logError(...args)
}
}
Object.assign(config, optimizeConfig)
}

return config
Expand Down
12 changes: 7 additions & 5 deletions packages/vitest/src/node/pool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { createBirpc } from 'birpc'
import type { RawSourceMap } from 'vite-node'
import type { ResolvedConfig, WorkerContext, WorkerRPC, WorkerTestEnvironment } from '../types'
import { distDir, rootDir } from '../constants'
import { AggregateError, groupBy } from '../utils'
import { AggregateError, getEnvironmentTransformMode, groupBy } from '../utils'
import { envsOrder, groupFilesByEnv } from '../utils/test-helpers'
import type { Vitest } from './core'

Expand Down Expand Up @@ -195,11 +195,13 @@ function createChannel(ctx: Vitest) {
const r = await ctx.vitenode.transformRequest(id)
return r?.map as RawSourceMap | undefined
},
fetch(id) {
return ctx.vitenode.fetchModule(id)
fetch(id, environment) {
const transformMode = getEnvironmentTransformMode(ctx.config, environment)
return ctx.vitenode.fetchModule(id, transformMode)
},
resolveId(id, importer) {
return ctx.vitenode.resolveId(id, importer)
resolveId(id, importer, environment) {
const transformMode = getEnvironmentTransformMode(ctx.config, environment)
return ctx.vitenode.resolveId(id, importer, transformMode)
},
onPathsCollected(paths) {
ctx.state.collectPaths(paths)
Expand Down
8 changes: 4 additions & 4 deletions packages/vitest/src/runtime/execute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@ export class VitestExecutor extends ViteNodeRunner {
return environment === 'node' ? !isNodeBuiltin(id) : !id.startsWith('node:')
}

async resolveUrl(id: string, importee?: string) {
if (importee && importee.startsWith('mock:'))
importee = importee.slice(5)
return super.resolveUrl(id, importee)
async resolveUrl(id: string, importer?: string) {
if (importer && importer.startsWith('mock:'))
importer = importer.slice(5)
return super.resolveUrl(id, importer)
}

async dependencyRequest(id: string, fsPath: string, callstack: string[]): Promise<any> {
Expand Down
2 changes: 1 addition & 1 deletion packages/vitest/src/runtime/loader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ export const resolve: Resolver = async (url, context, next) => {

const id = normalizeModuleId(url)
const importer = normalizeModuleId(parentURL)
const resolved = await resolver(id, importer)
const resolved = await resolver(id, importer, state.ctx.environment.name)

let result: ResolveResult
let filepath: string
Expand Down
Loading

0 comments on commit af8de36

Please sign in to comment.