Skip to content

Commit

Permalink
chore: cache specs
Browse files Browse the repository at this point in the history
  • Loading branch information
sheremet-va committed Aug 12, 2024
1 parent b477aa3 commit dbac2c8
Show file tree
Hide file tree
Showing 11 changed files with 106 additions and 64 deletions.
2 changes: 1 addition & 1 deletion packages/browser/src/node/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ export default (browserServer: BrowserServer, base = '/'): Plugin[] => {
name: 'vitest:browser:tests',
enforce: 'pre',
async config() {
const allTestFiles = await project.globTestFiles()
const { testFiles: allTestFiles } = await project.globTestFiles()
const browserTestFiles = allTestFiles.filter(
file => getFilePoolName(project, file) === 'browser',
)
Expand Down
4 changes: 2 additions & 2 deletions packages/browser/src/node/pool.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import * as nodeos from 'node:os'
import crypto from 'node:crypto'
import { relative } from 'pathe'
import type { BrowserProvider, ProcessPool, Vitest, WorkspaceProject } from 'vitest/node'
import type { BrowserProvider, ProcessPool, Vitest, WorkspaceProject, WorkspaceSpec } from 'vitest/node'
import { createDebugger } from 'vitest/node'

const debug = createDebugger('vitest:browser:pool')
Expand Down Expand Up @@ -92,7 +92,7 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
await Promise.all(promises)
}

const runWorkspaceTests = async (method: 'run' | 'collect', specs: [WorkspaceProject, string][]) => {
const runWorkspaceTests = async (method: 'run' | 'collect', specs: WorkspaceSpec[]) => {
const groupedFiles = new Map<WorkspaceProject, string[]>()
for (const [project, file] of specs) {
const files = groupedFiles.get(project) || []
Expand Down
2 changes: 1 addition & 1 deletion packages/browser/src/node/serverTester.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export async function resolveTester(
const { contextId, testFile } = server.resolveTesterUrl(url.pathname)
const project = server.project
const state = server.state
const testFiles = await project.globTestFiles()
const { testFiles } = await project.globTestFiles()
// if decoded test file is "__vitest_all__" or not in the list of known files, run all tests
const tests
= testFile === '__vitest_all__'
Expand Down
3 changes: 2 additions & 1 deletion packages/vitest/src/api/setup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,13 @@ export function setup(ctx: Vitest, _server?: ViteDevServer) {
},
async getTestFiles() {
const spec = await ctx.globTestFiles()
return spec.map(([project, file]) => [
return spec.map(([project, file, options]) => [
{
name: project.config.name,
root: project.config.root,
},
file,
options,
])
},
},
Expand Down
87 changes: 65 additions & 22 deletions packages/vitest/src/node/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { WebSocketReporter } from '../api/setup'
import type { SerializedCoverageConfig } from '../runtime/config'
import type { SerializedSpec } from '../runtime/types/utils'
import type { ArgumentsType, OnServerRestartHandler, ProvidedContext, UserConsoleLog } from '../types/general'
import { createPool } from './pool'
import { createPool, getFilePoolName } from './pool'
import type { ProcessPool, WorkspaceSpec } from './pool'
import { createBenchmarkReporters, createReporters } from './reporters/utils'
import { StateManager } from './state'
Expand Down Expand Up @@ -77,10 +77,14 @@ export class Vitest {

private resolvedProjects: WorkspaceProject[] = []
public projects: WorkspaceProject[] = []
private projectsTestFiles = new Map<string, Set<WorkspaceProject>>()

public distPath!: string

private _cachedSpecs = new Map<string, WorkspaceSpec[]>()

/** @deprecated use `_cachedSpecs` */
projectTestFiles = this._cachedSpecs

constructor(
public readonly mode: VitestRunMode,
options: VitestOptions = {},
Expand All @@ -103,7 +107,7 @@ export class Vitest {
this.coverageProvider = undefined
this.runningPromise = undefined
this.distPath = undefined!
this.projectsTestFiles.clear()
this._cachedSpecs.clear()

const resolved = resolveConfig(this.mode, options, server.config, this.logger)

Expand Down Expand Up @@ -202,6 +206,9 @@ export class Vitest {
return this.coreWorkspaceProject
}

/**
* @deprecated use Reported Task API instead
*/
public getProjectByTaskId(taskId: string): WorkspaceProject {
const task = this.state.idMap.get(taskId)
const projectName = (task as File).projectName || task?.file?.projectName || ''
Expand All @@ -216,7 +223,7 @@ export class Vitest {
|| this.projects[0]
}

private async getWorkspaceConfigPath() {
private async getWorkspaceConfigPath(): Promise<string | null> {
if (this.config.workspace) {
return this.config.workspace
}
Expand Down Expand Up @@ -423,8 +430,8 @@ export class Vitest {
}
}

private async getTestDependencies(filepath: WorkspaceSpec, deps = new Set<string>()) {
const addImports = async ([project, filepath]: WorkspaceSpec) => {
private async getTestDependencies([project, filepath]: WorkspaceSpec, deps = new Set<string>()) {
const addImports = async (project: WorkspaceProject, filepath: string) => {
if (deps.has(filepath)) {
return
}
Expand All @@ -440,13 +447,13 @@ export class Vitest {
const path = await project.server.pluginContainer.resolveId(dep, filepath, { ssr: true })
const fsPath = path && !path.external && path.id.split('?')[0]
if (fsPath && !fsPath.includes('node_modules') && !deps.has(fsPath) && existsSync(fsPath)) {
await addImports([project, fsPath])
await addImports(project, fsPath)
}
}))
}

await addImports(filepath)
deps.delete(filepath[1])
await addImports(project, filepath)
deps.delete(filepath)

return deps
}
Expand Down Expand Up @@ -500,12 +507,31 @@ export class Vitest {
return runningTests
}

/**
* @deprecated remove when vscode extension supports "getFileWorkspaceSpecs"
*/
getProjectsByTestFile(file: string) {
const projects = this.projectsTestFiles.get(file)
if (!projects) {
return []
return this.getFileWorkspaceSpecs(file)
}

getFileWorkspaceSpecs(file: string) {
const _cached = this._cachedSpecs.get(file)
if (_cached) {
return _cached
}
return Array.from(projects).map(project => [project, file] as WorkspaceSpec)

const specs: WorkspaceSpec[] = []
for (const project of this.projects) {
if (project.isTestFile(file)) {
const pool = getFilePoolName(project, file)
specs.push([project, file, { pool }])
}
if (project.isTypecheckFile(file)) {
specs.push([project, file, { pool: 'typescript' }])
}
}
specs.forEach(spec => this.ensureSpecCached(spec))
return specs
}

async initializeGlobalSetup(paths: WorkspaceSpec[]) {
Expand Down Expand Up @@ -538,8 +564,11 @@ export class Vitest {

await this.report('onPathsCollected', filepaths)
await this.report('onSpecsCollected', specs.map(
([project, file]) =>
[{ name: project.config.name, root: project.config.root }, file] as SerializedSpec,
([project, file, options]) =>
[{
name: project.config.name,
root: project.config.root,
}, file, options] satisfies SerializedSpec,
))

// previous run
Expand Down Expand Up @@ -856,7 +885,6 @@ export class Vitest {
}))

if (matchingProjects.length > 0) {
this.projectsTestFiles.set(id, new Set(matchingProjects))
this.changedTests.add(id)
this.scheduleRerun([id])
}
Expand Down Expand Up @@ -1054,17 +1082,32 @@ export class Vitest {
public async globTestFiles(filters: string[] = []) {
const files: WorkspaceSpec[] = []
await Promise.all(this.projects.map(async (project) => {
const specs = await project.globTestFiles(filters)
specs.forEach((file) => {
files.push([project, file])
const projects = this.projectsTestFiles.get(file) || new Set()
projects.add(project)
this.projectsTestFiles.set(file, projects)
const { testFiles, typecheckTestFiles } = await project.globTestFiles(filters)
testFiles.forEach((file) => {
const pool = getFilePoolName(project, file)
const spec: WorkspaceSpec = [project, file, { pool }]
this.ensureSpecCached(spec)
files.push(spec)
})
typecheckTestFiles.forEach((file) => {
const spec: WorkspaceSpec = [project, file, { pool: 'typecheck' }]
this.ensureSpecCached(spec)
files.push(spec)
})
}))
return files
}

private ensureSpecCached(spec: WorkspaceSpec) {
const file = spec[1]
const specs = this._cachedSpecs.get(file) || []
const included = specs.some(_s => _s[0] === spec[0] && _s[2].pool === spec[2].pool)
if (!included) {
specs.push(spec)
this._cachedSpecs.set(file, specs)
}
}

// The server needs to be running for communication
shouldKeepServer() {
return !!this.config?.watch
Expand Down
35 changes: 9 additions & 26 deletions packages/vitest/src/node/pool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import type { WorkspaceProject } from './workspace'
import { createTypecheckPool } from './pools/typecheck'
import { createVmForksPool } from './pools/vmForks'

export type WorkspaceSpec = [project: WorkspaceProject, testFile: string]
export type WorkspaceSpec = [project: WorkspaceProject, testFile: string, options: { pool: Pool }]
export type RunWithFiles = (
files: WorkspaceSpec[],
invalidates?: string[]
Expand Down Expand Up @@ -39,16 +39,11 @@ export const builtinPools: BuiltinPool[] = [
'typescript',
]

function getDefaultPoolName(project: WorkspaceProject, file: string): Pool | null {
for (const glob of project.config.include) {
if (mm.isMatch(file, glob, { cwd: project.config.root, ignore: project.config.exclude })) {
if (project.config.browser.enabled) {
return 'browser'
}
return project.config.pool
}
function getDefaultPoolName(project: WorkspaceProject): Pool {
if (project.config.browser.enabled) {
return 'browser'
}
return null
return project.config.pool
}

export function getFilePoolName(project: WorkspaceProject, file: string) {
Expand All @@ -62,7 +57,7 @@ export function getFilePoolName(project: WorkspaceProject, file: string) {
return pool as Pool
}
}
return getDefaultPoolName(project, file)
return getDefaultPoolName(project)
}

export function createPool(ctx: Vitest): ProcessPool {
Expand Down Expand Up @@ -170,21 +165,9 @@ export function createPool(ctx: Vitest): ProcessPool {
}

for (const spec of files) {
const [project, file] = spec
const pool = getFilePoolName(project, file)
if (pool != null) {
filesByPool[pool] ??= []
filesByPool[pool].push(spec)
}

if (project.config.typecheck.enabled) {
for (const glob of project.config.typecheck.include) {
if (mm.isMatch(file, glob, { cwd: project.config.root, ignore: project.config.typecheck.exclude })) {
filesByPool.typescript ??= []
filesByPool.typescript.push(spec)
}
}
}
const { pool } = spec[2]
filesByPool[pool] ??= []
filesByPool[pool].push(spec)
}

const Sequencer = ctx.config.sequence.sequencer
Expand Down
4 changes: 2 additions & 2 deletions packages/vitest/src/node/reporters/renderers/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,6 @@ export function formatProjectName(name: string | undefined, suffix = ' ') {
const index = name
.split('')
.reduce((acc, v, idx) => acc + v.charCodeAt(0) + idx, 0)
const colors = [c.blue, c.yellow, c.cyan, c.green, c.magenta]
return colors[index % colors.length](`|${name}|`) + suffix
const colors = [c.bgBlue, c.bgYellow, c.bgCyan, c.bgGreen, c.bgMagenta]
return colors[index % colors.length](` ${c.white(name)} `) + suffix
}
26 changes: 20 additions & 6 deletions packages/vitest/src/node/workspace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ export class WorkspaceProject {
closingPromise: Promise<unknown> | undefined

testFilesList: string[] | null = null
typecheckFilesList: string[] | null = null

public testProject!: TestProject

Expand Down Expand Up @@ -225,15 +226,24 @@ export class WorkspaceProject {
? []
: this.globAllTestFiles(include, exclude, includeSource, dir),
typecheck.enabled
? this.globFiles(typecheck.include, typecheck.exclude, dir)
? (this.typecheckFilesList || this.globFiles(typecheck.include, typecheck.exclude, dir))
: [],
])

return this.filterFiles(
[...testFiles, ...typecheckTestFiles],
filters,
dir,
)
this.typecheckFilesList = typecheckTestFiles

return {
testFiles: this.filterFiles(
testFiles,
filters,
dir,
),
typecheckTestFiles: this.filterFiles(
typecheckTestFiles,
filters,
dir,
),
}
}

async globAllTestFiles(
Expand Down Expand Up @@ -275,6 +285,10 @@ export class WorkspaceProject {
return this.testFilesList && this.testFilesList.includes(id)
}

isTypecheckFile(id: string) {
return this.typecheckFilesList && this.typecheckFilesList.includes(id)
}

async globFiles(include: string[], exclude: string[], cwd: string) {
const globOptions: fg.Options = {
dot: true,
Expand Down
1 change: 1 addition & 0 deletions packages/vitest/src/runtime/types/utils.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export type SerializedSpec = [
project: { name: string | undefined; root: string },
file: string,
options: { pool: string },
]
4 changes: 2 additions & 2 deletions packages/vitest/src/utils/test-helpers.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { promises as fs } from 'node:fs'
import mm from 'micromatch'
import type { WorkspaceProject } from '../node/workspace'
import type { EnvironmentOptions, TransformModePatterns, VitestEnvironment } from '../node/types/config'
import type { ContextTestEnvironment } from '../types/worker'
import type { WorkspaceSpec } from '../node/pool'
import { groupBy } from './base'

export const envsOrder = ['node', 'jsdom', 'happy-dom', 'edge-runtime']
Expand All @@ -27,7 +27,7 @@ function getTransformMode(
}

export async function groupFilesByEnv(
files: (readonly [WorkspaceProject, string])[],
files: Array<WorkspaceSpec>,
) {
const filesWithEnv = await Promise.all(
files.map(async ([project, file]) => {
Expand Down
2 changes: 1 addition & 1 deletion test/core/test/sequencers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ function buildWorkspace() {
const workspace = buildWorkspace()

function workspaced(files: string[]) {
return files.map(file => [workspace, file] as WorkspaceSpec)
return files.map(file => [workspace, file, { pool: 'forks' }] satisfies WorkspaceSpec)
}

describe('base sequencer', () => {
Expand Down

0 comments on commit dbac2c8

Please sign in to comment.