From 5e16bdf78865ddabb14cb31e55af21807735d756 Mon Sep 17 00:00:00 2001 From: Scott Hovestadt Date: Sun, 24 Mar 2019 00:46:56 -0700 Subject: [PATCH 1/5] Dramatically improve watch mode performance. --- .../src/__tests__/getMaxWorkers.test.ts | 2 +- packages/jest-config/src/getMaxWorkers.ts | 2 +- packages/jest-haste-map/src/ModuleMap.ts | 20 ++++++--- .../src/__tests__/testRunner.test.js | 2 + packages/jest-runner/src/index.ts | 3 ++ packages/jest-runner/src/testWorker.ts | 42 ++++++++++++------- 6 files changed, 48 insertions(+), 23 deletions(-) diff --git a/packages/jest-config/src/__tests__/getMaxWorkers.test.ts b/packages/jest-config/src/__tests__/getMaxWorkers.test.ts index aab3290c5357..a4a92a419131 100644 --- a/packages/jest-config/src/__tests__/getMaxWorkers.test.ts +++ b/packages/jest-config/src/__tests__/getMaxWorkers.test.ts @@ -32,7 +32,7 @@ describe('getMaxWorkers', () => { it('Returns based on the number of cpus', () => { expect(getMaxWorkers({})).toBe(3); - expect(getMaxWorkers({watch: true})).toBe(2); + expect(getMaxWorkers({watch: true})).toBe(3); }); describe('% based', () => { diff --git a/packages/jest-config/src/getMaxWorkers.ts b/packages/jest-config/src/getMaxWorkers.ts index 29e350b7a367..edf67a5de5d7 100644 --- a/packages/jest-config/src/getMaxWorkers.ts +++ b/packages/jest-config/src/getMaxWorkers.ts @@ -32,6 +32,6 @@ export default function getMaxWorkers( return parsed > 0 ? parsed : 1; } else { const cpus = os.cpus() ? os.cpus().length : 1; - return Math.max(argv.watch ? Math.floor(cpus / 2) : cpus - 1, 1); + return Math.max(cpus - 1, 1); } } diff --git a/packages/jest-haste-map/src/ModuleMap.ts b/packages/jest-haste-map/src/ModuleMap.ts index 8b86aa2845e5..73f55c26c7b2 100644 --- a/packages/jest-haste-map/src/ModuleMap.ts +++ b/packages/jest-haste-map/src/ModuleMap.ts @@ -32,10 +32,15 @@ export type SerializableModuleMap = { }; export default class ModuleMap { + public readonly uniqueID: number; private readonly _raw: RawModuleMap; + private json: SerializableModuleMap | undefined; static DuplicateHasteCandidatesError: typeof DuplicateHasteCandidatesError; + private static nextUniqueID = 0; constructor(raw: RawModuleMap) { + this.uniqueID = ModuleMap.nextUniqueID; + ModuleMap.nextUniqueID++; this._raw = raw; } @@ -84,12 +89,15 @@ export default class ModuleMap { } toJSON(): SerializableModuleMap { - return { - duplicates: Array.from(this._raw.duplicates), - map: Array.from(this._raw.map), - mocks: Array.from(this._raw.mocks), - rootDir: this._raw.rootDir, - }; + if (!this.json) { + this.json = { + duplicates: Array.from(this._raw.duplicates), + map: Array.from(this._raw.map), + mocks: Array.from(this._raw.mocks), + rootDir: this._raw.rootDir, + }; + } + return this.json; } static fromJSON(serializableModuleMap: SerializableModuleMap) { diff --git a/packages/jest-runner/src/__tests__/testRunner.test.js b/packages/jest-runner/src/__tests__/testRunner.test.js index ca604b10c0ee..60a131ccfda1 100644 --- a/packages/jest-runner/src/__tests__/testRunner.test.js +++ b/packages/jest-runner/src/__tests__/testRunner.test.js @@ -89,6 +89,7 @@ test('does not inject the serializable module map in serial mode', () => { config, context: runContext, globalConfig, + moduleMapUniqueID: null, path: './file.test.js', serializableModuleMap: null, }, @@ -98,6 +99,7 @@ test('does not inject the serializable module map in serial mode', () => { config, context: runContext, globalConfig, + moduleMapUniqueID: null, path: './file2.test.js', serializableModuleMap: null, }, diff --git a/packages/jest-runner/src/index.ts b/packages/jest-runner/src/index.ts index 94a379592501..e7fcab71b3ad 100644 --- a/packages/jest-runner/src/index.ts +++ b/packages/jest-runner/src/index.ts @@ -134,6 +134,9 @@ class TestRunner { Array.from(this._context.changedFiles), }, globalConfig: this._globalConfig, + moduleMapUniqueID: watcher.isWatchMode() + ? test.context.moduleMap.uniqueID + : null, path: test.path, serializableModuleMap: watcher.isWatchMode() ? test.context.moduleMap.toJSON() diff --git a/packages/jest-runner/src/testWorker.ts b/packages/jest-runner/src/testWorker.ts index 6bb94790b204..7c2049a82f6f 100644 --- a/packages/jest-runner/src/testWorker.ts +++ b/packages/jest-runner/src/testWorker.ts @@ -8,10 +8,11 @@ import {Config} from '@jest/types'; import {SerializableError, TestResult} from '@jest/test-result'; -import HasteMap, {SerializableModuleMap, ModuleMap} from 'jest-haste-map'; +import HasteMap, {ModuleMap, SerializableModuleMap} from 'jest-haste-map'; import exit from 'exit'; import {separateMessageFromStack} from 'jest-message-util'; import Runtime from 'jest-runtime'; +import Resolver from 'jest-resolve'; import {ErrorWithCode, TestRunnerSerializedContext} from './types'; import runTest from './runTest'; @@ -20,6 +21,7 @@ type WorkerData = { globalConfig: Config.GlobalConfig; path: Config.Path; serializableModuleMap: SerializableModuleMap | null; + moduleMapUniqueID: number | null; context?: TestRunnerSerializedContext; }; @@ -47,7 +49,7 @@ const formatError = (error: string | ErrorWithCode): SerializableError => { }; }; -const resolvers = Object.create(null); +const resolvers = new Map(); const getResolver = ( config: Config.ProjectConfig, moduleMap: ModuleMap | null, @@ -56,31 +58,41 @@ const getResolver = ( // the test runner to the watch command. This is because jest-haste-map's // watch mode does not persist the haste map on disk after every file change. // To make this fast and consistent, we pass it from the TestRunner. - if (moduleMap) { - return Runtime.createResolver(config, moduleMap); - } else { - const name = config.name; - if (!resolvers[name]) { - resolvers[name] = Runtime.createResolver( + const name = config.name; + if (moduleMap || !resolvers.has(name)) { + resolvers.set( + name, + Runtime.createResolver( config, - Runtime.createHasteMap(config).readModuleMap(), - ); - } - return resolvers[name]; + moduleMap || Runtime.createHasteMap(config).readModuleMap(), + ), + ); } + return resolvers.get(name)!; }; +const deserializedModuleMaps = new Map(); export async function worker({ config, globalConfig, path, serializableModuleMap, + moduleMapUniqueID, context, }: WorkerData): Promise { try { - const moduleMap = serializableModuleMap - ? HasteMap.ModuleMap.fromJSON(serializableModuleMap) - : null; + // If the module map ID does not match what is currently being used by the + // config's resolver was passed, deserialize it and update the resolver. + let moduleMap: ModuleMap | null = null; + if ( + serializableModuleMap && + moduleMapUniqueID && + deserializedModuleMaps.get(config.name) !== moduleMapUniqueID + ) { + deserializedModuleMaps.set(config.name, moduleMapUniqueID); + moduleMap = HasteMap.ModuleMap.fromJSON(serializableModuleMap); + } + return await runTest( path, globalConfig, From 551f1d728cea3f2ab8941aa40062b8967aaabe28 Mon Sep 17 00:00:00 2001 From: Scott Hovestadt Date: Sun, 24 Mar 2019 09:36:46 -0700 Subject: [PATCH 2/5] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64737724c07b..126eb8bdcbe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - `[jest-haste-map]` Avoid persisting haste map or processing files when not changed ([#8153](https://github.com/facebook/jest/pull/8153)) - `[jest-core]` Improve performance of SearchSource.findMatchingTests by 15% ([#8184](https://github.com/facebook/jest/pull/8184)) - `[jest-resolve]` Optimize internal cache lookup performance ([#8183](https://github.com/facebook/jest/pull/8183)) +- `[jest-core]` Dramatically improve watch mode performance ([#8201](https://github.com/facebook/jest/pull/8201)) ## 24.5.0 From e0ec4257c7d007f4c15cdc83836a9466f2ae9d68 Mon Sep 17 00:00:00 2001 From: Scott Hovestadt Date: Sun, 24 Mar 2019 09:36:55 -0700 Subject: [PATCH 3/5] Module map unique ID must be truthy. --- packages/jest-haste-map/src/ModuleMap.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jest-haste-map/src/ModuleMap.ts b/packages/jest-haste-map/src/ModuleMap.ts index 73f55c26c7b2..f92d44f45dde 100644 --- a/packages/jest-haste-map/src/ModuleMap.ts +++ b/packages/jest-haste-map/src/ModuleMap.ts @@ -36,7 +36,7 @@ export default class ModuleMap { private readonly _raw: RawModuleMap; private json: SerializableModuleMap | undefined; static DuplicateHasteCandidatesError: typeof DuplicateHasteCandidatesError; - private static nextUniqueID = 0; + private static nextUniqueID = 1; // Must be truthy. constructor(raw: RawModuleMap) { this.uniqueID = ModuleMap.nextUniqueID; From 67ea28fca1fda3e6fb0df184f5d08e001a265c06 Mon Sep 17 00:00:00 2001 From: Scott Hovestadt Date: Sun, 24 Mar 2019 09:37:41 -0700 Subject: [PATCH 4/5] Static above instance vars. --- packages/jest-haste-map/src/ModuleMap.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jest-haste-map/src/ModuleMap.ts b/packages/jest-haste-map/src/ModuleMap.ts index f92d44f45dde..0d53b5a4bfa0 100644 --- a/packages/jest-haste-map/src/ModuleMap.ts +++ b/packages/jest-haste-map/src/ModuleMap.ts @@ -32,11 +32,11 @@ export type SerializableModuleMap = { }; export default class ModuleMap { + static DuplicateHasteCandidatesError: typeof DuplicateHasteCandidatesError; + private static nextUniqueID = 1; // Must be truthy. public readonly uniqueID: number; private readonly _raw: RawModuleMap; private json: SerializableModuleMap | undefined; - static DuplicateHasteCandidatesError: typeof DuplicateHasteCandidatesError; - private static nextUniqueID = 1; // Must be truthy. constructor(raw: RawModuleMap) { this.uniqueID = ModuleMap.nextUniqueID; From 68c2c438fb9693cc3dbebd9d716f7f88146dc752 Mon Sep 17 00:00:00 2001 From: Scott Hovestadt Date: Sun, 24 Mar 2019 13:27:22 -0700 Subject: [PATCH 5/5] Further improve performance by sending module maps during worker setup. --- packages/jest-haste-map/src/ModuleMap.ts | 4 -- .../src/__tests__/testRunner.test.js | 6 --- packages/jest-runner/src/index.ts | 28 ++++++++--- packages/jest-runner/src/testWorker.ts | 50 +++++++++---------- 4 files changed, 45 insertions(+), 43 deletions(-) diff --git a/packages/jest-haste-map/src/ModuleMap.ts b/packages/jest-haste-map/src/ModuleMap.ts index 0d53b5a4bfa0..73390e105092 100644 --- a/packages/jest-haste-map/src/ModuleMap.ts +++ b/packages/jest-haste-map/src/ModuleMap.ts @@ -33,14 +33,10 @@ export type SerializableModuleMap = { export default class ModuleMap { static DuplicateHasteCandidatesError: typeof DuplicateHasteCandidatesError; - private static nextUniqueID = 1; // Must be truthy. - public readonly uniqueID: number; private readonly _raw: RawModuleMap; private json: SerializableModuleMap | undefined; constructor(raw: RawModuleMap) { - this.uniqueID = ModuleMap.nextUniqueID; - ModuleMap.nextUniqueID++; this._raw = raw; } diff --git a/packages/jest-runner/src/__tests__/testRunner.test.js b/packages/jest-runner/src/__tests__/testRunner.test.js index 60a131ccfda1..18a1333137c7 100644 --- a/packages/jest-runner/src/__tests__/testRunner.test.js +++ b/packages/jest-runner/src/__tests__/testRunner.test.js @@ -51,7 +51,6 @@ test('injects the serializable module map into each worker in watch mode', () => context: runContext, globalConfig, path: './file.test.js', - serializableModuleMap, }, ], [ @@ -60,7 +59,6 @@ test('injects the serializable module map into each worker in watch mode', () => context: runContext, globalConfig, path: './file2.test.js', - serializableModuleMap, }, ], ]); @@ -89,9 +87,7 @@ test('does not inject the serializable module map in serial mode', () => { config, context: runContext, globalConfig, - moduleMapUniqueID: null, path: './file.test.js', - serializableModuleMap: null, }, ], [ @@ -99,9 +95,7 @@ test('does not inject the serializable module map in serial mode', () => { config, context: runContext, globalConfig, - moduleMapUniqueID: null, path: './file2.test.js', - serializableModuleMap: null, }, ], ]); diff --git a/packages/jest-runner/src/index.ts b/packages/jest-runner/src/index.ts index e7fcab71b3ad..10712b94af9e 100644 --- a/packages/jest-runner/src/index.ts +++ b/packages/jest-runner/src/index.ts @@ -11,7 +11,7 @@ import exit from 'exit'; import throat from 'throat'; import Worker from 'jest-worker'; import runTest from './runTest'; -import {worker} from './testWorker'; +import {worker, SerializableResolver} from './testWorker'; import { OnTestFailure, OnTestStart, @@ -103,11 +103,31 @@ class TestRunner { onResult: OnTestSuccess, onFailure: OnTestFailure, ) { + let resolvers: Map | undefined = undefined; + if (watcher.isWatchMode()) { + resolvers = new Map(); + for (const test of tests) { + if (!resolvers.has(test.context.config.name)) { + resolvers.set(test.context.config.name, { + config: test.context.config, + serializableModuleMap: test.context.moduleMap.toJSON(), + }); + } + } + } + const worker = new Worker(TEST_WORKER_PATH, { exposedMethods: ['worker'], forkOptions: {stdio: 'pipe'}, maxRetries: 3, numWorkers: this._globalConfig.maxWorkers, + setupArgs: resolvers + ? [ + { + serializableResolvers: Array.from(resolvers.values()), + }, + ] + : undefined, }) as WorkerInterface; if (worker.getStdout()) worker.getStdout().pipe(process.stdout); @@ -134,13 +154,7 @@ class TestRunner { Array.from(this._context.changedFiles), }, globalConfig: this._globalConfig, - moduleMapUniqueID: watcher.isWatchMode() - ? test.context.moduleMap.uniqueID - : null, path: test.path, - serializableModuleMap: watcher.isWatchMode() - ? test.context.moduleMap.toJSON() - : null, }); }); diff --git a/packages/jest-runner/src/testWorker.ts b/packages/jest-runner/src/testWorker.ts index 7c2049a82f6f..a2b5edb0fa6d 100644 --- a/packages/jest-runner/src/testWorker.ts +++ b/packages/jest-runner/src/testWorker.ts @@ -16,12 +16,15 @@ import Resolver from 'jest-resolve'; import {ErrorWithCode, TestRunnerSerializedContext} from './types'; import runTest from './runTest'; +export type SerializableResolver = { + config: Config.ProjectConfig; + serializableModuleMap: SerializableModuleMap; +}; + type WorkerData = { config: Config.ProjectConfig; globalConfig: Config.GlobalConfig; path: Config.Path; - serializableModuleMap: SerializableModuleMap | null; - moduleMapUniqueID: number | null; context?: TestRunnerSerializedContext; }; @@ -50,14 +53,7 @@ const formatError = (error: string | ErrorWithCode): SerializableError => { }; const resolvers = new Map(); -const getResolver = ( - config: Config.ProjectConfig, - moduleMap: ModuleMap | null, -) => { - // In watch mode, the raw module map with all haste modules is passed from - // the test runner to the watch command. This is because jest-haste-map's - // watch mode does not persist the haste map on disk after every file change. - // To make this fast and consistent, we pass it from the TestRunner. +const getResolver = (config: Config.ProjectConfig, moduleMap?: ModuleMap) => { const name = config.name; if (moduleMap || !resolvers.has(name)) { resolvers.set( @@ -71,33 +67,35 @@ const getResolver = ( return resolvers.get(name)!; }; -const deserializedModuleMaps = new Map(); +export function setup(setupData?: { + serializableResolvers: Array; +}) { + // Setup data is only used in watch mode to pass the latest version of all + // module maps that will be used during the test runs. Otherwise, module maps + // are loaded from disk as needed. + if (setupData) { + for (const { + config, + serializableModuleMap, + } of setupData.serializableResolvers) { + const moduleMap = HasteMap.ModuleMap.fromJSON(serializableModuleMap); + getResolver(config, moduleMap); + } + } +} + export async function worker({ config, globalConfig, path, - serializableModuleMap, - moduleMapUniqueID, context, }: WorkerData): Promise { try { - // If the module map ID does not match what is currently being used by the - // config's resolver was passed, deserialize it and update the resolver. - let moduleMap: ModuleMap | null = null; - if ( - serializableModuleMap && - moduleMapUniqueID && - deserializedModuleMaps.get(config.name) !== moduleMapUniqueID - ) { - deserializedModuleMaps.set(config.name, moduleMapUniqueID); - moduleMap = HasteMap.ModuleMap.fromJSON(serializableModuleMap); - } - return await runTest( path, globalConfig, config, - getResolver(config, moduleMap), + getResolver(config), context && { ...context, changedFiles: context.changedFiles && new Set(context.changedFiles),