TestSequencer heuristic using dependency tree file sizes

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### Features
 
+- `[jest-resolve-dependencies]` Add `DependencyResolver.resolveRecursive` ([#7553](https://github.com/facebook/jest/pull/7553))
+- `[jest-resolve-dependencies]` Add `DependencyResolver includeCoreModules` option ([#7553](https://github.com/facebook/jest/pull/7553))
+
 ### Fixes
 
 - `[jest-cli]` Refactor `-o` and `--coverage` combined ([#7611](https://github.com/facebook/jest/pull/7611))
@@ -20,6 +23,8 @@
 
 ### Performance
 
+- `[jest-cli]` Better test order heuristics on first run without cache ([#7553](https://github.com/facebook/jest/pull/7553))
+
 ## 24.1.0
 
 ### Features

e2e/__tests__/__snapshots__/listTests.test.js.snap

@@ -3,6 +3,7 @@
 exports[`--listTests flag causes tests to be printed in different lines 1`] = `
 /MOCK_ABOLUTE_PATH/e2e/list-tests/__tests__/dummy.test.js
 /MOCK_ABOLUTE_PATH/e2e/list-tests/__tests__/other.test.js
+/MOCK_ABOLUTE_PATH/e2e/list-tests/__tests__/with-dep.test.js
 `;
 
-exports[`--listTests flag causes tests to be printed out as JSON when using the --json flag 1`] = `["/MOCK_ABOLUTE_PATH/e2e/list-tests/__tests__/dummy.test.js","/MOCK_ABOLUTE_PATH/e2e/list-tests/__tests__/other.test.js"]`;
+exports[`--listTests flag causes tests to be printed out as JSON when using the --json flag 1`] = `["/MOCK_ABOLUTE_PATH/e2e/list-tests/__tests__/dummy.test.js","/MOCK_ABOLUTE_PATH/e2e/list-tests/__tests__/other.test.js","/MOCK_ABOLUTE_PATH/e2e/list-tests/__tests__/with-dep.test.js"]`;

e2e/__tests__/listTests.test.js

@@ -35,6 +35,16 @@ describe('--listTests flag', () => {
     ).toMatchSnapshot();
   });
 
+  it('prints tests in the execution order determined by their dependency sizes', () => {
+    const {status, stdout} = runJest('list-tests', ['--listTests']);
+    expect(status).toBe(0);
+    expect(normalizePaths(stdout).split('\n')).toEqual([
+      expect.stringContaining('/with-dep.test.js'),
+      expect.stringContaining('/other.test.js'),
+      expect.stringContaining('/dummy.test.js'),
+    ]);
+  });
+
   it('causes tests to be printed out as JSON when using the --json flag', () => {
     const {status, stdout} = runJest('list-tests', ['--listTests', '--json']);
 

e2e/list-tests/__tests__/other.test.js

@@ -11,3 +11,6 @@ it("isn't actually run", () => {
   // (because it is only used for --listTests)
   expect(true).toBe(false);
 });
+
+// Because of this comment, other.test.js is slightly larger than dummy.test.js.
+// This matters for the order in which tests are sequenced.

e2e/list-tests/__tests__/with-dep.test.js

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+require('./dummy.test.js');
+
+it("isn't actually run", () => {
+  // (because it is only used for --listTests)
+  expect(true).toBe(false);
+});

packages/jest-cli/src/TestSequencer.js

@@ -10,10 +10,15 @@
 import type {AggregatedResult} from 'types/TestResult';
 import type {Context} from 'types/Context';
 import type {Test} from 'types/TestRunner';
+import type {Path} from 'types/Config';
+import type {Resolver} from 'types/Resolve';
+import type {HasteFS} from 'types/HasteMap';
 
 import fs from 'fs';
 // $FlowFixMe: Missing ESM export
 import {getCacheFilePath} from 'jest-haste-map';
+import DependencyResolver from 'jest-resolve-dependencies';
+import {buildSnapshotResolver} from 'jest-snapshot';
 
 const FAIL = 0;
 const SUCCESS = 1;
@@ -22,6 +27,17 @@ type Cache = {
   [key: string]: [0 | 1, number],
 };
 
+// If a test depends on one of these core modules,
+// we assume that the test may be slower because it is an "integration test"
+// that spawn child processes, accesses the file system, etc.
+// The weights are in bytes, just as the file sizes of regular modules.
+const coreModuleWeights = {
+  child_process: 1000000,
+  fs: 100000,
+  http: 10000,
+  https: 10000,
+};
+
 export default class TestSequencer {
   _cache: Map<Context, Cache>;
 
@@ -57,6 +73,46 @@ export default class TestSequencer {
     return cache;
   }
 
+  _fileSize(
+    path: Path,
+    sizes: Map<Path, number>,
+    resolver: Resolver,
+    hasteFS: HasteFS,
+  ): number {
+    const cachedSize = sizes.get(path);
+    if (cachedSize != null) {
+      return cachedSize;
+    }
+
+    const size =
+      (resolver.isCoreModule(path)
+        ? coreModuleWeights[path]
+        : hasteFS.getSize(path)) || 0;
+    sizes.set(path, size);
+    return size;
+  }
+
+  _fileSizeRecurseDependencies(test: Test, sizes: Map<Path, number>): number {
+    const {resolver, hasteFS, config} = test.context;
+
+    const dependencyResolver = new DependencyResolver(
+      resolver,
+      hasteFS,
+      buildSnapshotResolver(config),
+    );
+    const recursiveDependencies = dependencyResolver.resolveRecursive(
+      test.path,
+      {includeCoreModules: true},
+    );
+    const size =
+      Array.from(recursiveDependencies).reduce(
+        (sum, path) => sum + this._fileSize(path, sizes, resolver, hasteFS),
+        0,
+      ) + this._fileSize(test.path, sizes, resolver, hasteFS);
+
+    return size;
+  }
+
   // When running more tests than we have workers available, sort the tests
   // by size - big test files usually take longer to complete, so we run
   // them first in an effort to minimize worker idle time at the end of a
@@ -66,9 +122,7 @@ export default class TestSequencer {
   // subsequent runs we use that to run the slowest tests first, yielding the
   // fastest results.
   sort(tests: Array<Test>): Array<Test> {
-    const stats = {};
-    const fileSize = ({path, context: {hasteFS}}) =>
-      stats[path] || (stats[path] = hasteFS.getSize(path) || 0);
+    const sizes = new Map();
     const hasFailed = (cache, test) =>
       cache[test.path] && cache[test.path][0] === FAIL;
     const time = (cache, test) => cache[test.path] && cache[test.path][1];
@@ -88,7 +142,10 @@ export default class TestSequencer {
       } else if (testA.duration != null && testB.duration != null) {
         return testA.duration < testB.duration ? 1 : -1;
       } else {
-        return fileSize(testA) < fileSize(testB) ? 1 : -1;
+        return this._fileSizeRecurseDependencies(testA, sizes) <
+          this._fileSizeRecurseDependencies(testB, sizes)
+          ? 1
+          : -1;
       }
     });
   }

packages/jest-cli/src/__tests__/test_sequencer.test.js

@@ -12,11 +12,40 @@ jest.mock('fs');
 
 const fs = require('fs');
 const path = require('path');
+const mockResolveRecursive = jest.fn(
+  (path, options) =>
+    ({
+      '/test-a.js': ['/sut-a.js', '/test-util.js'],
+      '/test-ab.js': ['/sut-ab.js'],
+      '/test-b.js': ['/sut-b0.js'],
+      '/test-cp.js': ['child_process'],
+      '/test-e.js': ['/sut-e.js'],
+      '/test-fs.js': ['fs'],
+      '/test-http.js': ['http'],
+    }[path] || []),
+);
+afterEach(() => {
+  mockResolveRecursive.mockClear();
+});
+jest.mock(
+  'jest-resolve-dependencies',
+  () =>
+    class {
+      constructor() {
+        this.resolveRecursive = mockResolveRecursive;
+      }
+    },
+);
+
 const FAIL = 0;
 const SUCCESS = 1;
 
 let sequencer;
 
+const resolver = {
+  isCoreModule: path => !path.startsWith('/'),
+};
+
 const context = {
   config: {
     cache: true,
@@ -26,6 +55,7 @@ const context = {
   hasteFS: {
     getSize: path => path.length,
   },
+  resolver,
 };
 
 const secondContext = {
@@ -37,6 +67,7 @@ const secondContext = {
   hasteFS: {
     getSize: path => path.length,
   },
+  resolver,
 };
 
 const toTests = paths =>
@@ -54,11 +85,40 @@ beforeEach(() => {
   fs.writeFileSync = jest.fn();
 });
 
-test('sorts by file size if there is no timing information', () => {
-  expect(sequencer.sort(toTests(['/test-a.js', '/test-ab.js']))).toEqual([
+test('sorts by dependency file sizes if there is no timing information', () => {
+  expect(sequencer.sort(toTests(['/test-ab.js', '/test-a.js']))).toEqual([
+    {context, duration: undefined, path: '/test-a.js'},
+    {context, duration: undefined, path: '/test-ab.js'},
+  ]);
+  expect(mockResolveRecursive).toHaveBeenCalledWith(expect.any(String), {
+    includeCoreModules: true,
+  });
+});
+
+test('includes the test file itself during size calculation', () => {
+  expect(sequencer.sort(toTests(['/test-b.js', '/test-ab.js']))).toEqual([
     {context, duration: undefined, path: '/test-ab.js'},
+    {context, duration: undefined, path: '/test-b.js'},
+  ]);
+  expect(mockResolveRecursive).toHaveBeenCalledWith(expect.any(String), {
+    includeCoreModules: true,
+  });
+});
+
+test('prioritizes tests that depend on certain core modules', () => {
+  expect(
+    sequencer.sort(
+      toTests(['/test-a.js', '/test-cp.js', '/test-fs.js', '/test-http.js']),
+    ),
+  ).toEqual([
+    {context, duration: undefined, path: '/test-cp.js'},
+    {context, duration: undefined, path: '/test-fs.js'},
+    {context, duration: undefined, path: '/test-http.js'},
     {context, duration: undefined, path: '/test-a.js'},
   ]);
+  expect(mockResolveRecursive).toHaveBeenCalledWith(expect.any(String), {
+    includeCoreModules: true,
+  });
 });
 
 test('sorts based on timing information', () => {
@@ -95,29 +155,29 @@ test('sorts based on failures and timing information', () => {
   ]);
 });
 
-test('sorts based on failures, timing information and file size', () => {
+test('sorts based on failures, timing information and dependency file sizes', () => {
   fs.readFileSync = jest.fn(() =>
     JSON.stringify({
       '/test-a.js': [SUCCESS, 5],
       '/test-ab.js': [FAIL, 1],
-      '/test-c.js': [FAIL],
+      '/test-cd.js': [FAIL],
       '/test-d.js': [SUCCESS, 2],
-      '/test-efg.js': [FAIL],
+      '/test-e.js': [FAIL],
     }),
   );
   expect(
     sequencer.sort(
       toTests([
         '/test-a.js',
         '/test-ab.js',
-        '/test-c.js',
+        '/test-cd.js',
         '/test-d.js',
-        '/test-efg.js',
+        '/test-e.js',
       ]),
     ),
   ).toEqual([
-    {context, duration: undefined, path: '/test-efg.js'},
-    {context, duration: undefined, path: '/test-c.js'},
+    {context, duration: undefined, path: '/test-e.js'},
+    {context, duration: undefined, path: '/test-cd.js'},
     {context, duration: 1, path: '/test-ab.js'},
     {context, duration: 5, path: '/test-a.js'},
     {context, duration: 2, path: '/test-d.js'},

packages/jest-resolve-dependencies/src/__tests__/__fixtures__/circular/a.js

@@ -0,0 +1,10 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+'use strict';
+
+require('./b');

packages/jest-resolve-dependencies/src/__tests__/__fixtures__/circular/b.js

@@ -0,0 +1,10 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+'use strict';
+
+require('./c');

packages/jest-resolve-dependencies/src/__tests__/__fixtures__/circular/c.js

@@ -0,0 +1,10 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+'use strict';
+
+require('./a');

packages/jest-resolve-dependencies/src/__tests__/__fixtures__/core.test.js

@@ -0,0 +1,10 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+'use strict';
+
+require('fs');

packages/jest-resolve-dependencies/src/__tests__/__fixtures__/recursive/a.js

@@ -0,0 +1,10 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+'use strict';
+
+require('./b');

packages/jest-resolve-dependencies/src/__tests__/__fixtures__/recursive/b.js

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+'use strict';
+
+require('./c1');
+require('./c2');

packages/jest-resolve-dependencies/src/__tests__/__fixtures__/recursive/c1.js

@@ -0,0 +1,8 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+'use strict';

packages/jest-resolve-dependencies/src/__tests__/__fixtures__/recursive/c2.js

@@ -0,0 +1,8 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+'use strict';