From f4a77eead4ef2aca9ceab236d8bba55fea818b12 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Wed, 13 Sep 2017 19:17:38 -0700 Subject: [PATCH 1/5] Out with the old... --- src/harness/runner.ts | 374 +++++++++++++++++++++++++++++------------- 1 file changed, 264 insertions(+), 110 deletions(-) diff --git a/src/harness/runner.ts b/src/harness/runner.ts index 6e1f91b21af29..055a38b984d0f 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -85,9 +85,9 @@ let runUnitTests = true; interface TestConfig { light?: boolean; taskConfigsFolder?: string; + listenForWork?: boolean; workerCount?: number; stackTraceLimit?: number | "full"; - tasks?: TaskSet[]; test?: string[]; runUnitTests?: boolean; } @@ -97,138 +97,292 @@ interface TaskSet { files: string[]; } -if (testConfigContent !== "") { - const testConfig = JSON.parse(testConfigContent); - if (testConfig.light) { - Harness.lightMode = true; - } - if (testConfig.taskConfigsFolder) { - taskConfigsFolder = testConfig.taskConfigsFolder; - } - if (testConfig.runUnitTests !== undefined) { - runUnitTests = testConfig.runUnitTests; - } - if (testConfig.workerCount) { - workerCount = testConfig.workerCount; - } - if (testConfig.tasks) { - for (const taskSet of testConfig.tasks) { - const runner = createRunner(taskSet.runner); - for (const file of taskSet.files) { - runner.addTest(file); +function handleTestConfig() { + if (testConfigContent !== "") { + const testConfig = JSON.parse(testConfigContent); + if (testConfig.listenForWork) { + return true; + } + if (testConfig.light) { + Harness.lightMode = true; + } + if (testConfig.runUnitTests !== undefined) { + runUnitTests = testConfig.runUnitTests; + } + if (testConfig.workerCount) { + workerCount = testConfig.workerCount; + } + if (testConfig.taskConfigsFolder) { + taskConfigsFolder = testConfig.taskConfigsFolder; + } + + if (testConfig.stackTraceLimit === "full") { + (Error).stackTraceLimit = Infinity; + } + else if ((+testConfig.stackTraceLimit | 0) > 0) { + (Error).stackTraceLimit = testConfig.stackTraceLimit; + } + + if (testConfig.test && testConfig.test.length > 0) { + for (const option of testConfig.test) { + if (!option) { + continue; + } + + switch (option) { + case "compiler": + runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); + runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); + runners.push(new ProjectRunner()); + break; + case "conformance": + runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); + break; + case "project": + runners.push(new ProjectRunner()); + break; + case "fourslash": + runners.push(new FourSlashRunner(FourSlashTestType.Native)); + break; + case "fourslash-shims": + runners.push(new FourSlashRunner(FourSlashTestType.Shims)); + break; + case "fourslash-shims-pp": + runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess)); + break; + case "fourslash-server": + runners.push(new FourSlashRunner(FourSlashTestType.Server)); + break; + case "fourslash-generated": + runners.push(new GeneratedFourslashRunner(FourSlashTestType.Native)); + break; + case "rwc": + runners.push(new RWCRunner()); + break; + case "test262": + runners.push(new Test262BaselineRunner()); + break; + } } - runners.push(runner); } } - if (testConfig.stackTraceLimit === "full") { - (Error).stackTraceLimit = Infinity; + if (runners.length === 0) { + // compiler + runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); + runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); + + // TODO: project tests don't work in the browser yet + if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) { + runners.push(new ProjectRunner()); + } + + // language services + runners.push(new FourSlashRunner(FourSlashTestType.Native)); + runners.push(new FourSlashRunner(FourSlashTestType.Shims)); + runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess)); + runners.push(new FourSlashRunner(FourSlashTestType.Server)); + // runners.push(new GeneratedFourslashRunner()); } - else if ((+testConfig.stackTraceLimit | 0) > 0) { - (Error).stackTraceLimit = testConfig.stackTraceLimit; +} + +function beginTests() { + if (ts.Debug.isDebugging) { + ts.Debug.enableDebugInfo(); } - if (testConfig.test && testConfig.test.length > 0) { - for (const option of testConfig.test) { - if (!option) { - continue; - } + runTests(runners); - switch (option) { - case "compiler": - runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); - runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); - runners.push(new ProjectRunner()); - break; - case "conformance": - runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); - break; - case "project": - runners.push(new ProjectRunner()); - break; - case "fourslash": - runners.push(new FourSlashRunner(FourSlashTestType.Native)); - break; - case "fourslash-shims": - runners.push(new FourSlashRunner(FourSlashTestType.Shims)); - break; - case "fourslash-shims-pp": - runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess)); - break; - case "fourslash-server": - runners.push(new FourSlashRunner(FourSlashTestType.Server)); - break; - case "fourslash-generated": - runners.push(new GeneratedFourslashRunner(FourSlashTestType.Native)); - break; - case "rwc": - runners.push(new RWCRunner()); - break; - case "test262": - runners.push(new Test262BaselineRunner()); - break; - } - } + if (!runUnitTests) { + // patch `describe` to skip unit tests + describe = ts.noop as any; } } -if (runners.length === 0) { - // compiler - runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); - runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); +type ParallelTestMessage = { type: "test", payload: { runner: TestRunnerKind, file: string } } | never; +type ParallelCloseMessage = { type: "close" } | never; +type ParallelHostMessage = ParallelTestMessage | ParallelCloseMessage; - // TODO: project tests don't work in the browser yet - if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) { - runners.push(new ProjectRunner()); - } +type ParallelErrorMessage = { type: "error", payload: { error: string, stack: string } } | never; +type ErrorInfo = ParallelErrorMessage["payload"] & { name: string }; +type ParallelResultMessage = { type: "result", payload: { passing: number, errors: ErrorInfo[] } } | never; +type ParallelClientMessage = ParallelErrorMessage | ParallelResultMessage; - // language services - runners.push(new FourSlashRunner(FourSlashTestType.Native)); - runners.push(new FourSlashRunner(FourSlashTestType.Shims)); - runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess)); - runners.push(new FourSlashRunner(FourSlashTestType.Server)); - // runners.push(new GeneratedFourslashRunner()); +let errors: ErrorInfo[] = []; +let passing = 0; +function resetShimHarnessAndExecute(runner: RunnerBase) { + errors = []; + passing = 0; + runner.initializeTests(); + return { errors, passing }; } -if (taskConfigsFolder) { - // this instance of mocha should only partition work but not run actual tests - runUnitTests = false; - const workerConfigs: TestConfig[] = []; - for (let i = 0; i < workerCount; i++) { - // pass light mode settings to workers - workerConfigs.push({ light: Harness.lightMode, tasks: [] }); - } +function shimMochaHarness() { + let beforeEachFunc: Function; + describe = ((_name, callback) => { + const fakeContext: Mocha.ISuiteCallbackContext = { + retries() { return this; }, + slow() { return this; }, + timeout() { return this; }, + }; + (before as any) = (cb: Function) => cb(); + let afterFunc: Function; + (after as any) = (cb: Function) => afterFunc = cb; + const savedBeforeEach = beforeEachFunc; + (beforeEach as any) = (cb: Function) => beforeEachFunc = cb; + callback.call(fakeContext); + afterFunc && afterFunc(); + afterFunc = undefined; + beforeEachFunc = savedBeforeEach; + }) as Mocha.IContextDefinition; + it = ((name, callback) => { + const fakeContext: Mocha.ITestCallbackContext = { + skip() { return this; }, + timeout() { return this; }, + retries() { return this; }, + slow() { return this; }, + }; + // TODO: If we ever start using async test completions, polyfill the `done` parameter/promise return handling + if (beforeEachFunc) { + try { + beforeEachFunc(); + } + catch (error) { + errors.push({ error: error.message, stack: error.stack, name }); + return; + } + } + try { + callback.call(fakeContext); + } + catch (error) { + errors.push({ error: error.message, stack: error.stack, name }); + return; + } + passing++; + }) as Mocha.ITestDefinition; +} + +function beginWorkListener() { + shimMochaHarness(); + const runners = ts.createMap(); + process.on("message", (data: ParallelHostMessage) => { + switch (data.type) { + case "test": + const { runner, file } = data.payload; + if (!runners.has(runner)) { + runners.set(runner, createRunner(runner)); + } + const instance = runners.get(runner); + instance.tests = []; + instance.addTest(file); + const payload = resetShimHarnessAndExecute(instance); + const message: ParallelResultMessage = { type: "result", payload }; + process.send(message); + break; + case "close": + process.exit(0); + break; + } + }); + process.on("uncaughtException", error => { + const message: ParallelErrorMessage = { type: "error", payload: { error: error.message, stack: error.stack } }; + process.send(message); + }); +} +interface ChildProcessPartial { + send(message: any, callback?: (error: Error) => void): boolean; + on(event: "error", listener: (err: Error) => void): this; + on(event: "exit", listener: (code: number, signal: string) => void): this; + on(event: "message", listener: (message: any) => void): this; + disconnect(): void; +} + +function beginTestHost() { + const { statSync }: { statSync(path: string): { size: number }; } = require("fs"); + const tasks: { runner: TestRunnerKind, file: string, size: number }[] = []; for (const runner of runners) { const files = runner.enumerateTestFiles(); - const chunkSize = Math.floor(files.length / workerCount) + 1; // add extra 1 to prevent missing tests due to rounding - for (let i = 0; i < workerCount; i++) { - const startPos = i * chunkSize; - const len = Math.min(chunkSize, files.length - startPos); - if (len > 0) { - workerConfigs[i].tasks.push({ - runner: runner.kind(), - files: files.slice(startPos, startPos + len) - }); - } + for (const file in files) { + tasks.push({ runner: runner.kind(), file, size: statSync(file).size }); } } + tasks.sort((a, b) => a.size - b.size); + const { fork }: { fork(modulePath: string, args?: string[], options?: {}): ChildProcessPartial; } = require("child_process"); + const totalFiles = tasks.length; + console.log(`1..${totalFiles}`); + let passingFiles = 0; + let failingFiles = 0; + let errorResults: ErrorInfo[] = []; + let totalPassing = 0; for (let i = 0; i < workerCount; i++) { - const config = workerConfigs[i]; - // use last worker to run unit tests if we're not just running a single specific runner - config.runUnitTests = runners.length !== 1 && i === workerCount - 1; - Harness.IO.writeFile(ts.combinePaths(taskConfigsFolder, `task-config${i}.json`), JSON.stringify(workerConfigs[i])); + // TODO: Just send the config over the IPC channel or in the command line arguments + const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length !== 1 && i === workerCount - 1 }; + const configPath = ts.combinePaths(taskConfigsFolder, `task-config${i}.json`); + Harness.IO.writeFile(configPath, JSON.stringify(config)); + const child = fork(__filename, [`--config="${configPath}"`]); + child.on("error", err => { + child.disconnect(); + console.error(err); + }); + child.on("exit", (code, _signal) => { + child.disconnect(); + if (code !== 0) { + console.error("Test worker process exited with nonzero exit code!"); + } + }); + child.on("message", (data: ParallelClientMessage) => { + switch (data.type) { + case "error": + child.disconnect(); + console.error(`Test worker encounted unexpected error and was forced to close: +Message: ${data.payload.error} +Stack: ${data.payload.stack}`); + return; + case "result": + totalPassing += data.payload.passing; + if (data.payload.errors.length) { + errorResults = errorResults.concat(data.payload.errors); + failingFiles++; + console.log(`not ok ${passingFiles + failingFiles} - ${data.payload.errors.map(e => `test ${e.name} failed! Error: ${e.error.replace(/\r?\n/g, " ")} Stack: ${e.stack.replace(/\r?\n/g, " ")}`).join(" | ")}`); + } + else { + passingFiles++; + console.log(`ok ${passingFiles + failingFiles}`); + } + if (failingFiles + passingFiles === totalFiles) { + // Done. Finished every task and collected results. + child.send({ type: "close" }); + child.disconnect(); + return process.exit(errorResults.length); + } + if (tasks.length === 0) { + // No more tasks to distribute + child.send({ type: "close" }); + child.disconnect(); + return; + } + child.send({ type: "test", payload: tasks.pop() }); + } + }); + child.send({ type: "test", payload: tasks.pop() }); } } -else { - if (ts.Debug.isDebugging) { - ts.Debug.enableDebugInfo(); - } - runTests(runners); -} -if (!runUnitTests) { - // patch `describe` to skip unit tests - describe = ts.noop as any; +function startTestEnvironment() { + const isWorker = handleTestConfig(); + if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) { + if (isWorker) { + return beginWorkListener(); + } + else if (taskConfigsFolder && workerCount && workerCount > 1) { + return beginTestHost(); + } + } + beginTests(); } + +startTestEnvironment(); + From 68fc644342c751cedf788af2fb1f079f4d050d8c Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Wed, 13 Sep 2017 20:07:47 -0700 Subject: [PATCH 2/5] Brave new world --- Gulpfile.ts | 31 +-- Jakefile.js | 37 ++- scripts/mocha-none-reporter.js | 26 --- scripts/mocha-parallel.js | 405 --------------------------------- src/harness/runner.ts | 302 +++++++++++++++++++++--- 5 files changed, 295 insertions(+), 506 deletions(-) delete mode 100644 scripts/mocha-none-reporter.js delete mode 100644 scripts/mocha-parallel.js diff --git a/Gulpfile.ts b/Gulpfile.ts index a3db20dfd8a3e..676d07ec570e3 100644 --- a/Gulpfile.ts +++ b/Gulpfile.ts @@ -31,8 +31,6 @@ import merge2 = require("merge2"); import * as os from "os"; import fold = require("travis-fold"); const gulp = helpMaker(originalGulp); -const mochaParallel = require("./scripts/mocha-parallel.js"); -const {runTestsInParallel} = mochaParallel; Error.stackTraceLimit = 1000; @@ -668,26 +666,9 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done: } else { // run task to load all tests and partition them between workers - const args = []; - args.push("-R", "min"); - if (colors) { - args.push("--colors"); - } - else { - args.push("--no-colors"); - } - args.push(run); setNodeEnvToDevelopment(); - runTestsInParallel(taskConfigsFolder, run, { testTimeout, noColors: colors === " --no-colors " }, function(err) { - // last worker clean everything and runs linter in case if there were no errors - del(taskConfigsFolder).then(() => { - if (!err) { - lintThenFinish(); - } - else { - finish(err); - } - }); + exec(host, [run], lintThenFinish, function(e, status) { + finish(e, status); }); } }); @@ -711,7 +692,7 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done: function finish(error?: any, errorStatus?: number) { restoreSavedNodeEnv(); - deleteTemporaryProjectOutput().then(() => { + deleteTestConfig().then(deleteTemporaryProjectOutput).then(() => { if (error !== undefined || errorStatus !== undefined) { failWithStatus(error, errorStatus); } @@ -720,6 +701,10 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done: } }); } + + function deleteTestConfig() { + return del("test.config"); + } } gulp.task("runtests-parallel", "Runs all the tests in parallel using the built run.js file. Optional arguments are: --t[ests]=category1|category2|... --d[ebug]=true.", ["build-rules", "tests"], (done) => { @@ -836,7 +821,7 @@ function cleanTestDirs(done: (e?: any) => void) { // used to pass data from jake command line directly to run.js function writeTestConfigFile(tests: string, light: boolean, taskConfigsFolder?: string, workerCount?: number, stackTraceLimit?: string) { - const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light, workerCount, stackTraceLimit, taskConfigsFolder }); + const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light, workerCount, stackTraceLimit, taskConfigsFolder, noColor: !cmdLineOptions["colors"] }); console.log("Running tests with config: " + testConfigContents); fs.writeFileSync("test.config", testConfigContents); } diff --git a/Jakefile.js b/Jakefile.js index ad853238111e4..914839efbcce5 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -1,11 +1,11 @@ // This file contains the build logic for the public repo +// @ts-check var fs = require("fs"); var os = require("os"); var path = require("path"); var child_process = require("child_process"); var fold = require("travis-fold"); -var runTestsInParallel = require("./scripts/mocha-parallel").runTestsInParallel; var ts = require("./lib/typescript"); @@ -38,7 +38,7 @@ else if (process.env.PATH !== undefined) { function filesFromConfig(configPath) { var configText = fs.readFileSync(configPath).toString(); - var config = ts.parseConfigFileTextToJson(configPath, configText, /*stripComments*/ true); + var config = ts.parseConfigFileTextToJson(configPath, configText); if (config.error) { throw new Error(diagnosticsToString([config.error])); } @@ -596,7 +596,7 @@ file(typesMapOutputPath, function() { var content = fs.readFileSync(path.join(serverDirectory, 'typesMap.json')); // Validate that it's valid JSON try { - JSON.parse(content); + JSON.parse(content.toString()); } catch (e) { console.log("Parse error in typesMap.json: " + e); } @@ -740,7 +740,7 @@ desc("Builds the test infrastructure using the built compiler"); task("tests", ["local", run].concat(libraryTargets)); function exec(cmd, completeHandler, errorHandler) { - var ex = jake.createExec([cmd], { windowsVerbatimArguments: true }); + var ex = jake.createExec([cmd], { windowsVerbatimArguments: true, interactive: true }); // Add listeners for output and error ex.addListener("stdout", function (output) { process.stdout.write(output); @@ -783,13 +783,14 @@ function cleanTestDirs() { } // used to pass data from jake command line directly to run.js -function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit) { +function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit, colors) { var testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light: light, workerCount: workerCount, taskConfigsFolder: taskConfigsFolder, - stackTraceLimit: stackTraceLimit + stackTraceLimit: stackTraceLimit, + noColor: !colors }); fs.writeFileSync('test.config', testConfigContents); } @@ -831,7 +832,7 @@ function runConsoleTests(defaultReporter, runInParallel) { } if (tests || light || taskConfigsFolder) { - writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit); + writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit, colors); } if (tests && tests.toLocaleLowerCase() === "rwc") { @@ -894,19 +895,15 @@ function runConsoleTests(defaultReporter, runInParallel) { var savedNodeEnv = process.env.NODE_ENV; process.env.NODE_ENV = "development"; var startTime = mark(); - runTestsInParallel(taskConfigsFolder, run, { testTimeout: testTimeout, noColors: !colors }, function (err) { + exec(host + " " + run, function () { process.env.NODE_ENV = savedNodeEnv; measure(startTime); - // last worker clean everything and runs linter in case if there were no errors - deleteTemporaryProjectOutput(); - jake.rmRf(taskConfigsFolder); - if (err) { - fail(err); - } - else { - runLinter(); - complete(); - } + runLinter(); + finish(); + }, function (e, status) { + process.env.NODE_ENV = savedNodeEnv; + measure(startTime); + finish(status); }); } @@ -969,8 +966,8 @@ desc("Runs the tests using the built run.js file like 'jake runtests'. Syntax is task("runtests-browser", ["browserify", nodeServerOutFile], function () { cleanTestDirs(); host = "node"; - browser = process.env.browser || process.env.b || (os.platform() === "linux" ? "chrome" : "IE"); - tests = process.env.test || process.env.tests || process.env.t; + var browser = process.env.browser || process.env.b || (os.platform() === "linux" ? "chrome" : "IE"); + var tests = process.env.test || process.env.tests || process.env.t; var light = process.env.light || false; var testConfigFile = 'test.config'; if (fs.existsSync(testConfigFile)) { diff --git a/scripts/mocha-none-reporter.js b/scripts/mocha-none-reporter.js deleted file mode 100644 index 5787b0c042ef5..0000000000000 --- a/scripts/mocha-none-reporter.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Module dependencies. - */ - -var Base = require('mocha').reporters.Base; - -/** - * Expose `None`. - */ - -exports = module.exports = None; - -/** - * Initialize a new `None` test reporter. - * - * @api public - * @param {Runner} runner - */ -function None(runner) { - Base.call(this); -} - -/** - * Inherit from `Base.prototype`. - */ -None.prototype.__proto__ = Base.prototype; diff --git a/scripts/mocha-parallel.js b/scripts/mocha-parallel.js deleted file mode 100644 index 6a54c018e9add..0000000000000 --- a/scripts/mocha-parallel.js +++ /dev/null @@ -1,405 +0,0 @@ -var tty = require("tty") - , readline = require("readline") - , fs = require("fs") - , path = require("path") - , child_process = require("child_process") - , os = require("os") - , mocha = require("mocha") - , Base = mocha.reporters.Base - , color = Base.color - , cursor = Base.cursor - , ms = require("mocha/lib/ms"); - -var isatty = tty.isatty(1) && tty.isatty(2); -var tapRangePattern = /^(\d+)\.\.(\d+)(?:$|\r\n?|\n)/; -var tapTestPattern = /^(not\sok|ok)\s+(\d+)\s+(?:-\s+)?(.*)$/; -var tapCommentPattern = /^#(?: (tests|pass|fail) (\d+)$)?/; - -exports.runTestsInParallel = runTestsInParallel; -exports.ProgressBars = ProgressBars; - -function runTestsInParallel(taskConfigsFolder, run, options, cb) { - if (options === undefined) options = { }; - - return discoverTests(run, options, function (error) { - if (error) { - return cb(error); - } - - return runTests(taskConfigsFolder, run, options, cb); - }); -} - -function discoverTests(run, options, cb) { - console.log("Discovering tests..."); - - var cmd = "mocha -R " + require.resolve("./mocha-none-reporter.js") + " " + run; - var p = spawnProcess(cmd); - p.on("exit", function (status) { - if (status) { - cb(new Error("Process exited with code " + status)); - } - else { - cb(); - } - }); -} - -function runTests(taskConfigsFolder, run, options, cb) { - var configFiles = fs.readdirSync(taskConfigsFolder); - var numPartitions = configFiles.length; - if (numPartitions <= 0) { - cb(); - return; - } - - console.log("Running tests on " + numPartitions + " threads..."); - - var partitions = Array(numPartitions); - var progressBars = new ProgressBars(); - progressBars.enable(); - - var counter = numPartitions; - configFiles.forEach(runTestsInPartition); - - function runTestsInPartition(file, index) { - var partition = { - file: path.join(taskConfigsFolder, file), - tests: 0, - passed: 0, - failed: 0, - completed: 0, - current: undefined, - start: undefined, - end: undefined, - catastrophicError: "", - failures: [] - }; - partitions[index] = partition; - - // Set up the progress bar. - updateProgress(0); - - // Start the background process. - var cmd = "mocha -t " + (options.testTimeout || 20000) + " -R tap --no-colors " + run + " --config='" + partition.file + "'"; - var p = spawnProcess(cmd); - var rl = readline.createInterface({ - input: p.stdout, - terminal: false - }); - - var rlError = readline.createInterface({ - input: p.stderr, - terminal: false - }); - - rl.on("line", onmessage); - rlError.on("line", onErrorMessage); - p.on("exit", onexit) - - function onErrorMessage(line) { - partition.catastrophicError += line + os.EOL; - } - - function onmessage(line) { - if (partition.start === undefined) { - partition.start = Date.now(); - } - - var rangeMatch = tapRangePattern.exec(line); - if (rangeMatch) { - partition.tests = parseInt(rangeMatch[2]); - return; - } - - var testMatch = tapTestPattern.exec(line); - if (testMatch) { - var test = { - result: testMatch[1], - id: parseInt(testMatch[2]), - name: testMatch[3], - output: [] - }; - - partition.current = test; - partition.completed++; - - if (test.result === "ok") { - partition.passed++; - } - else { - partition.failed++; - partition.failures.push(test); - } - - var progress = partition.completed / partition.tests; - if (progress < 1) { - updateProgress(progress); - } - - return; - } - - var commentMatch = tapCommentPattern.exec(line); - if (commentMatch) { - switch (commentMatch[1]) { - case "tests": - partition.current = undefined; - partition.tests = parseInt(commentMatch[2]); - break; - - case "pass": - partition.passed = parseInt(commentMatch[2]); - break; - - case "fail": - partition.failed = parseInt(commentMatch[2]); - break; - } - - return; - } - - if (partition.current) { - partition.current.output.push(line); - } - } - - function onexit(code) { - if (partition.end === undefined) { - partition.end = Date.now(); - } - - partition.duration = partition.end - partition.start; - var isPartitionFail = partition.failed || code !== 0; - var summaryColor = isPartitionFail ? "fail" : "green"; - var summarySymbol = isPartitionFail ? Base.symbols.err : Base.symbols.ok; - - var summaryTests = (isPartitionFail ? partition.passed + "/" + partition.tests : partition.passed) + " passing"; - var summaryDuration = "(" + ms(partition.duration) + ")"; - var savedUseColors = Base.useColors; - Base.useColors = !options.noColors; - - var summary = color(summaryColor, summarySymbol + " " + summaryTests) + " " + color("light", summaryDuration); - Base.useColors = savedUseColors; - - updateProgress(1, summary); - - signal(); - } - - function updateProgress(percentComplete, title) { - var progressColor = "pending"; - if (partition.failed) { - progressColor = "fail"; - } - - progressBars.update( - index, - percentComplete, - progressColor, - title - ); - } - } - - function signal() { - counter--; - - if (counter <= 0) { - var reporter = new Base(), - stats = reporter.stats, - failures = reporter.failures; - - var duration = 0; - var catastrophicError = ""; - for (var i = 0; i < numPartitions; i++) { - var partition = partitions[i]; - stats.passes += partition.passed; - stats.failures += partition.failed; - stats.tests += partition.tests; - duration += partition.duration; - if (partition.catastrophicError !== "") { - // Partition is written out to a temporary file as a JSON object. - // Below is an example of how the partition JSON object looks like - // { - // "light":false, - // "tasks":[ - // { - // "runner":"compiler", - // "files":["tests/cases/compiler/es6ImportNamedImportParsingError.ts"] - // } - // ], - // "runUnitTests":false - // } - var jsonText = fs.readFileSync(partition.file); - var configObj = JSON.parse(jsonText); - if (configObj.tasks && configObj.tasks[0]) { - catastrophicError += "Error from one or more of these files: " + configObj.tasks[0].files + os.EOL; - catastrophicError += partition.catastrophicError; - catastrophicError += os.EOL; - } - } - for (var j = 0; j < partition.failures.length; j++) { - var failure = partition.failures[j]; - failures.push(makeMochaTest(failure)); - } - } - - stats.duration = duration; - progressBars.disable(); - - if (options.noColors) { - var savedUseColors = Base.useColors; - Base.useColors = false; - reporter.epilogue(); - Base.useColors = savedUseColors; - } - else { - reporter.epilogue(); - } - - if (catastrophicError !== "") { - return cb(new Error(catastrophicError)); - } - if (stats.failures) { - return cb(new Error("Test failures reported: " + stats.failures)); - } - else { - return cb(); - } - } - } - - function makeMochaTest(test) { - return { - fullTitle: function() { - return test.name; - }, - err: { - message: test.output[0], - stack: test.output.join(os.EOL) - } - }; - } -} - -var nodeModulesPathPrefix = path.resolve("./node_modules/.bin/") + path.delimiter; -if (process.env.path !== undefined) { - process.env.path = nodeModulesPathPrefix + process.env.path; -} else if (process.env.PATH !== undefined) { - process.env.PATH = nodeModulesPathPrefix + process.env.PATH; -} - -function spawnProcess(cmd, options) { - var shell = process.platform === "win32" ? "cmd" : "/bin/sh"; - var prefix = process.platform === "win32" ? "/c" : "-c"; - return child_process.spawn(shell, [prefix, cmd], { windowsVerbatimArguments: true }); -} - -function ProgressBars(options) { - if (!options) options = {}; - var open = options.open || '['; - var close = options.close || ']'; - var complete = options.complete || '▬'; - var incomplete = options.incomplete || Base.symbols.dot; - var maxWidth = Math.floor(Base.window.width * .30) - open.length - close.length - 2; - var width = minMax(options.width || maxWidth, 10, maxWidth); - this._options = { - open: open, - complete: complete, - incomplete: incomplete, - close: close, - width: width - }; - - this._progressBars = []; - this._lineCount = 0; - this._enabled = false; -} -ProgressBars.prototype = { - enable: function () { - if (!this._enabled) { - process.stdout.write(os.EOL); - this._enabled = true; - } - }, - disable: function () { - if (this._enabled) { - process.stdout.write(os.EOL); - this._enabled = false; - } - }, - update: function (index, percentComplete, color, title) { - percentComplete = minMax(percentComplete, 0, 1); - - var progressBar = this._progressBars[index] || (this._progressBars[index] = { }); - var width = this._options.width; - var n = Math.floor(width * percentComplete); - var i = width - n; - if (n === progressBar.lastN && title === progressBar.title && color === progressBar.progressColor) { - return; - } - - progressBar.lastN = n; - progressBar.title = title; - progressBar.progressColor = color; - - var progress = " "; - progress += this._color('progress', this._options.open); - progress += this._color(color, fill(this._options.complete, n)); - progress += this._color('progress', fill(this._options.incomplete, i)); - progress += this._color('progress', this._options.close); - - if (title) { - progress += this._color('progress', ' ' + title); - } - - if (progressBar.text !== progress) { - progressBar.text = progress; - this._render(index); - } - }, - _render: function (index) { - if (!this._enabled || !isatty) { - return; - } - - cursor.hide(); - readline.moveCursor(process.stdout, -process.stdout.columns, -this._lineCount); - var lineCount = 0; - var numProgressBars = this._progressBars.length; - for (var i = 0; i < numProgressBars; i++) { - if (i === index) { - readline.clearLine(process.stdout, 1); - process.stdout.write(this._progressBars[i].text + os.EOL); - } - else { - readline.moveCursor(process.stdout, -process.stdout.columns, +1); - } - - lineCount++; - } - - this._lineCount = lineCount; - cursor.show(); - }, - _color: function (type, text) { - return type && !this._options.noColors ? color(type, text) : text; - } -}; - -function fill(ch, size) { - var s = ""; - while (s.length < size) { - s += ch; - } - - return s.length > size ? s.substr(0, size) : s; -} - -function minMax(value, min, max) { - if (value < min) return min; - if (value > max) return max; - return value; -} \ No newline at end of file diff --git a/src/harness/runner.ts b/src/harness/runner.ts index 055a38b984d0f..900e1609ff877 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -81,6 +81,7 @@ let testConfigContent = let taskConfigsFolder: string; let workerCount: number; let runUnitTests = true; +let noColors = false; interface TestConfig { light?: boolean; @@ -90,6 +91,7 @@ interface TestConfig { stackTraceLimit?: number | "full"; test?: string[]; runUnitTests?: boolean; + noColors?: boolean; } interface TaskSet { @@ -115,6 +117,9 @@ function handleTestConfig() { if (testConfig.taskConfigsFolder) { taskConfigsFolder = testConfig.taskConfigsFolder; } + if (testConfig.noColors !== undefined) { + noColors = testConfig.noColors; + } if (testConfig.stackTraceLimit === "full") { (Error).stackTraceLimit = Infinity; @@ -172,7 +177,7 @@ function handleTestConfig() { runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); - // TODO: project tests don't work in the browser yet + // TODO: project tests don"t work in the browser yet if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) { runners.push(new ProjectRunner()); } @@ -218,6 +223,9 @@ function resetShimHarnessAndExecute(runner: RunnerBase) { } function shimMochaHarness() { + (global as any).before = undefined; + (global as any).after = undefined; + (global as any).beforeEach = undefined; let beforeEachFunc: Function; describe = ((_name, callback) => { const fakeContext: Mocha.ISuiteCallbackContext = { @@ -299,24 +307,172 @@ interface ChildProcessPartial { disconnect(): void; } +// tslint:disable-next-line +var describe: Mocha.IContextDefinition; // If launched without mocha for parallel mode, we still need a global describe visible to satisfy the parsing of the unit tests +// tslint:disable-next-line +var it: Mocha.ITestDefinition; function beginTestHost() { + const Mocha = require("mocha"); + const Base = Mocha.reporters.Base; + const color = Base.color; + const cursor = Base.cursor; + const readline = require("readline"); + const os = require("os"); + interface ProgressBarsOptions { + open: string; + close: string; + complete: string; + incomplete: string; + width: number; + noColors: boolean; + } + interface ProgressBar { + lastN?: number; + title?: string; + progressColor?: string; + text?: string; + } + const tty: { isatty(x: number): boolean } = require("tty"); + const isatty = tty.isatty(1) && tty.isatty(2); + + class ProgressBars { + private _options: ProgressBarsOptions; + private _enabled: boolean; + private _lineCount: number; + private _progressBars: ProgressBar[]; + constructor(options?: Partial) { + if (!options) options = {}; + const open = options.open || "["; + const close = options.close || "]"; + const complete = options.complete || "▬"; + const incomplete = options.incomplete || Base.symbols.dot; + const maxWidth = Base.window.width - open.length - close.length - 30; + const width = minMax(options.width || maxWidth, 10, maxWidth); + this._options = { + open, + complete, + incomplete, + close, + width, + noColors: options.noColors || false + }; + + this._progressBars = []; + this._lineCount = 0; + this._enabled = false; + } + enable() { + if (!this._enabled) { + process.stdout.write(os.EOL); + this._enabled = true; + } + } + disable() { + if (this._enabled) { + process.stdout.write(os.EOL); + this._enabled = false; + } + } + update(index: number, percentComplete: number, color: string, title: string) { + percentComplete = minMax(percentComplete, 0, 1); + + const progressBar = this._progressBars[index] || (this._progressBars[index] = { }); + const width = this._options.width; + const n = Math.floor(width * percentComplete); + const i = width - n; + if (n === progressBar.lastN && title === progressBar.title && color === progressBar.progressColor) { + return; + } + + progressBar.lastN = n; + progressBar.title = title; + progressBar.progressColor = color; + + let progress = " "; + progress += this._color("progress", this._options.open); + progress += this._color(color, fill(this._options.complete, n)); + progress += this._color("progress", fill(this._options.incomplete, i)); + progress += this._color("progress", this._options.close); + + if (title) { + progress += this._color("progress", " " + title); + } + + if (progressBar.text !== progress) { + progressBar.text = progress; + this._render(index); + } + } + private _render(index: number) { + if (!this._enabled || !isatty) { + return; + } + + cursor.hide(); + readline.moveCursor(process.stdout, -process.stdout.columns, -this._lineCount); + let lineCount = 0; + const numProgressBars = this._progressBars.length; + for (let i = 0; i < numProgressBars; i++) { + if (i === index) { + readline.clearLine(process.stdout, 1); + process.stdout.write(this._progressBars[i].text + os.EOL); + } + else { + readline.moveCursor(process.stdout, -process.stdout.columns, +1); + } + + lineCount++; + } + + this._lineCount = lineCount; + cursor.show(); + } + private _color(type: string, text: string) { + return type && !this._options.noColors ? color(type, text) : text; + } + } + + function fill(ch: string, size: number) { + let s = ""; + while (s.length < size) { + s += ch; + } + + return s.length > size ? s.substr(0, size) : s; + } + + function minMax(value: number, min: number, max: number) { + if (value < min) return min; + if (value > max) return max; + return value; + } + + console.log("Discovering tests..."); + const discoverStart = +(new Date()); const { statSync }: { statSync(path: string): { size: number }; } = require("fs"); const tasks: { runner: TestRunnerKind, file: string, size: number }[] = []; for (const runner of runners) { const files = runner.enumerateTestFiles(); - for (const file in files) { + for (const file of files) { tasks.push({ runner: runner.kind(), file, size: statSync(file).size }); } } tasks.sort((a, b) => a.size - b.size); + console.log(`Discovered ${tasks.length} test files in ${+(new Date()) - discoverStart}ms.`); + console.log(`Starting to run tests using ${workerCount} threads...`); const { fork }: { fork(modulePath: string, args?: string[], options?: {}): ChildProcessPartial; } = require("child_process"); const totalFiles = tasks.length; - console.log(`1..${totalFiles}`); let passingFiles = 0; let failingFiles = 0; let errorResults: ErrorInfo[] = []; let totalPassing = 0; + const startTime = Date.now(); + + const progressBars = new ProgressBars({ noColors }); + progressBars.enable(); + updateProgress(0); + for (let i = 0; i < workerCount; i++) { // TODO: Just send the config over the IPC channel or in the command line arguments const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length !== 1 && i === workerCount - 1 }; @@ -325,50 +481,133 @@ function beginTestHost() { const child = fork(__filename, [`--config="${configPath}"`]); child.on("error", err => { child.disconnect(); + console.error("Unexpected error in child process:"); console.error(err); + return process.exit(2); }); child.on("exit", (code, _signal) => { - child.disconnect(); if (code !== 0) { console.error("Test worker process exited with nonzero exit code!"); + return process.exit(2); } }); child.on("message", (data: ParallelClientMessage) => { switch (data.type) { - case "error": - child.disconnect(); - console.error(`Test worker encounted unexpected error and was forced to close: -Message: ${data.payload.error} -Stack: ${data.payload.stack}`); - return; - case "result": - totalPassing += data.payload.passing; - if (data.payload.errors.length) { - errorResults = errorResults.concat(data.payload.errors); - failingFiles++; - console.log(`not ok ${passingFiles + failingFiles} - ${data.payload.errors.map(e => `test ${e.name} failed! Error: ${e.error.replace(/\r?\n/g, " ")} Stack: ${e.stack.replace(/\r?\n/g, " ")}`).join(" | ")}`); - } - else { - passingFiles++; - console.log(`ok ${passingFiles + failingFiles}`); - } - if (failingFiles + passingFiles === totalFiles) { - // Done. Finished every task and collected results. - child.send({ type: "close" }); + case "error": { child.disconnect(); - return process.exit(errorResults.length); + console.error(`Test worker encounted unexpected error and was forced to close: + Message: ${data.payload.error} + Stack: ${data.payload.stack}`); + return process.exit(2); } - if (tasks.length === 0) { - // No more tasks to distribute - child.send({ type: "close" }); - child.disconnect(); - return; + case "result": { + totalPassing += data.payload.passing; + if (data.payload.errors.length) { + errorResults = errorResults.concat(data.payload.errors); + failingFiles++; + } + else { + passingFiles++; + } + + updateProgress((failingFiles + passingFiles) / totalFiles, errorResults.length ? `${totalPassing}/${totalPassing + errorResults.length} passing` : `${totalPassing} passing`); + + if (failingFiles + passingFiles === totalFiles) { + // Done. Finished every task and collected results. + child.send({ type: "close" }); + child.disconnect(); + return outputFinalResult(); + } + if (tasks.length === 0) { + // No more tasks to distribute + child.send({ type: "close" }); + child.disconnect(); + return; + } + child.send({ type: "test", payload: tasks.pop() }); } - child.send({ type: "test", payload: tasks.pop() }); } }); child.send({ type: "test", payload: tasks.pop() }); } + + let duration: number; + + const ms = require("mocha/lib/ms"); + function completeBar() { + const isPartitionFail = failingFiles !== 0; + const summaryColor = isPartitionFail ? "fail" : "green"; + const summarySymbol = isPartitionFail ? Base.symbols.err : Base.symbols.ok; + + const summaryTests = (isPartitionFail ? totalPassing + "/" + (errorResults.length + totalPassing) : totalPassing) + " passing"; + const summaryDuration = "(" + ms(duration) + ")"; + const savedUseColors = Base.useColors; + Base.useColors = !noColors; + + const summary = color(summaryColor, summarySymbol + " " + summaryTests) + " " + color("light", summaryDuration); + Base.useColors = savedUseColors; + + updateProgress(1, summary); + } + + function updateProgress(percentComplete: number, title?: string) { + let progressColor = "pending"; + if (failingFiles) { + progressColor = "fail"; + } + + progressBars.update( + 0, + percentComplete, + progressColor, + title + ); + } + + function outputFinalResult() { + duration = Date.now() - startTime; + completeBar(); + progressBars.disable(); + + const reporter = new Base(); + const stats = reporter.stats; + const failures = reporter.failures; + stats.passes = totalPassing; + stats.failures = errorResults.length; + stats.tests = totalPassing + errorResults.length; + stats.duration = duration; + for (let j = 0; j < errorResults.length; j++) { + const failure = errorResults[j]; + failures.push(makeMochaTest(failure)); + } + if (noColors) { + const savedUseColors = Base.useColors; + Base.useColors = false; + reporter.epilogue(); + Base.useColors = savedUseColors; + } + else { + reporter.epilogue(); + } + + process.exit(errorResults.length); + } + + function makeMochaTest(test: ErrorInfo) { + return { + fullTitle: () => { + return test.name; + }, + err: { + message: test.error, + stack: test.stack + } + }; + } + + describe = ts.noop as any; // Disable unit tests + + return; } function startTestEnvironment() { @@ -385,4 +624,3 @@ function startTestEnvironment() { } startTestEnvironment(); - From ed4f787bb21b3390996f38cc009b7bae82015cf6 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Thu, 14 Sep 2017 12:04:25 -0700 Subject: [PATCH 3/5] Throttle console output --- src/harness/runner.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/harness/runner.ts b/src/harness/runner.ts index 900e1609ff877..d02dc2d752eff 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -336,7 +336,7 @@ function beginTestHost() { const isatty = tty.isatty(1) && tty.isatty(2); class ProgressBars { - private _options: ProgressBarsOptions; + public readonly _options: Readonly; private _enabled: boolean; private _lineCount: number; private _progressBars: ProgressBar[]; @@ -373,7 +373,7 @@ function beginTestHost() { this._enabled = false; } } - update(index: number, percentComplete: number, color: string, title: string) { + update(index: number, percentComplete: number, color: string, title: string, titleColor?: string) { percentComplete = minMax(percentComplete, 0, 1); const progressBar = this._progressBars[index] || (this._progressBars[index] = { }); @@ -395,7 +395,7 @@ function beginTestHost() { progress += this._color("progress", this._options.close); if (title) { - progress += this._color("progress", " " + title); + progress += this._color(titleColor || "progress", " " + title); } if (progressBar.text !== progress) { @@ -472,7 +472,8 @@ function beginTestHost() { const progressBars = new ProgressBars({ noColors }); progressBars.enable(); updateProgress(0); - + const progressUpdateInterval = 1 / progressBars._options.width; + let nextProgress = progressUpdateInterval; for (let i = 0; i < workerCount; i++) { // TODO: Just send the config over the IPC channel or in the command line arguments const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length !== 1 && i === workerCount - 1 }; @@ -510,7 +511,13 @@ function beginTestHost() { passingFiles++; } - updateProgress((failingFiles + passingFiles) / totalFiles, errorResults.length ? `${totalPassing}/${totalPassing + errorResults.length} passing` : `${totalPassing} passing`); + const progress = (failingFiles + passingFiles) / totalFiles; + if (progress >= nextProgress) { + while (nextProgress < progress) { + nextProgress += progressUpdateInterval; + } + updateProgress(progress, errorResults.length ? `${errorResults.length} failing` : `${totalPassing} passing`, errorResults.length ? "fail" : undefined); + } if (failingFiles + passingFiles === totalFiles) { // Done. Finished every task and collected results. @@ -550,7 +557,7 @@ function beginTestHost() { updateProgress(1, summary); } - function updateProgress(percentComplete: number, title?: string) { + function updateProgress(percentComplete: number, title?: string, titleColor?: string) { let progressColor = "pending"; if (failingFiles) { progressColor = "fail"; @@ -560,7 +567,8 @@ function beginTestHost() { 0, percentComplete, progressColor, - title + title, + titleColor ); } From f44f5acc11f9ffa51110bfbdd4f6c9363307dea6 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Thu, 14 Sep 2017 14:26:30 -0700 Subject: [PATCH 4/5] Batches test messages on large inputs initially --- src/harness/runner.ts | 134 +++++++++++++++++++++++++++++++++++------- 1 file changed, 114 insertions(+), 20 deletions(-) diff --git a/src/harness/runner.ts b/src/harness/runner.ts index d02dc2d752eff..c16abdcb9527c 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -59,6 +59,7 @@ function createRunner(kind: TestRunnerKind): RunnerBase { case "test262": return new Test262BaselineRunner(); } + ts.Debug.fail(`Unknown runner kind ${kind}`); } if (Harness.IO.tryEnableSourceMapsForHost && /^development$/i.test(Harness.IO.getEnvironmentVariable("NODE_ENV"))) { @@ -102,9 +103,6 @@ interface TaskSet { function handleTestConfig() { if (testConfigContent !== "") { const testConfig = JSON.parse(testConfigContent); - if (testConfig.listenForWork) { - return true; - } if (testConfig.light) { Harness.lightMode = true; } @@ -112,7 +110,7 @@ function handleTestConfig() { runUnitTests = testConfig.runUnitTests; } if (testConfig.workerCount) { - workerCount = testConfig.workerCount; + workerCount = +testConfig.workerCount; } if (testConfig.taskConfigsFolder) { taskConfigsFolder = testConfig.taskConfigsFolder; @@ -127,6 +125,9 @@ function handleTestConfig() { else if ((+testConfig.stackTraceLimit | 0) > 0) { (Error).stackTraceLimit = testConfig.stackTraceLimit; } + if (testConfig.listenForWork) { + return true; + } if (testConfig.test && testConfig.test.length > 0) { for (const option of testConfig.test) { @@ -205,13 +206,15 @@ function beginTests() { } type ParallelTestMessage = { type: "test", payload: { runner: TestRunnerKind, file: string } } | never; +type ParallelBatchMessage = { type: "batch", payload: ParallelTestMessage["payload"][] } | never; type ParallelCloseMessage = { type: "close" } | never; -type ParallelHostMessage = ParallelTestMessage | ParallelCloseMessage; +type ParallelHostMessage = ParallelTestMessage | ParallelCloseMessage | ParallelBatchMessage; type ParallelErrorMessage = { type: "error", payload: { error: string, stack: string } } | never; type ErrorInfo = ParallelErrorMessage["payload"] & { name: string }; type ParallelResultMessage = { type: "result", payload: { passing: number, errors: ErrorInfo[] } } | never; -type ParallelClientMessage = ParallelErrorMessage | ParallelResultMessage; +type ParallelBatchProgressMessage = { type: "progress", payload: ParallelResultMessage["payload"] } | never; +type ParallelClientMessage = ParallelErrorMessage | ParallelResultMessage | ParallelBatchProgressMessage; let errors: ErrorInfo[] = []; let passing = 0; @@ -272,31 +275,67 @@ function shimMochaHarness() { } function beginWorkListener() { - shimMochaHarness(); + let initialized = false; const runners = ts.createMap(); process.on("message", (data: ParallelHostMessage) => { + if (!initialized) { + initialized = true; + shimMochaHarness(); + } switch (data.type) { case "test": const { runner, file } = data.payload; - if (!runners.has(runner)) { - runners.set(runner, createRunner(runner)); + if (!runner) { + console.error(data); } - const instance = runners.get(runner); - instance.tests = []; - instance.addTest(file); - const payload = resetShimHarnessAndExecute(instance); - const message: ParallelResultMessage = { type: "result", payload }; + const message: ParallelResultMessage = { type: "result", payload: handleTest(runner, file) }; process.send(message); break; case "close": process.exit(0); break; + case "batch": { + const items = data.payload; + for (let i = 0; i < items.length; i++) { + const { runner, file } = items[i]; + if (!runner) { + console.error(data); + } + let message: ParallelBatchProgressMessage | ParallelResultMessage; + const payload = handleTest(runner, file); + if (i === (items.length - 1)) { + message = { type: "result", payload }; + } + else { + message = { type: "progress", payload }; + } + process.send(message); + } + break; + } } }); process.on("uncaughtException", error => { const message: ParallelErrorMessage = { type: "error", payload: { error: error.message, stack: error.stack } }; process.send(message); }); + if (!runUnitTests) { + // ensure unit tests do not get run + describe = ts.noop as any; + } + else { + initialized = true; + shimMochaHarness(); + } + + function handleTest(runner: TestRunnerKind, file: string) { + if (!runners.has(runner)) { + runners.set(runner, createRunner(runner)); + } + const instance = runners.get(runner); + instance.tests = [file]; + return resetShimHarnessAndExecute(instance); + } } interface ChildProcessPartial { @@ -451,13 +490,17 @@ function beginTestHost() { const discoverStart = +(new Date()); const { statSync }: { statSync(path: string): { size: number }; } = require("fs"); const tasks: { runner: TestRunnerKind, file: string, size: number }[] = []; + let totalSize = 0; for (const runner of runners) { const files = runner.enumerateTestFiles(); for (const file of files) { - tasks.push({ runner: runner.kind(), file, size: statSync(file).size }); + const size = statSync(file).size; + tasks.push({ runner: runner.kind(), file, size }); + totalSize += size; } } tasks.sort((a, b) => a.size - b.size); + const batchSize = (totalSize / workerCount) * 0.9; console.log(`Discovered ${tasks.length} test files in ${+(new Date()) - discoverStart}ms.`); console.log(`Starting to run tests using ${workerCount} threads...`); const { fork }: { fork(modulePath: string, args?: string[], options?: {}): ChildProcessPartial; } = require("child_process"); @@ -470,13 +513,13 @@ function beginTestHost() { const startTime = Date.now(); const progressBars = new ProgressBars({ noColors }); - progressBars.enable(); - updateProgress(0); const progressUpdateInterval = 1 / progressBars._options.width; let nextProgress = progressUpdateInterval; + + const workers: ChildProcessPartial[] = []; for (let i = 0; i < workerCount; i++) { // TODO: Just send the config over the IPC channel or in the command line arguments - const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length !== 1 && i === workerCount - 1 }; + const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length === 1 ? false : i === workerCount - 1 }; const configPath = ts.combinePaths(taskConfigsFolder, `task-config${i}.json`); Harness.IO.writeFile(configPath, JSON.stringify(config)); const child = fork(__filename, [`--config="${configPath}"`]); @@ -501,6 +544,7 @@ function beginTestHost() { Stack: ${data.payload.stack}`); return process.exit(2); } + case "progress": case "result": { totalPassing += data.payload.passing; if (data.payload.errors.length) { @@ -531,13 +575,63 @@ function beginTestHost() { child.disconnect(); return; } - child.send({ type: "test", payload: tasks.pop() }); + if (data.type === "result") { + child.send({ type: "test", payload: tasks.pop() }); + } } } }); - child.send({ type: "test", payload: tasks.pop() }); + workers.push(child); + } + + // It's only really worth doing an initial batching if there are a ton of files to go through + if (totalFiles > 1000) { + console.log("Batching initial test lists..."); + const batches: { runner: TestRunnerKind, file: string, size: number }[][] = new Array(workerCount); + const doneBatching = new Array(workerCount); + batcher: while (true) { + for (let i = 0; i < workerCount; i++) { + if (tasks.length === 0) { + // TODO: This indicates a particularly suboptimal packing + break batcher; + } + if (doneBatching[i]) { + continue; + } + if (!batches[i]) { + batches[i] = []; + } + const total = batches[i].reduce((p, c) => p + c.size, 0); + if (total >= batchSize && !doneBatching[i]) { + doneBatching[i] = true; + continue; + } + batches[i].push(tasks.pop()); + } + for (let j = 0; j < workerCount; j++) { + if (!doneBatching[j]) { + continue; + } + } + break; + } + console.log(`Batched into ${workerCount} groups with approximate total file sizes of ${Math.floor(batchSize)} bytes in each group.`); + for (const worker of workers) { + const action: ParallelBatchMessage = { type: "batch", payload: batches.pop() }; + if (!action.payload[0]) { + throw new Error(`Tried to send invalid message ${action}`); + } + worker.send(action); + } + } + else { + for (let i = 0; i < workerCount; i++) { + workers[i].send({ type: "test", payload: tasks.pop() }); + } } + progressBars.enable(); + updateProgress(0); let duration: number; const ms = require("mocha/lib/ms"); From c0a812015cc9e607e648507e5fd0a2c08112be40 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Thu, 14 Sep 2017 15:29:48 -0700 Subject: [PATCH 5/5] Move parallel runner code into seperate files --- Jakefile.js | 3 + src/harness/parallel/host.ts | 376 ++++++++++++++++++++++++ src/harness/parallel/shared.ts | 14 + src/harness/parallel/worker.ts | 123 ++++++++ src/harness/runner.ts | 512 +-------------------------------- src/harness/tsconfig.json | 3 + 6 files changed, 522 insertions(+), 509 deletions(-) create mode 100644 src/harness/parallel/host.ts create mode 100644 src/harness/parallel/shared.ts create mode 100644 src/harness/parallel/worker.ts diff --git a/Jakefile.js b/Jakefile.js index 914839efbcce5..39e9e8a0421d5 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -104,6 +104,9 @@ var harnessCoreSources = [ "loggedIO.ts", "rwcRunner.ts", "test262Runner.ts", + "./parallel/shared.ts", + "./parallel/host.ts", + "./parallel/worker.ts", "runner.ts" ].map(function (f) { return path.join(harnessDirectory, f); diff --git a/src/harness/parallel/host.ts b/src/harness/parallel/host.ts new file mode 100644 index 0000000000000..58e794bbc7d9e --- /dev/null +++ b/src/harness/parallel/host.ts @@ -0,0 +1,376 @@ +// tslint:disable-next-line +var describe: Mocha.IContextDefinition; // If launched without mocha for parallel mode, we still need a global describe visible to satisfy the parsing of the unit tests +// tslint:disable-next-line +var it: Mocha.ITestDefinition; +namespace Harness.Parallel.Host { + + interface ChildProcessPartial { + send(message: any, callback?: (error: Error) => void): boolean; + on(event: "error", listener: (err: Error) => void): this; + on(event: "exit", listener: (code: number, signal: string) => void): this; + on(event: "message", listener: (message: any) => void): this; + disconnect(): void; + } + + interface ProgressBarsOptions { + open: string; + close: string; + complete: string; + incomplete: string; + width: number; + noColors: boolean; + } + interface ProgressBar { + lastN?: number; + title?: string; + progressColor?: string; + text?: string; + } + + export function start() { + console.log("Discovering tests..."); + const discoverStart = +(new Date()); + const { statSync }: { statSync(path: string): { size: number }; } = require("fs"); + const tasks: { runner: TestRunnerKind, file: string, size: number }[] = []; + let totalSize = 0; + for (const runner of runners) { + const files = runner.enumerateTestFiles(); + for (const file of files) { + const size = statSync(file).size; + tasks.push({ runner: runner.kind(), file, size }); + totalSize += size; + } + } + tasks.sort((a, b) => a.size - b.size); + const batchSize = (totalSize / workerCount) * 0.9; + console.log(`Discovered ${tasks.length} test files in ${+(new Date()) - discoverStart}ms.`); + console.log(`Starting to run tests using ${workerCount} threads...`); + const { fork }: { fork(modulePath: string, args?: string[], options?: {}): ChildProcessPartial; } = require("child_process"); + + const totalFiles = tasks.length; + let passingFiles = 0; + let failingFiles = 0; + let errorResults: ErrorInfo[] = []; + let totalPassing = 0; + const startTime = Date.now(); + + const progressBars = new ProgressBars({ noColors }); + const progressUpdateInterval = 1 / progressBars._options.width; + let nextProgress = progressUpdateInterval; + + const workers: ChildProcessPartial[] = []; + for (let i = 0; i < workerCount; i++) { + // TODO: Just send the config over the IPC channel or in the command line arguments + const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length === 1 ? false : i === workerCount - 1 }; + const configPath = ts.combinePaths(taskConfigsFolder, `task-config${i}.json`); + Harness.IO.writeFile(configPath, JSON.stringify(config)); + const child = fork(__filename, [`--config="${configPath}"`]); + child.on("error", err => { + child.disconnect(); + console.error("Unexpected error in child process:"); + console.error(err); + return process.exit(2); + }); + child.on("exit", (code, _signal) => { + if (code !== 0) { + console.error("Test worker process exited with nonzero exit code!"); + return process.exit(2); + } + }); + child.on("message", (data: ParallelClientMessage) => { + switch (data.type) { + case "error": { + child.disconnect(); + console.error(`Test worker encounted unexpected error and was forced to close: + Message: ${data.payload.error} + Stack: ${data.payload.stack}`); + return process.exit(2); + } + case "progress": + case "result": { + totalPassing += data.payload.passing; + if (data.payload.errors.length) { + errorResults = errorResults.concat(data.payload.errors); + failingFiles++; + } + else { + passingFiles++; + } + + const progress = (failingFiles + passingFiles) / totalFiles; + if (progress >= nextProgress) { + while (nextProgress < progress) { + nextProgress += progressUpdateInterval; + } + updateProgress(progress, errorResults.length ? `${errorResults.length} failing` : `${totalPassing} passing`, errorResults.length ? "fail" : undefined); + } + + if (failingFiles + passingFiles === totalFiles) { + // Done. Finished every task and collected results. + child.send({ type: "close" }); + child.disconnect(); + return outputFinalResult(); + } + if (tasks.length === 0) { + // No more tasks to distribute + child.send({ type: "close" }); + child.disconnect(); + return; + } + if (data.type === "result") { + child.send({ type: "test", payload: tasks.pop() }); + } + } + } + }); + workers.push(child); + } + + // It's only really worth doing an initial batching if there are a ton of files to go through + if (totalFiles > 1000) { + console.log("Batching initial test lists..."); + const batches: { runner: TestRunnerKind, file: string, size: number }[][] = new Array(workerCount); + const doneBatching = new Array(workerCount); + batcher: while (true) { + for (let i = 0; i < workerCount; i++) { + if (tasks.length === 0) { + // TODO: This indicates a particularly suboptimal packing + break batcher; + } + if (doneBatching[i]) { + continue; + } + if (!batches[i]) { + batches[i] = []; + } + const total = batches[i].reduce((p, c) => p + c.size, 0); + if (total >= batchSize && !doneBatching[i]) { + doneBatching[i] = true; + continue; + } + batches[i].push(tasks.pop()); + } + for (let j = 0; j < workerCount; j++) { + if (!doneBatching[j]) { + continue; + } + } + break; + } + console.log(`Batched into ${workerCount} groups with approximate total file sizes of ${Math.floor(batchSize)} bytes in each group.`); + for (const worker of workers) { + const action: ParallelBatchMessage = { type: "batch", payload: batches.pop() }; + if (!action.payload[0]) { + throw new Error(`Tried to send invalid message ${action}`); + } + worker.send(action); + } + } + else { + for (let i = 0; i < workerCount; i++) { + workers[i].send({ type: "test", payload: tasks.pop() }); + } + } + + progressBars.enable(); + updateProgress(0); + let duration: number; + + const ms = require("mocha/lib/ms"); + function completeBar() { + const isPartitionFail = failingFiles !== 0; + const summaryColor = isPartitionFail ? "fail" : "green"; + const summarySymbol = isPartitionFail ? Base.symbols.err : Base.symbols.ok; + + const summaryTests = (isPartitionFail ? totalPassing + "/" + (errorResults.length + totalPassing) : totalPassing) + " passing"; + const summaryDuration = "(" + ms(duration) + ")"; + const savedUseColors = Base.useColors; + Base.useColors = !noColors; + + const summary = color(summaryColor, summarySymbol + " " + summaryTests) + " " + color("light", summaryDuration); + Base.useColors = savedUseColors; + + updateProgress(1, summary); + } + + function updateProgress(percentComplete: number, title?: string, titleColor?: string) { + let progressColor = "pending"; + if (failingFiles) { + progressColor = "fail"; + } + + progressBars.update( + 0, + percentComplete, + progressColor, + title, + titleColor + ); + } + + function outputFinalResult() { + duration = Date.now() - startTime; + completeBar(); + progressBars.disable(); + + const reporter = new Base(); + const stats = reporter.stats; + const failures = reporter.failures; + stats.passes = totalPassing; + stats.failures = errorResults.length; + stats.tests = totalPassing + errorResults.length; + stats.duration = duration; + for (let j = 0; j < errorResults.length; j++) { + const failure = errorResults[j]; + failures.push(makeMochaTest(failure)); + } + if (noColors) { + const savedUseColors = Base.useColors; + Base.useColors = false; + reporter.epilogue(); + Base.useColors = savedUseColors; + } + else { + reporter.epilogue(); + } + + process.exit(errorResults.length); + } + + function makeMochaTest(test: ErrorInfo) { + return { + fullTitle: () => { + return test.name; + }, + err: { + message: test.error, + stack: test.stack + } + }; + } + + describe = ts.noop as any; // Disable unit tests + + return; + } + + const Mocha = require("mocha"); + const Base = Mocha.reporters.Base; + const color = Base.color; + const cursor = Base.cursor; + const readline = require("readline"); + const os = require("os"); + const tty: { isatty(x: number): boolean } = require("tty"); + const isatty = tty.isatty(1) && tty.isatty(2); + class ProgressBars { + public readonly _options: Readonly; + private _enabled: boolean; + private _lineCount: number; + private _progressBars: ProgressBar[]; + constructor(options?: Partial) { + if (!options) options = {}; + const open = options.open || "["; + const close = options.close || "]"; + const complete = options.complete || "▬"; + const incomplete = options.incomplete || Base.symbols.dot; + const maxWidth = Base.window.width - open.length - close.length - 30; + const width = minMax(options.width || maxWidth, 10, maxWidth); + this._options = { + open, + complete, + incomplete, + close, + width, + noColors: options.noColors || false + }; + + this._progressBars = []; + this._lineCount = 0; + this._enabled = false; + } + enable() { + if (!this._enabled) { + process.stdout.write(os.EOL); + this._enabled = true; + } + } + disable() { + if (this._enabled) { + process.stdout.write(os.EOL); + this._enabled = false; + } + } + update(index: number, percentComplete: number, color: string, title: string, titleColor?: string) { + percentComplete = minMax(percentComplete, 0, 1); + + const progressBar = this._progressBars[index] || (this._progressBars[index] = { }); + const width = this._options.width; + const n = Math.floor(width * percentComplete); + const i = width - n; + if (n === progressBar.lastN && title === progressBar.title && color === progressBar.progressColor) { + return; + } + + progressBar.lastN = n; + progressBar.title = title; + progressBar.progressColor = color; + + let progress = " "; + progress += this._color("progress", this._options.open); + progress += this._color(color, fill(this._options.complete, n)); + progress += this._color("progress", fill(this._options.incomplete, i)); + progress += this._color("progress", this._options.close); + + if (title) { + progress += this._color(titleColor || "progress", " " + title); + } + + if (progressBar.text !== progress) { + progressBar.text = progress; + this._render(index); + } + } + private _render(index: number) { + if (!this._enabled || !isatty) { + return; + } + + cursor.hide(); + readline.moveCursor(process.stdout, -process.stdout.columns, -this._lineCount); + let lineCount = 0; + const numProgressBars = this._progressBars.length; + for (let i = 0; i < numProgressBars; i++) { + if (i === index) { + readline.clearLine(process.stdout, 1); + process.stdout.write(this._progressBars[i].text + os.EOL); + } + else { + readline.moveCursor(process.stdout, -process.stdout.columns, +1); + } + + lineCount++; + } + + this._lineCount = lineCount; + cursor.show(); + } + private _color(type: string, text: string) { + return type && !this._options.noColors ? color(type, text) : text; + } + } + + function fill(ch: string, size: number) { + let s = ""; + while (s.length < size) { + s += ch; + } + + return s.length > size ? s.substr(0, size) : s; + } + + function minMax(value: number, min: number, max: number) { + if (value < min) return min; + if (value > max) return max; + return value; + } +} \ No newline at end of file diff --git a/src/harness/parallel/shared.ts b/src/harness/parallel/shared.ts new file mode 100644 index 0000000000000..ebfe327884993 --- /dev/null +++ b/src/harness/parallel/shared.ts @@ -0,0 +1,14 @@ +/// +/// +namespace Harness.Parallel { + export type ParallelTestMessage = { type: "test", payload: { runner: TestRunnerKind, file: string } } | never; + export type ParallelBatchMessage = { type: "batch", payload: ParallelTestMessage["payload"][] } | never; + export type ParallelCloseMessage = { type: "close" } | never; + export type ParallelHostMessage = ParallelTestMessage | ParallelCloseMessage | ParallelBatchMessage; + + export type ParallelErrorMessage = { type: "error", payload: { error: string, stack: string } } | never; + export type ErrorInfo = ParallelErrorMessage["payload"] & { name: string }; + export type ParallelResultMessage = { type: "result", payload: { passing: number, errors: ErrorInfo[] } } | never; + export type ParallelBatchProgressMessage = { type: "progress", payload: ParallelResultMessage["payload"] } | never; + export type ParallelClientMessage = ParallelErrorMessage | ParallelResultMessage | ParallelBatchProgressMessage; +} \ No newline at end of file diff --git a/src/harness/parallel/worker.ts b/src/harness/parallel/worker.ts new file mode 100644 index 0000000000000..34f89d37f130e --- /dev/null +++ b/src/harness/parallel/worker.ts @@ -0,0 +1,123 @@ +namespace Harness.Parallel.Worker { + let errors: ErrorInfo[] = []; + let passing = 0; + function resetShimHarnessAndExecute(runner: RunnerBase) { + errors = []; + passing = 0; + runner.initializeTests(); + return { errors, passing }; + } + + function shimMochaHarness() { + (global as any).before = undefined; + (global as any).after = undefined; + (global as any).beforeEach = undefined; + let beforeEachFunc: Function; + describe = ((_name, callback) => { + const fakeContext: Mocha.ISuiteCallbackContext = { + retries() { return this; }, + slow() { return this; }, + timeout() { return this; }, + }; + (before as any) = (cb: Function) => cb(); + let afterFunc: Function; + (after as any) = (cb: Function) => afterFunc = cb; + const savedBeforeEach = beforeEachFunc; + (beforeEach as any) = (cb: Function) => beforeEachFunc = cb; + callback.call(fakeContext); + afterFunc && afterFunc(); + afterFunc = undefined; + beforeEachFunc = savedBeforeEach; + }) as Mocha.IContextDefinition; + it = ((name, callback) => { + const fakeContext: Mocha.ITestCallbackContext = { + skip() { return this; }, + timeout() { return this; }, + retries() { return this; }, + slow() { return this; }, + }; + // TODO: If we ever start using async test completions, polyfill the `done` parameter/promise return handling + if (beforeEachFunc) { + try { + beforeEachFunc(); + } + catch (error) { + errors.push({ error: error.message, stack: error.stack, name }); + return; + } + } + try { + callback.call(fakeContext); + } + catch (error) { + errors.push({ error: error.message, stack: error.stack, name }); + return; + } + passing++; + }) as Mocha.ITestDefinition; + } + + export function start() { + let initialized = false; + const runners = ts.createMap(); + process.on("message", (data: ParallelHostMessage) => { + if (!initialized) { + initialized = true; + shimMochaHarness(); + } + switch (data.type) { + case "test": + const { runner, file } = data.payload; + if (!runner) { + console.error(data); + } + const message: ParallelResultMessage = { type: "result", payload: handleTest(runner, file) }; + process.send(message); + break; + case "close": + process.exit(0); + break; + case "batch": { + const items = data.payload; + for (let i = 0; i < items.length; i++) { + const { runner, file } = items[i]; + if (!runner) { + console.error(data); + } + let message: ParallelBatchProgressMessage | ParallelResultMessage; + const payload = handleTest(runner, file); + if (i === (items.length - 1)) { + message = { type: "result", payload }; + } + else { + message = { type: "progress", payload }; + } + process.send(message); + } + break; + } + } + }); + process.on("uncaughtException", error => { + const message: ParallelErrorMessage = { type: "error", payload: { error: error.message, stack: error.stack } }; + process.send(message); + }); + if (!runUnitTests) { + // ensure unit tests do not get run + describe = ts.noop as any; + } + else { + initialized = true; + shimMochaHarness(); + } + + function handleTest(runner: TestRunnerKind, file: string) { + if (!runners.has(runner)) { + runners.set(runner, createRunner(runner)); + } + const instance = runners.get(runner); + instance.tests = [file]; + return resetShimHarnessAndExecute(instance); + } + } +} \ No newline at end of file diff --git a/src/harness/runner.ts b/src/harness/runner.ts index c16abdcb9527c..0b361e7fc9eb1 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -19,6 +19,7 @@ /// /// /// +/// let runners: RunnerBase[] = []; let iterations = 1; @@ -205,521 +206,14 @@ function beginTests() { } } -type ParallelTestMessage = { type: "test", payload: { runner: TestRunnerKind, file: string } } | never; -type ParallelBatchMessage = { type: "batch", payload: ParallelTestMessage["payload"][] } | never; -type ParallelCloseMessage = { type: "close" } | never; -type ParallelHostMessage = ParallelTestMessage | ParallelCloseMessage | ParallelBatchMessage; - -type ParallelErrorMessage = { type: "error", payload: { error: string, stack: string } } | never; -type ErrorInfo = ParallelErrorMessage["payload"] & { name: string }; -type ParallelResultMessage = { type: "result", payload: { passing: number, errors: ErrorInfo[] } } | never; -type ParallelBatchProgressMessage = { type: "progress", payload: ParallelResultMessage["payload"] } | never; -type ParallelClientMessage = ParallelErrorMessage | ParallelResultMessage | ParallelBatchProgressMessage; - -let errors: ErrorInfo[] = []; -let passing = 0; -function resetShimHarnessAndExecute(runner: RunnerBase) { - errors = []; - passing = 0; - runner.initializeTests(); - return { errors, passing }; -} - -function shimMochaHarness() { - (global as any).before = undefined; - (global as any).after = undefined; - (global as any).beforeEach = undefined; - let beforeEachFunc: Function; - describe = ((_name, callback) => { - const fakeContext: Mocha.ISuiteCallbackContext = { - retries() { return this; }, - slow() { return this; }, - timeout() { return this; }, - }; - (before as any) = (cb: Function) => cb(); - let afterFunc: Function; - (after as any) = (cb: Function) => afterFunc = cb; - const savedBeforeEach = beforeEachFunc; - (beforeEach as any) = (cb: Function) => beforeEachFunc = cb; - callback.call(fakeContext); - afterFunc && afterFunc(); - afterFunc = undefined; - beforeEachFunc = savedBeforeEach; - }) as Mocha.IContextDefinition; - it = ((name, callback) => { - const fakeContext: Mocha.ITestCallbackContext = { - skip() { return this; }, - timeout() { return this; }, - retries() { return this; }, - slow() { return this; }, - }; - // TODO: If we ever start using async test completions, polyfill the `done` parameter/promise return handling - if (beforeEachFunc) { - try { - beforeEachFunc(); - } - catch (error) { - errors.push({ error: error.message, stack: error.stack, name }); - return; - } - } - try { - callback.call(fakeContext); - } - catch (error) { - errors.push({ error: error.message, stack: error.stack, name }); - return; - } - passing++; - }) as Mocha.ITestDefinition; -} - -function beginWorkListener() { - let initialized = false; - const runners = ts.createMap(); - process.on("message", (data: ParallelHostMessage) => { - if (!initialized) { - initialized = true; - shimMochaHarness(); - } - switch (data.type) { - case "test": - const { runner, file } = data.payload; - if (!runner) { - console.error(data); - } - const message: ParallelResultMessage = { type: "result", payload: handleTest(runner, file) }; - process.send(message); - break; - case "close": - process.exit(0); - break; - case "batch": { - const items = data.payload; - for (let i = 0; i < items.length; i++) { - const { runner, file } = items[i]; - if (!runner) { - console.error(data); - } - let message: ParallelBatchProgressMessage | ParallelResultMessage; - const payload = handleTest(runner, file); - if (i === (items.length - 1)) { - message = { type: "result", payload }; - } - else { - message = { type: "progress", payload }; - } - process.send(message); - } - break; - } - } - }); - process.on("uncaughtException", error => { - const message: ParallelErrorMessage = { type: "error", payload: { error: error.message, stack: error.stack } }; - process.send(message); - }); - if (!runUnitTests) { - // ensure unit tests do not get run - describe = ts.noop as any; - } - else { - initialized = true; - shimMochaHarness(); - } - - function handleTest(runner: TestRunnerKind, file: string) { - if (!runners.has(runner)) { - runners.set(runner, createRunner(runner)); - } - const instance = runners.get(runner); - instance.tests = [file]; - return resetShimHarnessAndExecute(instance); - } -} - -interface ChildProcessPartial { - send(message: any, callback?: (error: Error) => void): boolean; - on(event: "error", listener: (err: Error) => void): this; - on(event: "exit", listener: (code: number, signal: string) => void): this; - on(event: "message", listener: (message: any) => void): this; - disconnect(): void; -} - -// tslint:disable-next-line -var describe: Mocha.IContextDefinition; // If launched without mocha for parallel mode, we still need a global describe visible to satisfy the parsing of the unit tests -// tslint:disable-next-line -var it: Mocha.ITestDefinition; -function beginTestHost() { - const Mocha = require("mocha"); - const Base = Mocha.reporters.Base; - const color = Base.color; - const cursor = Base.cursor; - const readline = require("readline"); - const os = require("os"); - interface ProgressBarsOptions { - open: string; - close: string; - complete: string; - incomplete: string; - width: number; - noColors: boolean; - } - interface ProgressBar { - lastN?: number; - title?: string; - progressColor?: string; - text?: string; - } - const tty: { isatty(x: number): boolean } = require("tty"); - const isatty = tty.isatty(1) && tty.isatty(2); - - class ProgressBars { - public readonly _options: Readonly; - private _enabled: boolean; - private _lineCount: number; - private _progressBars: ProgressBar[]; - constructor(options?: Partial) { - if (!options) options = {}; - const open = options.open || "["; - const close = options.close || "]"; - const complete = options.complete || "▬"; - const incomplete = options.incomplete || Base.symbols.dot; - const maxWidth = Base.window.width - open.length - close.length - 30; - const width = minMax(options.width || maxWidth, 10, maxWidth); - this._options = { - open, - complete, - incomplete, - close, - width, - noColors: options.noColors || false - }; - - this._progressBars = []; - this._lineCount = 0; - this._enabled = false; - } - enable() { - if (!this._enabled) { - process.stdout.write(os.EOL); - this._enabled = true; - } - } - disable() { - if (this._enabled) { - process.stdout.write(os.EOL); - this._enabled = false; - } - } - update(index: number, percentComplete: number, color: string, title: string, titleColor?: string) { - percentComplete = minMax(percentComplete, 0, 1); - - const progressBar = this._progressBars[index] || (this._progressBars[index] = { }); - const width = this._options.width; - const n = Math.floor(width * percentComplete); - const i = width - n; - if (n === progressBar.lastN && title === progressBar.title && color === progressBar.progressColor) { - return; - } - - progressBar.lastN = n; - progressBar.title = title; - progressBar.progressColor = color; - - let progress = " "; - progress += this._color("progress", this._options.open); - progress += this._color(color, fill(this._options.complete, n)); - progress += this._color("progress", fill(this._options.incomplete, i)); - progress += this._color("progress", this._options.close); - - if (title) { - progress += this._color(titleColor || "progress", " " + title); - } - - if (progressBar.text !== progress) { - progressBar.text = progress; - this._render(index); - } - } - private _render(index: number) { - if (!this._enabled || !isatty) { - return; - } - - cursor.hide(); - readline.moveCursor(process.stdout, -process.stdout.columns, -this._lineCount); - let lineCount = 0; - const numProgressBars = this._progressBars.length; - for (let i = 0; i < numProgressBars; i++) { - if (i === index) { - readline.clearLine(process.stdout, 1); - process.stdout.write(this._progressBars[i].text + os.EOL); - } - else { - readline.moveCursor(process.stdout, -process.stdout.columns, +1); - } - - lineCount++; - } - - this._lineCount = lineCount; - cursor.show(); - } - private _color(type: string, text: string) { - return type && !this._options.noColors ? color(type, text) : text; - } - } - - function fill(ch: string, size: number) { - let s = ""; - while (s.length < size) { - s += ch; - } - - return s.length > size ? s.substr(0, size) : s; - } - - function minMax(value: number, min: number, max: number) { - if (value < min) return min; - if (value > max) return max; - return value; - } - - console.log("Discovering tests..."); - const discoverStart = +(new Date()); - const { statSync }: { statSync(path: string): { size: number }; } = require("fs"); - const tasks: { runner: TestRunnerKind, file: string, size: number }[] = []; - let totalSize = 0; - for (const runner of runners) { - const files = runner.enumerateTestFiles(); - for (const file of files) { - const size = statSync(file).size; - tasks.push({ runner: runner.kind(), file, size }); - totalSize += size; - } - } - tasks.sort((a, b) => a.size - b.size); - const batchSize = (totalSize / workerCount) * 0.9; - console.log(`Discovered ${tasks.length} test files in ${+(new Date()) - discoverStart}ms.`); - console.log(`Starting to run tests using ${workerCount} threads...`); - const { fork }: { fork(modulePath: string, args?: string[], options?: {}): ChildProcessPartial; } = require("child_process"); - - const totalFiles = tasks.length; - let passingFiles = 0; - let failingFiles = 0; - let errorResults: ErrorInfo[] = []; - let totalPassing = 0; - const startTime = Date.now(); - - const progressBars = new ProgressBars({ noColors }); - const progressUpdateInterval = 1 / progressBars._options.width; - let nextProgress = progressUpdateInterval; - - const workers: ChildProcessPartial[] = []; - for (let i = 0; i < workerCount; i++) { - // TODO: Just send the config over the IPC channel or in the command line arguments - const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length === 1 ? false : i === workerCount - 1 }; - const configPath = ts.combinePaths(taskConfigsFolder, `task-config${i}.json`); - Harness.IO.writeFile(configPath, JSON.stringify(config)); - const child = fork(__filename, [`--config="${configPath}"`]); - child.on("error", err => { - child.disconnect(); - console.error("Unexpected error in child process:"); - console.error(err); - return process.exit(2); - }); - child.on("exit", (code, _signal) => { - if (code !== 0) { - console.error("Test worker process exited with nonzero exit code!"); - return process.exit(2); - } - }); - child.on("message", (data: ParallelClientMessage) => { - switch (data.type) { - case "error": { - child.disconnect(); - console.error(`Test worker encounted unexpected error and was forced to close: - Message: ${data.payload.error} - Stack: ${data.payload.stack}`); - return process.exit(2); - } - case "progress": - case "result": { - totalPassing += data.payload.passing; - if (data.payload.errors.length) { - errorResults = errorResults.concat(data.payload.errors); - failingFiles++; - } - else { - passingFiles++; - } - - const progress = (failingFiles + passingFiles) / totalFiles; - if (progress >= nextProgress) { - while (nextProgress < progress) { - nextProgress += progressUpdateInterval; - } - updateProgress(progress, errorResults.length ? `${errorResults.length} failing` : `${totalPassing} passing`, errorResults.length ? "fail" : undefined); - } - - if (failingFiles + passingFiles === totalFiles) { - // Done. Finished every task and collected results. - child.send({ type: "close" }); - child.disconnect(); - return outputFinalResult(); - } - if (tasks.length === 0) { - // No more tasks to distribute - child.send({ type: "close" }); - child.disconnect(); - return; - } - if (data.type === "result") { - child.send({ type: "test", payload: tasks.pop() }); - } - } - } - }); - workers.push(child); - } - - // It's only really worth doing an initial batching if there are a ton of files to go through - if (totalFiles > 1000) { - console.log("Batching initial test lists..."); - const batches: { runner: TestRunnerKind, file: string, size: number }[][] = new Array(workerCount); - const doneBatching = new Array(workerCount); - batcher: while (true) { - for (let i = 0; i < workerCount; i++) { - if (tasks.length === 0) { - // TODO: This indicates a particularly suboptimal packing - break batcher; - } - if (doneBatching[i]) { - continue; - } - if (!batches[i]) { - batches[i] = []; - } - const total = batches[i].reduce((p, c) => p + c.size, 0); - if (total >= batchSize && !doneBatching[i]) { - doneBatching[i] = true; - continue; - } - batches[i].push(tasks.pop()); - } - for (let j = 0; j < workerCount; j++) { - if (!doneBatching[j]) { - continue; - } - } - break; - } - console.log(`Batched into ${workerCount} groups with approximate total file sizes of ${Math.floor(batchSize)} bytes in each group.`); - for (const worker of workers) { - const action: ParallelBatchMessage = { type: "batch", payload: batches.pop() }; - if (!action.payload[0]) { - throw new Error(`Tried to send invalid message ${action}`); - } - worker.send(action); - } - } - else { - for (let i = 0; i < workerCount; i++) { - workers[i].send({ type: "test", payload: tasks.pop() }); - } - } - - progressBars.enable(); - updateProgress(0); - let duration: number; - - const ms = require("mocha/lib/ms"); - function completeBar() { - const isPartitionFail = failingFiles !== 0; - const summaryColor = isPartitionFail ? "fail" : "green"; - const summarySymbol = isPartitionFail ? Base.symbols.err : Base.symbols.ok; - - const summaryTests = (isPartitionFail ? totalPassing + "/" + (errorResults.length + totalPassing) : totalPassing) + " passing"; - const summaryDuration = "(" + ms(duration) + ")"; - const savedUseColors = Base.useColors; - Base.useColors = !noColors; - - const summary = color(summaryColor, summarySymbol + " " + summaryTests) + " " + color("light", summaryDuration); - Base.useColors = savedUseColors; - - updateProgress(1, summary); - } - - function updateProgress(percentComplete: number, title?: string, titleColor?: string) { - let progressColor = "pending"; - if (failingFiles) { - progressColor = "fail"; - } - - progressBars.update( - 0, - percentComplete, - progressColor, - title, - titleColor - ); - } - - function outputFinalResult() { - duration = Date.now() - startTime; - completeBar(); - progressBars.disable(); - - const reporter = new Base(); - const stats = reporter.stats; - const failures = reporter.failures; - stats.passes = totalPassing; - stats.failures = errorResults.length; - stats.tests = totalPassing + errorResults.length; - stats.duration = duration; - for (let j = 0; j < errorResults.length; j++) { - const failure = errorResults[j]; - failures.push(makeMochaTest(failure)); - } - if (noColors) { - const savedUseColors = Base.useColors; - Base.useColors = false; - reporter.epilogue(); - Base.useColors = savedUseColors; - } - else { - reporter.epilogue(); - } - - process.exit(errorResults.length); - } - - function makeMochaTest(test: ErrorInfo) { - return { - fullTitle: () => { - return test.name; - }, - err: { - message: test.error, - stack: test.stack - } - }; - } - - describe = ts.noop as any; // Disable unit tests - - return; -} - function startTestEnvironment() { const isWorker = handleTestConfig(); if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) { if (isWorker) { - return beginWorkListener(); + return Harness.Parallel.Worker.start(); } else if (taskConfigsFolder && workerCount && workerCount > 1) { - return beginTestHost(); + return Harness.Parallel.Host.start(); } } beginTests(); diff --git a/src/harness/tsconfig.json b/src/harness/tsconfig.json index bd7c9bc2ffab4..c6e7813863852 100644 --- a/src/harness/tsconfig.json +++ b/src/harness/tsconfig.json @@ -92,6 +92,9 @@ "loggedIO.ts", "rwcRunner.ts", "test262Runner.ts", + "./parallel/shared.ts", + "./parallel/host.ts", + "./parallel/worker.ts", "runner.ts", "../server/protocol.ts", "../server/session.ts",