diff --git a/Gulpfile.ts b/Gulpfile.ts index a3db20dfd8a3e..676d07ec570e3 100644 --- a/Gulpfile.ts +++ b/Gulpfile.ts @@ -31,8 +31,6 @@ import merge2 = require("merge2"); import * as os from "os"; import fold = require("travis-fold"); const gulp = helpMaker(originalGulp); -const mochaParallel = require("./scripts/mocha-parallel.js"); -const {runTestsInParallel} = mochaParallel; Error.stackTraceLimit = 1000; @@ -668,26 +666,9 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done: } else { // run task to load all tests and partition them between workers - const args = []; - args.push("-R", "min"); - if (colors) { - args.push("--colors"); - } - else { - args.push("--no-colors"); - } - args.push(run); setNodeEnvToDevelopment(); - runTestsInParallel(taskConfigsFolder, run, { testTimeout, noColors: colors === " --no-colors " }, function(err) { - // last worker clean everything and runs linter in case if there were no errors - del(taskConfigsFolder).then(() => { - if (!err) { - lintThenFinish(); - } - else { - finish(err); - } - }); + exec(host, [run], lintThenFinish, function(e, status) { + finish(e, status); }); } }); @@ -711,7 +692,7 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done: function finish(error?: any, errorStatus?: number) { restoreSavedNodeEnv(); - deleteTemporaryProjectOutput().then(() => { + deleteTestConfig().then(deleteTemporaryProjectOutput).then(() => { if (error !== undefined || errorStatus !== undefined) { failWithStatus(error, errorStatus); } @@ -720,6 +701,10 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done: } }); } + + function deleteTestConfig() { + return del("test.config"); + } } gulp.task("runtests-parallel", "Runs all the tests in parallel using the built run.js file. Optional arguments are: --t[ests]=category1|category2|... --d[ebug]=true.", ["build-rules", "tests"], (done) => { @@ -836,7 +821,7 @@ function cleanTestDirs(done: (e?: any) => void) { // used to pass data from jake command line directly to run.js function writeTestConfigFile(tests: string, light: boolean, taskConfigsFolder?: string, workerCount?: number, stackTraceLimit?: string) { - const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light, workerCount, stackTraceLimit, taskConfigsFolder }); + const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light, workerCount, stackTraceLimit, taskConfigsFolder, noColor: !cmdLineOptions["colors"] }); console.log("Running tests with config: " + testConfigContents); fs.writeFileSync("test.config", testConfigContents); } diff --git a/Jakefile.js b/Jakefile.js index ad853238111e4..39e9e8a0421d5 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -1,11 +1,11 @@ // This file contains the build logic for the public repo +// @ts-check var fs = require("fs"); var os = require("os"); var path = require("path"); var child_process = require("child_process"); var fold = require("travis-fold"); -var runTestsInParallel = require("./scripts/mocha-parallel").runTestsInParallel; var ts = require("./lib/typescript"); @@ -38,7 +38,7 @@ else if (process.env.PATH !== undefined) { function filesFromConfig(configPath) { var configText = fs.readFileSync(configPath).toString(); - var config = ts.parseConfigFileTextToJson(configPath, configText, /*stripComments*/ true); + var config = ts.parseConfigFileTextToJson(configPath, configText); if (config.error) { throw new Error(diagnosticsToString([config.error])); } @@ -104,6 +104,9 @@ var harnessCoreSources = [ "loggedIO.ts", "rwcRunner.ts", "test262Runner.ts", + "./parallel/shared.ts", + "./parallel/host.ts", + "./parallel/worker.ts", "runner.ts" ].map(function (f) { return path.join(harnessDirectory, f); @@ -596,7 +599,7 @@ file(typesMapOutputPath, function() { var content = fs.readFileSync(path.join(serverDirectory, 'typesMap.json')); // Validate that it's valid JSON try { - JSON.parse(content); + JSON.parse(content.toString()); } catch (e) { console.log("Parse error in typesMap.json: " + e); } @@ -740,7 +743,7 @@ desc("Builds the test infrastructure using the built compiler"); task("tests", ["local", run].concat(libraryTargets)); function exec(cmd, completeHandler, errorHandler) { - var ex = jake.createExec([cmd], { windowsVerbatimArguments: true }); + var ex = jake.createExec([cmd], { windowsVerbatimArguments: true, interactive: true }); // Add listeners for output and error ex.addListener("stdout", function (output) { process.stdout.write(output); @@ -783,13 +786,14 @@ function cleanTestDirs() { } // used to pass data from jake command line directly to run.js -function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit) { +function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit, colors) { var testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light: light, workerCount: workerCount, taskConfigsFolder: taskConfigsFolder, - stackTraceLimit: stackTraceLimit + stackTraceLimit: stackTraceLimit, + noColor: !colors }); fs.writeFileSync('test.config', testConfigContents); } @@ -831,7 +835,7 @@ function runConsoleTests(defaultReporter, runInParallel) { } if (tests || light || taskConfigsFolder) { - writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit); + writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit, colors); } if (tests && tests.toLocaleLowerCase() === "rwc") { @@ -894,19 +898,15 @@ function runConsoleTests(defaultReporter, runInParallel) { var savedNodeEnv = process.env.NODE_ENV; process.env.NODE_ENV = "development"; var startTime = mark(); - runTestsInParallel(taskConfigsFolder, run, { testTimeout: testTimeout, noColors: !colors }, function (err) { + exec(host + " " + run, function () { process.env.NODE_ENV = savedNodeEnv; measure(startTime); - // last worker clean everything and runs linter in case if there were no errors - deleteTemporaryProjectOutput(); - jake.rmRf(taskConfigsFolder); - if (err) { - fail(err); - } - else { - runLinter(); - complete(); - } + runLinter(); + finish(); + }, function (e, status) { + process.env.NODE_ENV = savedNodeEnv; + measure(startTime); + finish(status); }); } @@ -969,8 +969,8 @@ desc("Runs the tests using the built run.js file like 'jake runtests'. Syntax is task("runtests-browser", ["browserify", nodeServerOutFile], function () { cleanTestDirs(); host = "node"; - browser = process.env.browser || process.env.b || (os.platform() === "linux" ? "chrome" : "IE"); - tests = process.env.test || process.env.tests || process.env.t; + var browser = process.env.browser || process.env.b || (os.platform() === "linux" ? "chrome" : "IE"); + var tests = process.env.test || process.env.tests || process.env.t; var light = process.env.light || false; var testConfigFile = 'test.config'; if (fs.existsSync(testConfigFile)) { diff --git a/scripts/mocha-none-reporter.js b/scripts/mocha-none-reporter.js deleted file mode 100644 index 5787b0c042ef5..0000000000000 --- a/scripts/mocha-none-reporter.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Module dependencies. - */ - -var Base = require('mocha').reporters.Base; - -/** - * Expose `None`. - */ - -exports = module.exports = None; - -/** - * Initialize a new `None` test reporter. - * - * @api public - * @param {Runner} runner - */ -function None(runner) { - Base.call(this); -} - -/** - * Inherit from `Base.prototype`. - */ -None.prototype.__proto__ = Base.prototype; diff --git a/scripts/mocha-parallel.js b/scripts/mocha-parallel.js deleted file mode 100644 index 6a54c018e9add..0000000000000 --- a/scripts/mocha-parallel.js +++ /dev/null @@ -1,405 +0,0 @@ -var tty = require("tty") - , readline = require("readline") - , fs = require("fs") - , path = require("path") - , child_process = require("child_process") - , os = require("os") - , mocha = require("mocha") - , Base = mocha.reporters.Base - , color = Base.color - , cursor = Base.cursor - , ms = require("mocha/lib/ms"); - -var isatty = tty.isatty(1) && tty.isatty(2); -var tapRangePattern = /^(\d+)\.\.(\d+)(?:$|\r\n?|\n)/; -var tapTestPattern = /^(not\sok|ok)\s+(\d+)\s+(?:-\s+)?(.*)$/; -var tapCommentPattern = /^#(?: (tests|pass|fail) (\d+)$)?/; - -exports.runTestsInParallel = runTestsInParallel; -exports.ProgressBars = ProgressBars; - -function runTestsInParallel(taskConfigsFolder, run, options, cb) { - if (options === undefined) options = { }; - - return discoverTests(run, options, function (error) { - if (error) { - return cb(error); - } - - return runTests(taskConfigsFolder, run, options, cb); - }); -} - -function discoverTests(run, options, cb) { - console.log("Discovering tests..."); - - var cmd = "mocha -R " + require.resolve("./mocha-none-reporter.js") + " " + run; - var p = spawnProcess(cmd); - p.on("exit", function (status) { - if (status) { - cb(new Error("Process exited with code " + status)); - } - else { - cb(); - } - }); -} - -function runTests(taskConfigsFolder, run, options, cb) { - var configFiles = fs.readdirSync(taskConfigsFolder); - var numPartitions = configFiles.length; - if (numPartitions <= 0) { - cb(); - return; - } - - console.log("Running tests on " + numPartitions + " threads..."); - - var partitions = Array(numPartitions); - var progressBars = new ProgressBars(); - progressBars.enable(); - - var counter = numPartitions; - configFiles.forEach(runTestsInPartition); - - function runTestsInPartition(file, index) { - var partition = { - file: path.join(taskConfigsFolder, file), - tests: 0, - passed: 0, - failed: 0, - completed: 0, - current: undefined, - start: undefined, - end: undefined, - catastrophicError: "", - failures: [] - }; - partitions[index] = partition; - - // Set up the progress bar. - updateProgress(0); - - // Start the background process. - var cmd = "mocha -t " + (options.testTimeout || 20000) + " -R tap --no-colors " + run + " --config='" + partition.file + "'"; - var p = spawnProcess(cmd); - var rl = readline.createInterface({ - input: p.stdout, - terminal: false - }); - - var rlError = readline.createInterface({ - input: p.stderr, - terminal: false - }); - - rl.on("line", onmessage); - rlError.on("line", onErrorMessage); - p.on("exit", onexit) - - function onErrorMessage(line) { - partition.catastrophicError += line + os.EOL; - } - - function onmessage(line) { - if (partition.start === undefined) { - partition.start = Date.now(); - } - - var rangeMatch = tapRangePattern.exec(line); - if (rangeMatch) { - partition.tests = parseInt(rangeMatch[2]); - return; - } - - var testMatch = tapTestPattern.exec(line); - if (testMatch) { - var test = { - result: testMatch[1], - id: parseInt(testMatch[2]), - name: testMatch[3], - output: [] - }; - - partition.current = test; - partition.completed++; - - if (test.result === "ok") { - partition.passed++; - } - else { - partition.failed++; - partition.failures.push(test); - } - - var progress = partition.completed / partition.tests; - if (progress < 1) { - updateProgress(progress); - } - - return; - } - - var commentMatch = tapCommentPattern.exec(line); - if (commentMatch) { - switch (commentMatch[1]) { - case "tests": - partition.current = undefined; - partition.tests = parseInt(commentMatch[2]); - break; - - case "pass": - partition.passed = parseInt(commentMatch[2]); - break; - - case "fail": - partition.failed = parseInt(commentMatch[2]); - break; - } - - return; - } - - if (partition.current) { - partition.current.output.push(line); - } - } - - function onexit(code) { - if (partition.end === undefined) { - partition.end = Date.now(); - } - - partition.duration = partition.end - partition.start; - var isPartitionFail = partition.failed || code !== 0; - var summaryColor = isPartitionFail ? "fail" : "green"; - var summarySymbol = isPartitionFail ? Base.symbols.err : Base.symbols.ok; - - var summaryTests = (isPartitionFail ? partition.passed + "/" + partition.tests : partition.passed) + " passing"; - var summaryDuration = "(" + ms(partition.duration) + ")"; - var savedUseColors = Base.useColors; - Base.useColors = !options.noColors; - - var summary = color(summaryColor, summarySymbol + " " + summaryTests) + " " + color("light", summaryDuration); - Base.useColors = savedUseColors; - - updateProgress(1, summary); - - signal(); - } - - function updateProgress(percentComplete, title) { - var progressColor = "pending"; - if (partition.failed) { - progressColor = "fail"; - } - - progressBars.update( - index, - percentComplete, - progressColor, - title - ); - } - } - - function signal() { - counter--; - - if (counter <= 0) { - var reporter = new Base(), - stats = reporter.stats, - failures = reporter.failures; - - var duration = 0; - var catastrophicError = ""; - for (var i = 0; i < numPartitions; i++) { - var partition = partitions[i]; - stats.passes += partition.passed; - stats.failures += partition.failed; - stats.tests += partition.tests; - duration += partition.duration; - if (partition.catastrophicError !== "") { - // Partition is written out to a temporary file as a JSON object. - // Below is an example of how the partition JSON object looks like - // { - // "light":false, - // "tasks":[ - // { - // "runner":"compiler", - // "files":["tests/cases/compiler/es6ImportNamedImportParsingError.ts"] - // } - // ], - // "runUnitTests":false - // } - var jsonText = fs.readFileSync(partition.file); - var configObj = JSON.parse(jsonText); - if (configObj.tasks && configObj.tasks[0]) { - catastrophicError += "Error from one or more of these files: " + configObj.tasks[0].files + os.EOL; - catastrophicError += partition.catastrophicError; - catastrophicError += os.EOL; - } - } - for (var j = 0; j < partition.failures.length; j++) { - var failure = partition.failures[j]; - failures.push(makeMochaTest(failure)); - } - } - - stats.duration = duration; - progressBars.disable(); - - if (options.noColors) { - var savedUseColors = Base.useColors; - Base.useColors = false; - reporter.epilogue(); - Base.useColors = savedUseColors; - } - else { - reporter.epilogue(); - } - - if (catastrophicError !== "") { - return cb(new Error(catastrophicError)); - } - if (stats.failures) { - return cb(new Error("Test failures reported: " + stats.failures)); - } - else { - return cb(); - } - } - } - - function makeMochaTest(test) { - return { - fullTitle: function() { - return test.name; - }, - err: { - message: test.output[0], - stack: test.output.join(os.EOL) - } - }; - } -} - -var nodeModulesPathPrefix = path.resolve("./node_modules/.bin/") + path.delimiter; -if (process.env.path !== undefined) { - process.env.path = nodeModulesPathPrefix + process.env.path; -} else if (process.env.PATH !== undefined) { - process.env.PATH = nodeModulesPathPrefix + process.env.PATH; -} - -function spawnProcess(cmd, options) { - var shell = process.platform === "win32" ? "cmd" : "/bin/sh"; - var prefix = process.platform === "win32" ? "/c" : "-c"; - return child_process.spawn(shell, [prefix, cmd], { windowsVerbatimArguments: true }); -} - -function ProgressBars(options) { - if (!options) options = {}; - var open = options.open || '['; - var close = options.close || ']'; - var complete = options.complete || '▬'; - var incomplete = options.incomplete || Base.symbols.dot; - var maxWidth = Math.floor(Base.window.width * .30) - open.length - close.length - 2; - var width = minMax(options.width || maxWidth, 10, maxWidth); - this._options = { - open: open, - complete: complete, - incomplete: incomplete, - close: close, - width: width - }; - - this._progressBars = []; - this._lineCount = 0; - this._enabled = false; -} -ProgressBars.prototype = { - enable: function () { - if (!this._enabled) { - process.stdout.write(os.EOL); - this._enabled = true; - } - }, - disable: function () { - if (this._enabled) { - process.stdout.write(os.EOL); - this._enabled = false; - } - }, - update: function (index, percentComplete, color, title) { - percentComplete = minMax(percentComplete, 0, 1); - - var progressBar = this._progressBars[index] || (this._progressBars[index] = { }); - var width = this._options.width; - var n = Math.floor(width * percentComplete); - var i = width - n; - if (n === progressBar.lastN && title === progressBar.title && color === progressBar.progressColor) { - return; - } - - progressBar.lastN = n; - progressBar.title = title; - progressBar.progressColor = color; - - var progress = " "; - progress += this._color('progress', this._options.open); - progress += this._color(color, fill(this._options.complete, n)); - progress += this._color('progress', fill(this._options.incomplete, i)); - progress += this._color('progress', this._options.close); - - if (title) { - progress += this._color('progress', ' ' + title); - } - - if (progressBar.text !== progress) { - progressBar.text = progress; - this._render(index); - } - }, - _render: function (index) { - if (!this._enabled || !isatty) { - return; - } - - cursor.hide(); - readline.moveCursor(process.stdout, -process.stdout.columns, -this._lineCount); - var lineCount = 0; - var numProgressBars = this._progressBars.length; - for (var i = 0; i < numProgressBars; i++) { - if (i === index) { - readline.clearLine(process.stdout, 1); - process.stdout.write(this._progressBars[i].text + os.EOL); - } - else { - readline.moveCursor(process.stdout, -process.stdout.columns, +1); - } - - lineCount++; - } - - this._lineCount = lineCount; - cursor.show(); - }, - _color: function (type, text) { - return type && !this._options.noColors ? color(type, text) : text; - } -}; - -function fill(ch, size) { - var s = ""; - while (s.length < size) { - s += ch; - } - - return s.length > size ? s.substr(0, size) : s; -} - -function minMax(value, min, max) { - if (value < min) return min; - if (value > max) return max; - return value; -} \ No newline at end of file diff --git a/src/harness/parallel/host.ts b/src/harness/parallel/host.ts new file mode 100644 index 0000000000000..58e794bbc7d9e --- /dev/null +++ b/src/harness/parallel/host.ts @@ -0,0 +1,376 @@ +// tslint:disable-next-line +var describe: Mocha.IContextDefinition; // If launched without mocha for parallel mode, we still need a global describe visible to satisfy the parsing of the unit tests +// tslint:disable-next-line +var it: Mocha.ITestDefinition; +namespace Harness.Parallel.Host { + + interface ChildProcessPartial { + send(message: any, callback?: (error: Error) => void): boolean; + on(event: "error", listener: (err: Error) => void): this; + on(event: "exit", listener: (code: number, signal: string) => void): this; + on(event: "message", listener: (message: any) => void): this; + disconnect(): void; + } + + interface ProgressBarsOptions { + open: string; + close: string; + complete: string; + incomplete: string; + width: number; + noColors: boolean; + } + interface ProgressBar { + lastN?: number; + title?: string; + progressColor?: string; + text?: string; + } + + export function start() { + console.log("Discovering tests..."); + const discoverStart = +(new Date()); + const { statSync }: { statSync(path: string): { size: number }; } = require("fs"); + const tasks: { runner: TestRunnerKind, file: string, size: number }[] = []; + let totalSize = 0; + for (const runner of runners) { + const files = runner.enumerateTestFiles(); + for (const file of files) { + const size = statSync(file).size; + tasks.push({ runner: runner.kind(), file, size }); + totalSize += size; + } + } + tasks.sort((a, b) => a.size - b.size); + const batchSize = (totalSize / workerCount) * 0.9; + console.log(`Discovered ${tasks.length} test files in ${+(new Date()) - discoverStart}ms.`); + console.log(`Starting to run tests using ${workerCount} threads...`); + const { fork }: { fork(modulePath: string, args?: string[], options?: {}): ChildProcessPartial; } = require("child_process"); + + const totalFiles = tasks.length; + let passingFiles = 0; + let failingFiles = 0; + let errorResults: ErrorInfo[] = []; + let totalPassing = 0; + const startTime = Date.now(); + + const progressBars = new ProgressBars({ noColors }); + const progressUpdateInterval = 1 / progressBars._options.width; + let nextProgress = progressUpdateInterval; + + const workers: ChildProcessPartial[] = []; + for (let i = 0; i < workerCount; i++) { + // TODO: Just send the config over the IPC channel or in the command line arguments + const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length === 1 ? false : i === workerCount - 1 }; + const configPath = ts.combinePaths(taskConfigsFolder, `task-config${i}.json`); + Harness.IO.writeFile(configPath, JSON.stringify(config)); + const child = fork(__filename, [`--config="${configPath}"`]); + child.on("error", err => { + child.disconnect(); + console.error("Unexpected error in child process:"); + console.error(err); + return process.exit(2); + }); + child.on("exit", (code, _signal) => { + if (code !== 0) { + console.error("Test worker process exited with nonzero exit code!"); + return process.exit(2); + } + }); + child.on("message", (data: ParallelClientMessage) => { + switch (data.type) { + case "error": { + child.disconnect(); + console.error(`Test worker encounted unexpected error and was forced to close: + Message: ${data.payload.error} + Stack: ${data.payload.stack}`); + return process.exit(2); + } + case "progress": + case "result": { + totalPassing += data.payload.passing; + if (data.payload.errors.length) { + errorResults = errorResults.concat(data.payload.errors); + failingFiles++; + } + else { + passingFiles++; + } + + const progress = (failingFiles + passingFiles) / totalFiles; + if (progress >= nextProgress) { + while (nextProgress < progress) { + nextProgress += progressUpdateInterval; + } + updateProgress(progress, errorResults.length ? `${errorResults.length} failing` : `${totalPassing} passing`, errorResults.length ? "fail" : undefined); + } + + if (failingFiles + passingFiles === totalFiles) { + // Done. Finished every task and collected results. + child.send({ type: "close" }); + child.disconnect(); + return outputFinalResult(); + } + if (tasks.length === 0) { + // No more tasks to distribute + child.send({ type: "close" }); + child.disconnect(); + return; + } + if (data.type === "result") { + child.send({ type: "test", payload: tasks.pop() }); + } + } + } + }); + workers.push(child); + } + + // It's only really worth doing an initial batching if there are a ton of files to go through + if (totalFiles > 1000) { + console.log("Batching initial test lists..."); + const batches: { runner: TestRunnerKind, file: string, size: number }[][] = new Array(workerCount); + const doneBatching = new Array(workerCount); + batcher: while (true) { + for (let i = 0; i < workerCount; i++) { + if (tasks.length === 0) { + // TODO: This indicates a particularly suboptimal packing + break batcher; + } + if (doneBatching[i]) { + continue; + } + if (!batches[i]) { + batches[i] = []; + } + const total = batches[i].reduce((p, c) => p + c.size, 0); + if (total >= batchSize && !doneBatching[i]) { + doneBatching[i] = true; + continue; + } + batches[i].push(tasks.pop()); + } + for (let j = 0; j < workerCount; j++) { + if (!doneBatching[j]) { + continue; + } + } + break; + } + console.log(`Batched into ${workerCount} groups with approximate total file sizes of ${Math.floor(batchSize)} bytes in each group.`); + for (const worker of workers) { + const action: ParallelBatchMessage = { type: "batch", payload: batches.pop() }; + if (!action.payload[0]) { + throw new Error(`Tried to send invalid message ${action}`); + } + worker.send(action); + } + } + else { + for (let i = 0; i < workerCount; i++) { + workers[i].send({ type: "test", payload: tasks.pop() }); + } + } + + progressBars.enable(); + updateProgress(0); + let duration: number; + + const ms = require("mocha/lib/ms"); + function completeBar() { + const isPartitionFail = failingFiles !== 0; + const summaryColor = isPartitionFail ? "fail" : "green"; + const summarySymbol = isPartitionFail ? Base.symbols.err : Base.symbols.ok; + + const summaryTests = (isPartitionFail ? totalPassing + "/" + (errorResults.length + totalPassing) : totalPassing) + " passing"; + const summaryDuration = "(" + ms(duration) + ")"; + const savedUseColors = Base.useColors; + Base.useColors = !noColors; + + const summary = color(summaryColor, summarySymbol + " " + summaryTests) + " " + color("light", summaryDuration); + Base.useColors = savedUseColors; + + updateProgress(1, summary); + } + + function updateProgress(percentComplete: number, title?: string, titleColor?: string) { + let progressColor = "pending"; + if (failingFiles) { + progressColor = "fail"; + } + + progressBars.update( + 0, + percentComplete, + progressColor, + title, + titleColor + ); + } + + function outputFinalResult() { + duration = Date.now() - startTime; + completeBar(); + progressBars.disable(); + + const reporter = new Base(); + const stats = reporter.stats; + const failures = reporter.failures; + stats.passes = totalPassing; + stats.failures = errorResults.length; + stats.tests = totalPassing + errorResults.length; + stats.duration = duration; + for (let j = 0; j < errorResults.length; j++) { + const failure = errorResults[j]; + failures.push(makeMochaTest(failure)); + } + if (noColors) { + const savedUseColors = Base.useColors; + Base.useColors = false; + reporter.epilogue(); + Base.useColors = savedUseColors; + } + else { + reporter.epilogue(); + } + + process.exit(errorResults.length); + } + + function makeMochaTest(test: ErrorInfo) { + return { + fullTitle: () => { + return test.name; + }, + err: { + message: test.error, + stack: test.stack + } + }; + } + + describe = ts.noop as any; // Disable unit tests + + return; + } + + const Mocha = require("mocha"); + const Base = Mocha.reporters.Base; + const color = Base.color; + const cursor = Base.cursor; + const readline = require("readline"); + const os = require("os"); + const tty: { isatty(x: number): boolean } = require("tty"); + const isatty = tty.isatty(1) && tty.isatty(2); + class ProgressBars { + public readonly _options: Readonly; + private _enabled: boolean; + private _lineCount: number; + private _progressBars: ProgressBar[]; + constructor(options?: Partial) { + if (!options) options = {}; + const open = options.open || "["; + const close = options.close || "]"; + const complete = options.complete || "▬"; + const incomplete = options.incomplete || Base.symbols.dot; + const maxWidth = Base.window.width - open.length - close.length - 30; + const width = minMax(options.width || maxWidth, 10, maxWidth); + this._options = { + open, + complete, + incomplete, + close, + width, + noColors: options.noColors || false + }; + + this._progressBars = []; + this._lineCount = 0; + this._enabled = false; + } + enable() { + if (!this._enabled) { + process.stdout.write(os.EOL); + this._enabled = true; + } + } + disable() { + if (this._enabled) { + process.stdout.write(os.EOL); + this._enabled = false; + } + } + update(index: number, percentComplete: number, color: string, title: string, titleColor?: string) { + percentComplete = minMax(percentComplete, 0, 1); + + const progressBar = this._progressBars[index] || (this._progressBars[index] = { }); + const width = this._options.width; + const n = Math.floor(width * percentComplete); + const i = width - n; + if (n === progressBar.lastN && title === progressBar.title && color === progressBar.progressColor) { + return; + } + + progressBar.lastN = n; + progressBar.title = title; + progressBar.progressColor = color; + + let progress = " "; + progress += this._color("progress", this._options.open); + progress += this._color(color, fill(this._options.complete, n)); + progress += this._color("progress", fill(this._options.incomplete, i)); + progress += this._color("progress", this._options.close); + + if (title) { + progress += this._color(titleColor || "progress", " " + title); + } + + if (progressBar.text !== progress) { + progressBar.text = progress; + this._render(index); + } + } + private _render(index: number) { + if (!this._enabled || !isatty) { + return; + } + + cursor.hide(); + readline.moveCursor(process.stdout, -process.stdout.columns, -this._lineCount); + let lineCount = 0; + const numProgressBars = this._progressBars.length; + for (let i = 0; i < numProgressBars; i++) { + if (i === index) { + readline.clearLine(process.stdout, 1); + process.stdout.write(this._progressBars[i].text + os.EOL); + } + else { + readline.moveCursor(process.stdout, -process.stdout.columns, +1); + } + + lineCount++; + } + + this._lineCount = lineCount; + cursor.show(); + } + private _color(type: string, text: string) { + return type && !this._options.noColors ? color(type, text) : text; + } + } + + function fill(ch: string, size: number) { + let s = ""; + while (s.length < size) { + s += ch; + } + + return s.length > size ? s.substr(0, size) : s; + } + + function minMax(value: number, min: number, max: number) { + if (value < min) return min; + if (value > max) return max; + return value; + } +} \ No newline at end of file diff --git a/src/harness/parallel/shared.ts b/src/harness/parallel/shared.ts new file mode 100644 index 0000000000000..ebfe327884993 --- /dev/null +++ b/src/harness/parallel/shared.ts @@ -0,0 +1,14 @@ +/// +/// +namespace Harness.Parallel { + export type ParallelTestMessage = { type: "test", payload: { runner: TestRunnerKind, file: string } } | never; + export type ParallelBatchMessage = { type: "batch", payload: ParallelTestMessage["payload"][] } | never; + export type ParallelCloseMessage = { type: "close" } | never; + export type ParallelHostMessage = ParallelTestMessage | ParallelCloseMessage | ParallelBatchMessage; + + export type ParallelErrorMessage = { type: "error", payload: { error: string, stack: string } } | never; + export type ErrorInfo = ParallelErrorMessage["payload"] & { name: string }; + export type ParallelResultMessage = { type: "result", payload: { passing: number, errors: ErrorInfo[] } } | never; + export type ParallelBatchProgressMessage = { type: "progress", payload: ParallelResultMessage["payload"] } | never; + export type ParallelClientMessage = ParallelErrorMessage | ParallelResultMessage | ParallelBatchProgressMessage; +} \ No newline at end of file diff --git a/src/harness/parallel/worker.ts b/src/harness/parallel/worker.ts new file mode 100644 index 0000000000000..34f89d37f130e --- /dev/null +++ b/src/harness/parallel/worker.ts @@ -0,0 +1,123 @@ +namespace Harness.Parallel.Worker { + let errors: ErrorInfo[] = []; + let passing = 0; + function resetShimHarnessAndExecute(runner: RunnerBase) { + errors = []; + passing = 0; + runner.initializeTests(); + return { errors, passing }; + } + + function shimMochaHarness() { + (global as any).before = undefined; + (global as any).after = undefined; + (global as any).beforeEach = undefined; + let beforeEachFunc: Function; + describe = ((_name, callback) => { + const fakeContext: Mocha.ISuiteCallbackContext = { + retries() { return this; }, + slow() { return this; }, + timeout() { return this; }, + }; + (before as any) = (cb: Function) => cb(); + let afterFunc: Function; + (after as any) = (cb: Function) => afterFunc = cb; + const savedBeforeEach = beforeEachFunc; + (beforeEach as any) = (cb: Function) => beforeEachFunc = cb; + callback.call(fakeContext); + afterFunc && afterFunc(); + afterFunc = undefined; + beforeEachFunc = savedBeforeEach; + }) as Mocha.IContextDefinition; + it = ((name, callback) => { + const fakeContext: Mocha.ITestCallbackContext = { + skip() { return this; }, + timeout() { return this; }, + retries() { return this; }, + slow() { return this; }, + }; + // TODO: If we ever start using async test completions, polyfill the `done` parameter/promise return handling + if (beforeEachFunc) { + try { + beforeEachFunc(); + } + catch (error) { + errors.push({ error: error.message, stack: error.stack, name }); + return; + } + } + try { + callback.call(fakeContext); + } + catch (error) { + errors.push({ error: error.message, stack: error.stack, name }); + return; + } + passing++; + }) as Mocha.ITestDefinition; + } + + export function start() { + let initialized = false; + const runners = ts.createMap(); + process.on("message", (data: ParallelHostMessage) => { + if (!initialized) { + initialized = true; + shimMochaHarness(); + } + switch (data.type) { + case "test": + const { runner, file } = data.payload; + if (!runner) { + console.error(data); + } + const message: ParallelResultMessage = { type: "result", payload: handleTest(runner, file) }; + process.send(message); + break; + case "close": + process.exit(0); + break; + case "batch": { + const items = data.payload; + for (let i = 0; i < items.length; i++) { + const { runner, file } = items[i]; + if (!runner) { + console.error(data); + } + let message: ParallelBatchProgressMessage | ParallelResultMessage; + const payload = handleTest(runner, file); + if (i === (items.length - 1)) { + message = { type: "result", payload }; + } + else { + message = { type: "progress", payload }; + } + process.send(message); + } + break; + } + } + }); + process.on("uncaughtException", error => { + const message: ParallelErrorMessage = { type: "error", payload: { error: error.message, stack: error.stack } }; + process.send(message); + }); + if (!runUnitTests) { + // ensure unit tests do not get run + describe = ts.noop as any; + } + else { + initialized = true; + shimMochaHarness(); + } + + function handleTest(runner: TestRunnerKind, file: string) { + if (!runners.has(runner)) { + runners.set(runner, createRunner(runner)); + } + const instance = runners.get(runner); + instance.tests = [file]; + return resetShimHarnessAndExecute(instance); + } + } +} \ No newline at end of file diff --git a/src/harness/runner.ts b/src/harness/runner.ts index 6e1f91b21af29..0b361e7fc9eb1 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -19,6 +19,7 @@ /// /// /// +/// let runners: RunnerBase[] = []; let iterations = 1; @@ -59,6 +60,7 @@ function createRunner(kind: TestRunnerKind): RunnerBase { case "test262": return new Test262BaselineRunner(); } + ts.Debug.fail(`Unknown runner kind ${kind}`); } if (Harness.IO.tryEnableSourceMapsForHost && /^development$/i.test(Harness.IO.getEnvironmentVariable("NODE_ENV"))) { @@ -81,15 +83,17 @@ let testConfigContent = let taskConfigsFolder: string; let workerCount: number; let runUnitTests = true; +let noColors = false; interface TestConfig { light?: boolean; taskConfigsFolder?: string; + listenForWork?: boolean; workerCount?: number; stackTraceLimit?: number | "full"; - tasks?: TaskSet[]; test?: string[]; runUnitTests?: boolean; + noColors?: boolean; } interface TaskSet { @@ -97,138 +101,122 @@ interface TaskSet { files: string[]; } -if (testConfigContent !== "") { - const testConfig = JSON.parse(testConfigContent); - if (testConfig.light) { - Harness.lightMode = true; - } - if (testConfig.taskConfigsFolder) { - taskConfigsFolder = testConfig.taskConfigsFolder; - } - if (testConfig.runUnitTests !== undefined) { - runUnitTests = testConfig.runUnitTests; - } - if (testConfig.workerCount) { - workerCount = testConfig.workerCount; - } - if (testConfig.tasks) { - for (const taskSet of testConfig.tasks) { - const runner = createRunner(taskSet.runner); - for (const file of taskSet.files) { - runner.addTest(file); - } - runners.push(runner); +function handleTestConfig() { + if (testConfigContent !== "") { + const testConfig = JSON.parse(testConfigContent); + if (testConfig.light) { + Harness.lightMode = true; + } + if (testConfig.runUnitTests !== undefined) { + runUnitTests = testConfig.runUnitTests; + } + if (testConfig.workerCount) { + workerCount = +testConfig.workerCount; + } + if (testConfig.taskConfigsFolder) { + taskConfigsFolder = testConfig.taskConfigsFolder; + } + if (testConfig.noColors !== undefined) { + noColors = testConfig.noColors; } - } - - if (testConfig.stackTraceLimit === "full") { - (Error).stackTraceLimit = Infinity; - } - else if ((+testConfig.stackTraceLimit | 0) > 0) { - (Error).stackTraceLimit = testConfig.stackTraceLimit; - } - if (testConfig.test && testConfig.test.length > 0) { - for (const option of testConfig.test) { - if (!option) { - continue; - } + if (testConfig.stackTraceLimit === "full") { + (Error).stackTraceLimit = Infinity; + } + else if ((+testConfig.stackTraceLimit | 0) > 0) { + (Error).stackTraceLimit = testConfig.stackTraceLimit; + } + if (testConfig.listenForWork) { + return true; + } - switch (option) { - case "compiler": - runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); - runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); - runners.push(new ProjectRunner()); - break; - case "conformance": - runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); - break; - case "project": - runners.push(new ProjectRunner()); - break; - case "fourslash": - runners.push(new FourSlashRunner(FourSlashTestType.Native)); - break; - case "fourslash-shims": - runners.push(new FourSlashRunner(FourSlashTestType.Shims)); - break; - case "fourslash-shims-pp": - runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess)); - break; - case "fourslash-server": - runners.push(new FourSlashRunner(FourSlashTestType.Server)); - break; - case "fourslash-generated": - runners.push(new GeneratedFourslashRunner(FourSlashTestType.Native)); - break; - case "rwc": - runners.push(new RWCRunner()); - break; - case "test262": - runners.push(new Test262BaselineRunner()); - break; + if (testConfig.test && testConfig.test.length > 0) { + for (const option of testConfig.test) { + if (!option) { + continue; + } + + switch (option) { + case "compiler": + runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); + runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); + runners.push(new ProjectRunner()); + break; + case "conformance": + runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); + break; + case "project": + runners.push(new ProjectRunner()); + break; + case "fourslash": + runners.push(new FourSlashRunner(FourSlashTestType.Native)); + break; + case "fourslash-shims": + runners.push(new FourSlashRunner(FourSlashTestType.Shims)); + break; + case "fourslash-shims-pp": + runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess)); + break; + case "fourslash-server": + runners.push(new FourSlashRunner(FourSlashTestType.Server)); + break; + case "fourslash-generated": + runners.push(new GeneratedFourslashRunner(FourSlashTestType.Native)); + break; + case "rwc": + runners.push(new RWCRunner()); + break; + case "test262": + runners.push(new Test262BaselineRunner()); + break; + } } } } -} - -if (runners.length === 0) { - // compiler - runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); - runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); - - // TODO: project tests don't work in the browser yet - if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) { - runners.push(new ProjectRunner()); - } - // language services - runners.push(new FourSlashRunner(FourSlashTestType.Native)); - runners.push(new FourSlashRunner(FourSlashTestType.Shims)); - runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess)); - runners.push(new FourSlashRunner(FourSlashTestType.Server)); - // runners.push(new GeneratedFourslashRunner()); -} + if (runners.length === 0) { + // compiler + runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); + runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); -if (taskConfigsFolder) { - // this instance of mocha should only partition work but not run actual tests - runUnitTests = false; - const workerConfigs: TestConfig[] = []; - for (let i = 0; i < workerCount; i++) { - // pass light mode settings to workers - workerConfigs.push({ light: Harness.lightMode, tasks: [] }); - } - - for (const runner of runners) { - const files = runner.enumerateTestFiles(); - const chunkSize = Math.floor(files.length / workerCount) + 1; // add extra 1 to prevent missing tests due to rounding - for (let i = 0; i < workerCount; i++) { - const startPos = i * chunkSize; - const len = Math.min(chunkSize, files.length - startPos); - if (len > 0) { - workerConfigs[i].tasks.push({ - runner: runner.kind(), - files: files.slice(startPos, startPos + len) - }); - } + // TODO: project tests don"t work in the browser yet + if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) { + runners.push(new ProjectRunner()); } - } - for (let i = 0; i < workerCount; i++) { - const config = workerConfigs[i]; - // use last worker to run unit tests if we're not just running a single specific runner - config.runUnitTests = runners.length !== 1 && i === workerCount - 1; - Harness.IO.writeFile(ts.combinePaths(taskConfigsFolder, `task-config${i}.json`), JSON.stringify(workerConfigs[i])); + // language services + runners.push(new FourSlashRunner(FourSlashTestType.Native)); + runners.push(new FourSlashRunner(FourSlashTestType.Shims)); + runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess)); + runners.push(new FourSlashRunner(FourSlashTestType.Server)); + // runners.push(new GeneratedFourslashRunner()); } } -else { + +function beginTests() { if (ts.Debug.isDebugging) { ts.Debug.enableDebugInfo(); } runTests(runners); + + if (!runUnitTests) { + // patch `describe` to skip unit tests + describe = ts.noop as any; + } } -if (!runUnitTests) { - // patch `describe` to skip unit tests - describe = ts.noop as any; + +function startTestEnvironment() { + const isWorker = handleTestConfig(); + if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) { + if (isWorker) { + return Harness.Parallel.Worker.start(); + } + else if (taskConfigsFolder && workerCount && workerCount > 1) { + return Harness.Parallel.Host.start(); + } + } + beginTests(); } + +startTestEnvironment(); diff --git a/src/harness/tsconfig.json b/src/harness/tsconfig.json index bd7c9bc2ffab4..c6e7813863852 100644 --- a/src/harness/tsconfig.json +++ b/src/harness/tsconfig.json @@ -92,6 +92,9 @@ "loggedIO.ts", "rwcRunner.ts", "test262Runner.ts", + "./parallel/shared.ts", + "./parallel/host.ts", + "./parallel/worker.ts", "runner.ts", "../server/protocol.ts", "../server/session.ts",