diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts index 4b647a4787015..222adcf0fb772 100644 --- a/test-runner/src/cli.ts +++ b/test-runner/src/cli.ts @@ -18,7 +18,15 @@ import program from 'commander'; import * as fs from 'fs'; import * as path from 'path'; import { collectTests, runTests, RunnerConfig } from '.'; -import { reporters } from './reporters'; +import { DotReporter } from './reporters/dot'; +import { ListReporter } from './reporters/list'; +import { JSONReporter } from './reporters/json'; + +export const reporters = { + 'dot': DotReporter, + 'list': ListReporter, + 'json': JSONReporter +}; program .version('Version ' + /** @type {any} */ (require)('../package.json').version) @@ -66,8 +74,8 @@ program process.exit(1); } - const reporterFactory = reporters[command.reporter || 'dot']; - await runTests(config, suite, reporterFactory); + const reporter = new (reporters[command.reporter || 'dot'])(); + await runTests(config, suite, reporter); const hasFailures = suite.eachTest(t => t.error); process.exit(hasFailures ? 1 : 0); }); diff --git a/test-runner/src/index.ts b/test-runner/src/index.ts index 70661b0aaf8e1..2dcf62151306b 100644 --- a/test-runner/src/index.ts +++ b/test-runner/src/index.ts @@ -20,13 +20,14 @@ import * as path from 'path'; import './builtin.fixtures'; import './expect'; import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT } from './fixtures'; -import { reporters } from './reporters'; +import { Reporter } from './reporter'; import { Runner } from './runner'; import { RunnerConfig } from './runnerConfig'; import { Suite, Test } from './test'; import { Matrix, TestCollector } from './testCollector'; import { installTransform } from './transform'; export { parameters, registerParameter } from './fixtures'; +export { Reporter } from './reporter'; export { RunnerConfig } from './runnerConfig'; export { Suite, Test } from './test'; @@ -76,11 +77,10 @@ export function collectTests(config: RunnerConfig, files: string[]): Suite { return testCollector.suite; } -export async function runTests(config: RunnerConfig, suite: Suite, reporterFactory: any) { +export async function runTests(config: RunnerConfig, suite: Suite, reporter: Reporter) { // Trial run does not need many workers, use one. const jobs = (config.trialRun || config.debug) ? 1 : config.jobs; - const runner = new Runner(suite, { ...config, jobs }); - new reporterFactory(runner); + const runner = new Runner(suite, { ...config, jobs }, reporter); try { for (const f of beforeFunctions) diff --git a/test-runner/src/reporter.ts b/test-runner/src/reporter.ts new file mode 100644 index 0000000000000..559c0b5561187 --- /dev/null +++ b/test-runner/src/reporter.ts @@ -0,0 +1,27 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { RunnerConfig } from './runnerConfig'; +import { Suite, Test } from './test'; + +export interface Reporter { + onBegin(config: RunnerConfig, suite: Suite): void; + onTest(test: Test): void; + onPending(test: Test): void; + onPass(test: Test): void; + onFail(test: Test): void; + onEnd(): void; +} diff --git a/test-runner/src/reporters.ts b/test-runner/src/reporters.ts deleted file mode 100644 index a4d2a97536b04..0000000000000 --- a/test-runner/src/reporters.ts +++ /dev/null @@ -1,234 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import colors from 'colors/safe'; -import milliseconds from 'ms'; -import { codeFrameColumns } from '@babel/code-frame'; -import path from 'path'; -import fs from 'fs'; -import os from 'os'; -import terminalLink from 'terminal-link'; -import StackUtils from 'stack-utils'; -import { Test, Suite } from './test'; -import { EventEmitter } from 'ws'; -import { RunnerConfig } from './runnerConfig'; - -const stackUtils = new StackUtils(); - -class BaseReporter { - pending: Test[] = []; - passes: Test[] = []; - failures: Test[] = []; - duration = 0; - startTime: number; - config: RunnerConfig; - suite: Suite; - - constructor(runner: EventEmitter) { - process.on('SIGINT', async () => { - this.epilogue(); - process.exit(130); - }); - - runner.on('pending', (test: Test) => { - this.pending.push(test); - }); - - runner.on('pass', (test: Test) => { - this.passes.push(test); - }); - - runner.on('fail', (test: Test) => { - this.failures.push(test); - }); - - runner.once('begin', (options: { config: RunnerConfig, suite: Suite }) => { - this.startTime = Date.now(); - this.config = options.config; - this.suite = options.suite; - }); - - runner.once('end', () => { - this.duration = Date.now() - this.startTime; - }); - } - - epilogue() { - console.log(''); - - console.log(colors.green(` ${this.passes.length} passing`) + colors.dim(` (${milliseconds(this.duration)})`)); - - if (this.pending.length) - console.log(colors.yellow(` ${this.pending.length} skipped`)); - - if (this.failures.length) { - console.log(colors.red(` ${this.failures.length} failing`)); - console.log(''); - this.failures.forEach((failure, index) => { - const relativePath = path.relative(process.cwd(), failure.file); - const header = ` ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`; - console.log(colors.bold(colors.red(header))); - const stack = failure.error.stack; - if (stack) { - console.log(''); - const messageLocation = failure.error.stack.indexOf(failure.error.message); - const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length); - console.log(indent(preamble, ' ')); - const position = positionInFile(stack, failure.file); - if (position) { - const source = fs.readFileSync(failure.file, 'utf8'); - console.log(''); - console.log(indent(codeFrameColumns(source, { - start: position, - }, - { highlightCode: true} - ), ' ')); - } - console.log(''); - console.log(indent(colors.dim(stack.substring(preamble.length + 1)), ' ')); - } else { - console.log(''); - console.log(indent(String(failure.error), ' ')); - } - console.log(''); - }); - } - } -} - -export class DotReporter extends BaseReporter { - constructor(runner: EventEmitter) { - super(runner); - - runner.on('pending', () => { - process.stdout.write(colors.yellow('∘')) - }); - - runner.on('pass', () => { - process.stdout.write(colors.green('\u00B7')); - }); - - runner.on('fail', (test: Test) => { - if (test.duration >= test.timeout) - process.stdout.write(colors.red('T')); - else - process.stdout.write(colors.red('F')); - }); - - runner.once('end', () => { - process.stdout.write('\n'); - this.epilogue(); - }); - } -} - -export class ListReporter extends BaseReporter { - constructor(runner: EventEmitter) { - super(runner); - - runner.on('begin', () => { - console.log(); - }); - - runner.on('test', test => { - process.stdout.write(' ' + colors.gray(test.fullTitle() + ': ')); - }); - - runner.on('pending', test => { - process.stdout.write(colors.green(' - ') + colors.cyan(test.fullTitle())); - process.stdout.write('\n'); - }); - - runner.on('pass', test => { - process.stdout.write('\u001b[2K\u001b[0G'); - process.stdout.write(colors.green(' ✓ ') + colors.gray(test.fullTitle())); - process.stdout.write('\n'); - }); - - let failure = 0; - runner.on('fail', (test: Test) => { - process.stdout.write('\u001b[2K\u001b[0G'); - process.stdout.write(colors.red(` ${++failure}) ` + test.fullTitle())); - process.stdout.write('\n'); - }); - - runner.once('end', () => { - process.stdout.write('\n'); - this.epilogue(); - }); - } -} - -export class JSONReporter extends BaseReporter { - constructor(runner: EventEmitter) { - super(runner); - - runner.once('end', () => { - const result = { - config: this.config, - suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) - }; - console.log(JSON.stringify(result, undefined, 2)); - }); - } - - private _serializeSuite(suite: Suite): any { - if (!suite.eachTest(test => true)) - return null; - const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s); - return { - title: suite.title, - file: suite.file, - configuration: suite.configuration, - tests: suite.tests.map(test => this._serializeTest(test)), - suites: suites.length ? suites : undefined - }; - } - - private _serializeTest(test: Test): any { - return { - title: test.title, - file: test.file, - only: test.only, - pending: test.pending, - slow: test.slow, - duration: test.duration, - timeout: test.timeout, - error: test.error - }; - } -} - -function indent(lines: string, tab: string) { - return lines.replace(/^/gm, tab); -} - -function positionInFile(stack: string, file: string): { column: number; line: number; } { - for (const line of stack.split('\n')) { - const parsed = stackUtils.parseLine(line); - if (!parsed) - continue; - if (path.resolve(process.cwd(), parsed.file) === file) - return {column: parsed.column, line: parsed.line}; - } - return null; -} - -export const reporters = { - 'dot': DotReporter, - 'list': ListReporter, - 'json': JSONReporter -}; diff --git a/test-runner/src/reporters/base.ts b/test-runner/src/reporters/base.ts new file mode 100644 index 0000000000000..49452ea650e96 --- /dev/null +++ b/test-runner/src/reporters/base.ts @@ -0,0 +1,128 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { codeFrameColumns } from '@babel/code-frame'; +import colors from 'colors/safe'; +import fs from 'fs'; +import milliseconds from 'ms'; +import os from 'os'; +import path from 'path'; +import StackUtils from 'stack-utils'; +import terminalLink from 'terminal-link'; +import { Reporter } from '../reporter'; +import { RunnerConfig } from '../runnerConfig'; +import { Suite, Test } from '../test'; + +const stackUtils = new StackUtils() + +export class BaseReporter implements Reporter { + pending: Test[] = []; + passes: Test[] = []; + failures: Test[] = []; + duration = 0; + startTime: number; + config: RunnerConfig; + suite: Suite; + + constructor() { + process.on('SIGINT', async () => { + this.epilogue(); + process.exit(130); + }); + } + + onBegin(config: RunnerConfig, suite: Suite) { + this.startTime = Date.now(); + this.config = config; + this.suite = suite; + } + + onTest(test: Test) { + } + + onPending(test: Test) { + this.pending.push(test); + } + + onPass(test: Test) { + this.passes.push(test); + } + + onFail(test: Test) { + this.failures.push(test); + } + + onEnd() { + this.duration = Date.now() - this.startTime; + } + + epilogue() { + console.log(''); + + console.log(colors.green(` ${this.passes.length} passing`) + colors.dim(` (${milliseconds(this.duration)})`)); + + if (this.pending.length) + console.log(colors.yellow(` ${this.pending.length} skipped`)); + + if (this.failures.length) { + console.log(colors.red(` ${this.failures.length} failing`)); + console.log(''); + this.failures.forEach((failure, index) => { + const relativePath = path.relative(process.cwd(), failure.file); + const header = ` ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`; + console.log(colors.bold(colors.red(header))); + const stack = failure.error.stack; + if (stack) { + console.log(''); + const messageLocation = failure.error.stack.indexOf(failure.error.message); + const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length); + console.log(indent(preamble, ' ')); + const position = positionInFile(stack, failure.file); + if (position) { + const source = fs.readFileSync(failure.file, 'utf8'); + console.log(''); + console.log(indent(codeFrameColumns(source, { + start: position, + }, + { highlightCode: true} + ), ' ')); + } + console.log(''); + console.log(indent(colors.dim(stack.substring(preamble.length + 1)), ' ')); + } else { + console.log(''); + console.log(indent(String(failure.error), ' ')); + } + console.log(''); + }); + } + } +} + +function indent(lines: string, tab: string) { + return lines.replace(/^/gm, tab); +} + +function positionInFile(stack: string, file: string): { column: number; line: number; } { + for (const line of stack.split('\n')) { + const parsed = stackUtils.parseLine(line); + if (!parsed) + continue; + if (path.resolve(process.cwd(), parsed.file) === file) + return {column: parsed.column, line: parsed.line}; + } + return null; +} diff --git a/test-runner/src/reporters/dot.ts b/test-runner/src/reporters/dot.ts new file mode 100644 index 0000000000000..6ad20a7f140bc --- /dev/null +++ b/test-runner/src/reporters/dot.ts @@ -0,0 +1,45 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import colors from 'colors/safe'; +import { BaseReporter } from './base'; +import { Test } from '../test'; + +export class DotReporter extends BaseReporter { + onPending(test: Test) { + super.onPending(test); + process.stdout.write(colors.yellow('∘')) + } + + onPass(test: Test) { + super.onPass(test); + process.stdout.write(colors.green('\u00B7')); + } + + onFail(test: Test) { + super.onFail(test); + if (test.duration >= test.timeout) + process.stdout.write(colors.red('T')); + else + process.stdout.write(colors.red('F')); + } + + onEnd() { + super.onEnd(); + process.stdout.write('\n'); + this.epilogue(); + } +} diff --git a/test-runner/src/reporters/json.ts b/test-runner/src/reporters/json.ts new file mode 100644 index 0000000000000..403cdff01fb50 --- /dev/null +++ b/test-runner/src/reporters/json.ts @@ -0,0 +1,55 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BaseReporter } from './base'; +import { Suite, Test } from '../test'; + +export class JSONReporter extends BaseReporter { + onEnd() { + super.onEnd(); + const result = { + config: this.config, + suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) + }; + console.log(JSON.stringify(result, undefined, 2)); + } + + private _serializeSuite(suite: Suite): any { + if (!suite.eachTest(test => true)) + return null; + const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s); + return { + title: suite.title, + file: suite.file, + configuration: suite.configuration, + tests: suite.tests.map(test => this._serializeTest(test)), + suites: suites.length ? suites : undefined + }; + } + + private _serializeTest(test: Test): any { + return { + title: test.title, + file: test.file, + only: test.only, + pending: test.pending, + slow: test.slow, + duration: test.duration, + timeout: test.timeout, + error: test.error + }; + } +} diff --git a/test-runner/src/reporters/list.ts b/test-runner/src/reporters/list.ts new file mode 100644 index 0000000000000..c218d3955151b --- /dev/null +++ b/test-runner/src/reporters/list.ts @@ -0,0 +1,60 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import colors from 'colors/safe'; +import { BaseReporter } from './base'; +import { RunnerConfig } from '../runnerConfig'; +import { Suite, Test } from '../test'; + +export class ListReporter extends BaseReporter { + _failure = 0; + + onBegin(config: RunnerConfig, suite: Suite) { + super.onBegin(config, suite); + console.log(); + } + + onTest(test: Test) { + super.onTest(test); + process.stdout.write(' ' + colors.gray(test.fullTitle() + ': ')); + } + + onPending(test: Test) { + super.onPending(test); + process.stdout.write(colors.green(' - ') + colors.cyan(test.fullTitle())); + process.stdout.write('\n'); +} + + onPass(test: Test) { + super.onPass(test); + process.stdout.write('\u001b[2K\u001b[0G'); + process.stdout.write(colors.green(' ✓ ') + colors.gray(test.fullTitle())); + process.stdout.write('\n'); +} + + onFail(test: Test) { + super.onFail(test); + process.stdout.write('\u001b[2K\u001b[0G'); + process.stdout.write(colors.red(` ${++this._failure}) ` + test.fullTitle())); + process.stdout.write('\n'); +} + + onEnd() { + super.onEnd(); + process.stdout.write('\n'); + this.epilogue(); + } +} diff --git a/test-runner/src/runner.ts b/test-runner/src/runner.ts index 8cc0662bec4f2..4a1e1720b8862 100644 --- a/test-runner/src/runner.ts +++ b/test-runner/src/runner.ts @@ -19,11 +19,12 @@ import crypto from 'crypto'; import path from 'path'; import { EventEmitter } from 'events'; import { lookupRegistrations, FixturePool } from './fixtures'; -import { Suite, Test, Configuration } from './test'; +import { Suite, Test } from './test'; import { TestRunnerEntry } from './testRunner'; import { RunnerConfig } from './runnerConfig'; +import { Reporter } from './reporter'; -export class Runner extends EventEmitter { +export class Runner { private _workers = new Set(); private _freeWorkers: Worker[] = []; private _workerClaimers: (() => void)[] = []; @@ -34,11 +35,11 @@ export class Runner extends EventEmitter { private _stopCallback: () => void; readonly _config: RunnerConfig; private _suite: Suite; + private _reporter: Reporter; - constructor(suite: Suite, config: RunnerConfig) { - super(); - + constructor(suite: Suite, config: RunnerConfig, reporter: Reporter) { this._config = config; + this._reporter = reporter; this.stats = { duration: 0, failures: 0, @@ -80,12 +81,12 @@ export class Runner extends EventEmitter { } async run() { - this.emit('begin', { config: this._config, suite: this._suite }); + this._reporter.onBegin(this._config, this._suite); this._queue = this._filesSortedByWorkerHash(); // Loop in case job schedules more jobs while (this._queue.length) await this._dispatchQueue(); - this.emit('end', {}); + this._reporter.onEnd(); } async _dispatchQueue() { @@ -146,16 +147,16 @@ export class Runner extends EventEmitter { const worker = this._config.debug ? new InProcessWorker(this) : new OopWorker(this); worker.on('test', params => { ++this.stats.tests; - this.emit('test', this._updateTest(params.test)); + this._reporter.onTest(this._updateTest(params.test)); }); worker.on('pending', params => { ++this.stats.tests; ++this.stats.pending; - this.emit('pending', this._updateTest(params.test)); + this._reporter.onPending(this._updateTest(params.test)); }); worker.on('pass', params => { ++this.stats.passes; - this.emit('pass', this._updateTest(params.test)); + this._reporter.onPass(this._updateTest(params.test)); }); worker.on('fail', params => { ++this.stats.failures; @@ -165,7 +166,7 @@ export class Runner extends EventEmitter { const err = worker.takeErr(); if (err.length) params.test.error.stack += '\n\x1b[33mstderr: ' + err.join('\n') + '\x1b[0m'; - this.emit('fail', this._updateTest(params.test)); + this._reporter.onFail(this._updateTest(params.test)); }); worker.on('exit', () => { this._workers.delete(worker); diff --git a/test-runner/src/fixturesUI.ts b/test-runner/src/spec.ts similarity index 97% rename from test-runner/src/fixturesUI.ts rename to test-runner/src/spec.ts index 3476aab53078c..2e27fca9b0453 100644 --- a/test-runner/src/fixturesUI.ts +++ b/test-runner/src/spec.ts @@ -49,7 +49,7 @@ function specBuilder(modifiers, specCallback) { return builder({}, null); } -export function fixturesUI(suite: Suite, file: string, timeout: number): () => void { +export function spec(suite: Suite, file: string, timeout: number): () => void { const suites = [suite]; suite.file = file; diff --git a/test-runner/src/testCollector.ts b/test-runner/src/testCollector.ts index 14775393f2de9..b9b6d1f1ab8e9 100644 --- a/test-runner/src/testCollector.ts +++ b/test-runner/src/testCollector.ts @@ -17,7 +17,7 @@ import path from 'path'; import { fixturesForCallback } from './fixtures'; import { Test, Suite } from './test'; -import { fixturesUI } from './fixturesUI'; +import { spec } from './spec'; import { RunnerConfig } from './runnerConfig'; export type Matrix = { @@ -53,7 +53,7 @@ export class TestCollector { private _addFile(file: string) { const suite = new Suite(''); - const revertBabelRequire = fixturesUI(suite, file, this._config.timeout); + const revertBabelRequire = spec(suite, file, this._config.timeout); require(file); revertBabelRequire(); suite._renumber(); diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts index 210b858ea56dc..762a003b41436 100644 --- a/test-runner/src/testRunner.ts +++ b/test-runner/src/testRunner.ts @@ -18,7 +18,7 @@ import { FixturePool, rerunRegistrations, setParameters } from './fixtures'; import { EventEmitter } from 'events'; import { setCurrentTestFile } from './expect'; import { Test, Suite, Configuration } from './test'; -import { fixturesUI } from './fixturesUI'; +import { spec } from './spec'; import { RunnerConfig } from './runnerConfig'; export const fixturePool = new FixturePool(); @@ -67,7 +67,7 @@ export class TestRunner extends EventEmitter { setParameters(this._parsedGeneratorConfiguration); const suite = new Suite(''); - const revertBabelRequire = fixturesUI(suite, this._file, this._timeout); + const revertBabelRequire = spec(suite, this._file, this._timeout); require(this._file); revertBabelRequire(); suite._renumber();