From 51bd3709ff2eecfee970d74abfefd562cac8e348 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Wed, 12 Aug 2020 22:01:37 -0700 Subject: [PATCH] Revert "chore(test): run doclint tests with mocha, delete utils/testrunner (#3428)" (#3432) --- package.json | 3 +- test/runner/worker.js | 2 +- utils/doclint/check_public_api/test/test.js | 96 +- utils/doclint/preprocessor/test.js | 11 +- utils/testrunner/.npmignore | 13 + .../testrunner}/GoldenUtils.js | 0 utils/testrunner/Location.js | 88 ++ utils/testrunner/Matchers.js | 247 ++++ utils/testrunner/README.md | 55 + utils/testrunner/Reporter.js | 271 +++++ utils/testrunner/SourceMap.js | 460 +++++++ utils/testrunner/SourceMapSupport.js | 84 ++ utils/testrunner/Test.js | 179 +++ utils/testrunner/TestCollector.js | 259 ++++ utils/testrunner/TestRunner.js | 581 +++++++++ utils/testrunner/diffstyle.css | 13 + utils/testrunner/examples/fail.js | 54 + utils/testrunner/examples/hookfail.js | 35 + utils/testrunner/examples/hooktimeout.js | 35 + utils/testrunner/examples/timeout.js | 32 + .../examples/unhandledpromiserejection.js | 35 + utils/testrunner/index.js | 168 +++ utils/testrunner/test/test.js | 4 + utils/testrunner/test/testrunner.spec.js | 1052 +++++++++++++++++ 24 files changed, 3732 insertions(+), 45 deletions(-) create mode 100644 utils/testrunner/.npmignore rename {test/runner => utils/testrunner}/GoldenUtils.js (100%) create mode 100644 utils/testrunner/Location.js create mode 100644 utils/testrunner/Matchers.js create mode 100644 utils/testrunner/README.md create mode 100644 utils/testrunner/Reporter.js create mode 100644 utils/testrunner/SourceMap.js create mode 100644 utils/testrunner/SourceMapSupport.js create mode 100644 utils/testrunner/Test.js create mode 100644 utils/testrunner/TestCollector.js create mode 100644 utils/testrunner/TestRunner.js create mode 100644 utils/testrunner/diffstyle.css create mode 100644 utils/testrunner/examples/fail.js create mode 100644 utils/testrunner/examples/hookfail.js create mode 100644 utils/testrunner/examples/hooktimeout.js create mode 100644 utils/testrunner/examples/timeout.js create mode 100644 utils/testrunner/examples/unhandledpromiserejection.js create mode 100644 utils/testrunner/index.js create mode 100644 utils/testrunner/test/test.js create mode 100644 utils/testrunner/test/testrunner.spec.js diff --git a/package.json b/package.json index d1fa6a712f16c..76850f280a113 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,9 @@ "tsc-installer": "tsc -p ./src/install/tsconfig.json", "doc": "node utils/doclint/cli.js", "doc-no-channel": "node utils/doclint/cli.js --no-channel", - "test-infra": "node test/runner utils/doclint/check_public_api/test/test.js && node test/runner utils/doclint/preprocessor/test.js", + "test-infra": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js", "lint": "npm run eslint && npm run tsc && npm run doc && npm run doc-no-channel && npm run check-deps && npm run generate-channels && npm run test-types && npm run test-infra", + "debug-test": "node --inspect-brk test/test.js", "clean": "rimraf lib && rimraf types", "prepare": "node install-from-github.js", "build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run generate-types", diff --git a/test/runner/worker.js b/test/runner/worker.js index e2eea1e771c8a..848543e2e9828 100644 --- a/test/runner/worker.js +++ b/test/runner/worker.js @@ -18,7 +18,7 @@ const path = require('path'); const Mocha = require('mocha'); const { fixturesUI, fixturePool } = require('./fixturesUI'); const { gracefullyCloseAll } = require('../../lib/server/processLauncher'); -const GoldenUtils = require('./GoldenUtils'); +const GoldenUtils = require('../../utils/testrunner/GoldenUtils'); const browserName = process.env.BROWSER || 'chromium'; const goldenPath = path.join(__dirname, '..', 'golden-' + browserName); diff --git a/utils/doclint/check_public_api/test/test.js b/utils/doclint/check_public_api/test/test.js index 09af9380066bb..ee17e5f9f672a 100644 --- a/utils/doclint/check_public_api/test/test.js +++ b/utils/doclint/check_public_api/test/test.js @@ -21,58 +21,70 @@ const Source = require('../../Source'); const mdBuilder = require('../MDBuilder'); const jsBuilder = require('../JSBuilder'); -registerWorkerFixture('page', async({}, test) => { - const browser = await playwright.chromium.launch(); - const page = await browser.newPage(); - await test(page); +const TestRunner = require('../../../testrunner/'); +const runner = new TestRunner({ + goldenPath: __dirname, + outputPath: __dirname +}); + +const {describe, xdescribe, fdescribe} = runner.api(); +const {it, fit, xit} = runner.api(); +const {beforeAll, beforeEach, afterAll, afterEach} = runner.api(); +const {expect} = runner.api(); + +let browser; +let page; + +beforeAll(async function() { + browser = await playwright.chromium.launch(); + page = await browser.newPage(); +}); + +afterAll(async function() { await browser.close(); }); describe('checkPublicAPI', function() { - testLint('diff-classes'); - testLint('diff-methods'); - testLint('diff-properties'); - testLint('diff-arguments'); - testLint('diff-events'); - testLint('check-duplicates'); - testLint('check-sorting'); - testLint('check-returns'); - testLint('check-nullish'); - testJSBuilder('js-builder-common'); - testJSBuilder('js-builder-inheritance'); - testMDBuilder('md-builder-common'); - testMDBuilder('md-builder-comments'); + it('diff-classes', testLint); + it('diff-methods', testLint); + it('diff-properties', testLint); + it('diff-arguments', testLint); + it('diff-events', testLint); + it('check-duplicates', testLint); + it('check-sorting', testLint); + it('check-returns', testLint); + it('check-nullish', testLint); + it('js-builder-common', testJSBuilder); + it('js-builder-inheritance', testJSBuilder); + it('md-builder-common', testMDBuilder); + it('md-builder-comments', testMDBuilder); }); -async function testLint(name) { - it(name, async({page}) => { - const dirPath = path.join(__dirname, name); - const mdSources = await Source.readdir(dirPath, '.md'); - const tsSources = await Source.readdir(dirPath, '.ts'); - const jsSources = await Source.readdir(dirPath, '.js'); - const messages = await checkPublicAPI(page, mdSources, jsSources.concat(tsSources)); - const errors = messages.map(message => message.text); - expect(errors.join('\n')).toBeGolden(path.join(dirPath, 'result.txt')); - }); +runner.run(); + +async function testLint(state, testRun) { + const dirPath = path.join(__dirname, testRun.test().name()); + const mdSources = await Source.readdir(dirPath, '.md'); + const tsSources = await Source.readdir(dirPath, '.ts'); + const jsSources = await Source.readdir(dirPath, '.js'); + const messages = await checkPublicAPI(page, mdSources, jsSources.concat(tsSources)); + const errors = messages.map(message => message.text); + expect(errors.join('\n')).toBeGolden(path.join(testRun.test().name(), 'result.txt')); } -async function testMDBuilder(name) { - it(name, async({page}) => { - const dirPath = path.join(__dirname, name); - const sources = await Source.readdir(dirPath, '.md'); - const {documentation} = await mdBuilder(page, sources); - expect(serialize(documentation)).toBeGolden(path.join(dirPath, 'result.txt')); - }); +async function testMDBuilder(state, testRun) { + const dirPath = path.join(__dirname, testRun.test().name()); + const sources = await Source.readdir(dirPath, '.md'); + const {documentation} = await mdBuilder(page, sources); + expect(serialize(documentation)).toBeGolden(path.join(testRun.test().name(), 'result.txt')); } -async function testJSBuilder(name) { - it(name, async() => { - const dirPath = path.join(__dirname, name); - const jsSources = await Source.readdir(dirPath, '.js'); - const tsSources = await Source.readdir(dirPath, '.ts'); - const {documentation} = await jsBuilder.checkSources(jsSources.concat(tsSources)); - expect(serialize(documentation)).toBeGolden(path.join(dirPath, 'result.txt')); - }); +async function testJSBuilder(state, testRun) { + const dirPath = path.join(__dirname, testRun.test().name()); + const jsSources = await Source.readdir(dirPath, '.js'); + const tsSources = await Source.readdir(dirPath, '.ts'); + const {documentation} = await jsBuilder.checkSources(jsSources.concat(tsSources)); + expect(serialize(documentation)).toBeGolden(path.join(testRun.test().name(), 'result.txt')); } /** diff --git a/utils/doclint/preprocessor/test.js b/utils/doclint/preprocessor/test.js index 01d8c0cb4ac31..94a8645c9249a 100644 --- a/utils/doclint/preprocessor/test.js +++ b/utils/doclint/preprocessor/test.js @@ -14,8 +14,15 @@ * limitations under the License. */ -const {runCommands} = require('.'); +const {runCommands, ensureTipOfTreeAPILinks} = require('.'); const Source = require('../Source'); +const TestRunner = require('../../testrunner/'); +const runner = new TestRunner(); + +const {describe, xdescribe, fdescribe} = runner.api(); +const {it, fit, xit} = runner.api(); +const {beforeAll, beforeEach, afterAll, afterEach} = runner.api(); +const {expect} = runner.api(); describe('runCommands', function() { const OPTIONS_REL = { @@ -195,3 +202,5 @@ describe('runCommands', function() { }); }); +runner.run(); + diff --git a/utils/testrunner/.npmignore b/utils/testrunner/.npmignore new file mode 100644 index 0000000000000..7b4e15d4a0008 --- /dev/null +++ b/utils/testrunner/.npmignore @@ -0,0 +1,13 @@ +# exclude all examples and README.md +examples/ +README.md + +# repeats from .gitignore +node_modules +.npmignore +.DS_Store +*.swp +*.pyc +.vscode +package-lock.json +yarn.lock diff --git a/test/runner/GoldenUtils.js b/utils/testrunner/GoldenUtils.js similarity index 100% rename from test/runner/GoldenUtils.js rename to utils/testrunner/GoldenUtils.js diff --git a/utils/testrunner/Location.js b/utils/testrunner/Location.js new file mode 100644 index 0000000000000..2a6eeacd3d1ad --- /dev/null +++ b/utils/testrunner/Location.js @@ -0,0 +1,88 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const path = require('path'); + +// Hack for our own tests. +const testRunnerTestFile = path.join(__dirname, 'test', 'testrunner.spec.js'); + +class Location { + constructor() { + this._fileName = ''; + this._filePath = ''; + this._lineNumber = 0; + this._columnNumber = 0; + } + + fileName() { + return this._fileName; + } + + filePath() { + return this._filePath; + } + + lineNumber() { + return this._lineNumber; + } + + columnNumber() { + return this._columnNumber; + } + + toString() { + return this._fileName + ':' + this._lineNumber; + } + + toDetailedString() { + return this._fileName + ':' + this._lineNumber + ':' + this._columnNumber; + } + + static getCallerLocation(ignorePrefix = __dirname) { + const error = new Error(); + const stackFrames = error.stack.split('\n').slice(1); + const location = new Location(); + // Find first stackframe that doesn't point to this file. + for (let frame of stackFrames) { + frame = frame.trim(); + if (!frame.startsWith('at ')) + return null; + if (frame.endsWith(')')) { + const from = frame.indexOf('('); + frame = frame.substring(from + 1, frame.length - 1); + } else { + frame = frame.substring('at '.length); + } + + const match = frame.match(/^(.*):(\d+):(\d+)$/); + if (!match) + return null; + const filePath = match[1]; + if (filePath === __filename || (filePath.startsWith(ignorePrefix) && filePath !== testRunnerTestFile)) + continue; + + location._filePath = filePath; + location._fileName = filePath.split(path.sep).pop(); + location._lineNumber = parseInt(match[2], 10); + location._columnNumber = parseInt(match[3], 10); + return location; + } + return location; + } +} + +module.exports = Location; diff --git a/utils/testrunner/Matchers.js b/utils/testrunner/Matchers.js new file mode 100644 index 0000000000000..e9b64de90f56b --- /dev/null +++ b/utils/testrunner/Matchers.js @@ -0,0 +1,247 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const Location = require('./Location.js'); +const colors = require('colors/safe'); +const Diff = require('text-diff'); +const GoldenUtils = require('./GoldenUtils'); + +class Matchers { + constructor(config) { + this.expect = this.expect.bind(this); + + this._matchers = { + toBe: function(received, expected, message) { + message = message || `${received} == ${expected}`; + return { pass: received === expected, message, formatter: toBeFormatter.bind(null, received, expected) }; + }, + + toBeFalsy: function(received, message) { + message = message || `${received}`; + return { pass: !received, message }; + }, + + toBeTruthy: function(received, message) { + message = message || `${received}`; + return { pass: !!received, message }; + }, + + toBeGreaterThan: function(received, other, message) { + message = message || `${received} > ${other}`; + return { pass: received > other, message }; + }, + + toBeGreaterThanOrEqual: function(received, other, message) { + message = message || `${received} >= ${other}`; + return { pass: received >= other, message }; + }, + + toBeLessThan: function(received, other, message) { + message = message || `${received} < ${other}`; + return { pass: received < other, message }; + }, + + toBeLessThanOrEqual: function(received, other, message) { + message = message || `${received} <= ${other}`; + return { pass: received <= other, message }; + }, + + toBeNull: function(received, message) { + message = message || `${received} == null`; + return { pass: received === null, message }; + }, + + toContain: function(received, other, message) { + message = message || `${received} ⊇ ${other}`; + return { pass: received.includes(other), message }; + }, + + toEqual: function(received, other, message) { + let receivedJson = stringify(received); + let otherJson = stringify(other); + let formatter = objectFormatter.bind(null, receivedJson, otherJson); + if (receivedJson.length < 40 && otherJson.length < 40) { + receivedJson = receivedJson.split('\n').map(line => line.trim()).join(' '); + otherJson = otherJson.split('\n').map(line => line.trim()).join(' '); + formatter = stringFormatter.bind(null, receivedJson, otherJson); + } + message = message || `\n${receivedJson} ≈ ${otherJson}`; + return { pass: receivedJson === otherJson, message, formatter }; + }, + + toBeCloseTo: function(received, other, precision, message) { + return { + pass: Math.abs(received - other) < Math.pow(10, -precision), + message + }; + }, + + toBeInstanceOf: function(received, other, message) { + message = message || `${received.constructor.name} instanceof ${other.name}`; + return { pass: received instanceof other, message }; + }, + + toBeGolden: function(received, goldenName) { + return GoldenUtils.compare(received, { + goldenPath: config.goldenPath, + outputPath: config.outputPath, + goldenName + }); + }, + + toMatch: function(received, other, message) { + message = message || `${received}`; + return { pass: received.match(other), message }; + } + } + } + + expect(received) { + return new Expect(received, this._matchers); + } +}; + +class MatchError extends Error { + constructor(message, formatter) { + super(message); + this.name = this.constructor.name; + this.formatter = formatter; + this.location = Location.getCallerLocation(); + Error.captureStackTrace(this, this.constructor); + } +} + +module.exports = {Matchers, MatchError}; + +class Expect { + constructor(received, matchers) { + this.not = {}; + this.not.not = this; + for (const matcherName of Object.keys(matchers)) { + const matcher = matchers[matcherName]; + this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false /* inverse */, received); + this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true /* inverse */, received); + } + + function applyMatcher(matcherName, matcher, inverse, received, ...args) { + const result = matcher.call(null, received, ...args); + const message = `expect.${inverse ? 'not.' : ''}${matcherName} failed` + (result.message ? `: ${result.message}` : ''); + if (result.pass === inverse) + throw new MatchError(message, result.formatter || defaultFormatter.bind(null, received)); + } + } +} + +function defaultFormatter(received) { + return `Received: ${colors.red(JSON.stringify(received))}`; +} + +function stringFormatter(received, expected) { + const diff = new Diff(); + const result = diff.main(expected, received); + diff.cleanupSemantic(result); + const highlighted = result.map(([type, text]) => { + if (type === -1) + return colors.bgRed(text); + if (type === 1) + return colors.bgGreen.black(text); + return text; + }).join(''); + const output = [ + `Expected: ${expected}`, + `Received: ${received}`, + ` Diff: ${highlighted}`, + ]; + for (let i = 0; i < Math.min(expected.length, received.length); ++i) { + if (expected[i] !== received[i]) { + const padding = ' '.repeat(' Diff: '.length); + const firstDiffCharacter = '~'.repeat(i) + '^'; + output.push(colors.red(padding + firstDiffCharacter)); + break; + } + } + return output.join('\n'); +} + +function objectFormatter(received, expected) { + const encodingMap = new Map(); + const decodingMap = new Map(); + + const doEncodeLines = (lines) => { + let encoded = ''; + for (const line of lines) { + let code = encodingMap.get(line); + if (!code) { + code = String.fromCodePoint(encodingMap.size); + encodingMap.set(line, code); + decodingMap.set(code, line); + } + encoded += code; + } + return encoded; + }; + + const doDecodeLines = (text) => { + let decoded = []; + for (const codepoint of [...text]) + decoded.push(decodingMap.get(codepoint)); + return decoded; + } + + let receivedEncoded = doEncodeLines(received.split('\n')); + let expectedEncoded = doEncodeLines(expected.split('\n')); + + const diff = new Diff(); + const result = diff.main(expectedEncoded, receivedEncoded); + diff.cleanupSemantic(result); + + const highlighted = result.map(([type, text]) => { + const lines = doDecodeLines(text); + if (type === -1) + return lines.map(line => '- ' + colors.bgRed(line)); + if (type === 1) + return lines.map(line => '+ ' + colors.bgGreen.black(line)); + return lines.map(line => ' ' + line); + }); + + const flattened = []; + for (const list of highlighted) + flattened.push(...list); + return `Received:\n${flattened.join('\n')}`; +} + +function toBeFormatter(received, expected) { + if (typeof expected === 'string' && typeof received === 'string') { + return stringFormatter(JSON.stringify(received), JSON.stringify(expected)); + } + return [ + `Expected: ${JSON.stringify(expected)}`, + `Received: ${colors.red(JSON.stringify(received))}`, + ].join('\n'); +} + +function stringify(value) { + function stabilize(key, object) { + if (typeof object !== 'object' || object === undefined || object === null || Array.isArray(object)) + return object; + const result = {}; + for (const key of Object.keys(object).sort()) + result[key] = object[key]; + return result; + } + + return JSON.stringify(stabilize(null, value), stabilize, 2); +} diff --git a/utils/testrunner/README.md b/utils/testrunner/README.md new file mode 100644 index 0000000000000..9bf5fc6e99797 --- /dev/null +++ b/utils/testrunner/README.md @@ -0,0 +1,55 @@ +# TestRunner + +This test runner is used internally by Playwright to test Playwright itself. + +- testrunner is a *library*: tests are `node.js` scripts +- parallel wrt IO operations +- supports async/await +- modular +- well-isolated state per execution thread + +### Example + +Save the following as `test.js` and run using `node`: + +```sh +node test.js +``` + +```js +const {TestRunner, Reporter, Matchers} = require('.'); + +// Runner holds and runs all the tests +const runner = new TestRunner({ + parallel: 2, // run 2 parallel threads + timeout: 1000, // setup timeout of 1 second per test +}); +// Simple expect-like matchers +const {expect} = new Matchers(); + +// Extract jasmine-like DSL into the global namespace +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +// Test hooks can be async. +beforeAll(async state => { + state.parallelIndex; // either 0 or 1 in this example, depending on the executing thread + state.foo = 'bar'; // set state for every test +}); + +describe('math', () => { + it('to be sane', async (state, test) => { + state.parallelIndex; // Very first test will always be ran by the 0's thread + state.foo; // this will be 'bar' + expect(2 + 2).toBe(4); + }); +}); + +// Reporter subscribes to TestRunner events and displays information in terminal +const reporter = new Reporter(runner); + +// Run all tests. +runner.run(); +``` + diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js new file mode 100644 index 0000000000000..803cc820de365 --- /dev/null +++ b/utils/testrunner/Reporter.js @@ -0,0 +1,271 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fs = require('fs'); +const path = require('path'); +const colors = require('colors/safe'); +const {MatchError} = require('./Matchers.js'); + +class Reporter { + constructor(delegate, options = {}) { + const { + showSlowTests = 3, + showMarkedAsFailingTests = Infinity, + verbose = false, + summary = true, + lineBreak = 0, + } = options; + this._filePathToLines = new Map(); + this._delegate = delegate; + this._showSlowTests = showSlowTests; + this._showMarkedAsFailingTests = showMarkedAsFailingTests; + this._verbose = verbose; + this._summary = summary; + this._lineBreak = lineBreak; + this._testCounter = 0; + } + + onStarted(testRuns) { + this._testCounter = 0; + this._timestamp = Date.now(); + if (!this._delegate.hasFocusedTestsOrSuitesOrFiles()) { + console.log(`Running all ${colors.yellow(testRuns.length)} tests on ${colors.yellow(this._delegate.parallel())} worker${this._delegate.parallel() > 1 ? 's' : ''}:\n`); + } else { + console.log(`Running ${colors.yellow(testRuns.length)} focused tests out of total ${colors.yellow(this._delegate.testCount())} on ${colors.yellow(this._delegate.parallel())} worker${this._delegate.parallel() > 1 ? 's' : ''}`); + console.log(''); + const focusedFilePaths = this._delegate.focusedFilePaths(); + if (focusedFilePaths.length) { + console.log('Focused Files:'); + for (let i = 0; i < focusedFilePaths.length; ++i) + console.log(` ${i + 1}) ${colors.yellow(path.basename(focusedFilePaths[i]))}`); + console.log(''); + } + const focusedEntities = [ + ...this._delegate.focusedSuites(), + ...this._delegate.focusedTests(), + ]; + + if (focusedEntities.length) { + console.log('Focused Suites and Tests:'); + for (let i = 0; i < focusedEntities.length; ++i) + console.log(` ${i + 1}) ${focusedEntities[i].fullName()} (${formatLocation(focusedEntities[i].location())})`); + console.log(''); + } + } + } + + _printFailedResult(result) { + console.log(colors.red(`## ${result.result.toUpperCase()} ##`)); + if (result.message) { + console.log('Message:'); + console.log(` ${colors.red(result.message)}`); + } + + for (let i = 0; i < result.errors.length; i++) { + const { message, error, runs } = result.errors[i]; + console.log(`\n${colors.magenta('NON-TEST ERROR #' + i)}: ${message}`); + if (error && error.stack) + console.log(padLines(error.stack, 2)); + const lastRuns = runs.slice(runs.length - Math.min(10, runs.length)); + if (lastRuns.length) + console.log(`WORKER STATE`); + for (let j = 0; j < lastRuns.length; j++) + this._printVerboseTestRunResult(j, lastRuns[j]); + } + console.log(''); + console.log(''); + } + + onFinished(result) { + this._printTestResults(result); + if (!result.ok()) + this._printFailedResult(result); + process.exitCode = result.exitCode; + } + + _printTestResults(result) { + // 2 newlines after completing all tests. + console.log('\n'); + + const runs = result.runs; + const failedRuns = runs.filter(run => run.isFailure()); + const executedRuns = runs.filter(run => run.result()); + const okRuns = runs.filter(run => run.ok()); + const skippedRuns = runs.filter(run => run.result() === 'skipped'); + const markedAsFailingRuns = runs.filter(run => run.result() === 'markedAsFailing'); + + if (this._summary && failedRuns.length > 0) { + console.log('\nFailures:'); + for (let i = 0; i < failedRuns.length; ++i) { + this._printVerboseTestRunResult(i + 1, failedRuns[i]); + console.log(''); + } + } + + if (this._showMarkedAsFailingTests && this._summary && markedAsFailingRuns.length) { + if (markedAsFailingRuns.length > 0) { + console.log('\nMarked as failing:'); + markedAsFailingRuns.slice(0, this._showMarkedAsFailingTests).forEach((testRun, index) => { + console.log(`${index + 1}) ${testRun.test().fullName()} (${formatLocation(testRun.test().location())})`); + }); + } + if (this._showMarkedAsFailingTests < markedAsFailingRuns.length) { + console.log(''); + console.log(`... and ${colors.yellow(markedAsFailingRuns.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`); + } + } + + if (this._showSlowTests) { + const slowRuns = okRuns.sort((a, b) => b.duration() - a.duration()).slice(0, this._showSlowTests); + console.log(`\nSlowest tests:`); + for (let i = 0; i < slowRuns.length; ++i) { + const run = slowRuns[i]; + console.log(` (${i + 1}) ${colors.yellow((run.duration() / 1000) + 's')} - ${run.test().fullName()} (${formatLocation(run.test().location())})`); + } + } + + let summaryText = ''; + if (failedRuns.length || markedAsFailingRuns.length) { + const summary = [`ok - ${colors.green(okRuns.length)}`]; + if (failedRuns.length) + summary.push(`failed - ${colors.red(failedRuns.length)}`); + if (markedAsFailingRuns.length) + summary.push(`marked as failing - ${colors.yellow(markedAsFailingRuns.length)}`); + if (skippedRuns.length) + summary.push(`skipped - ${colors.yellow(skippedRuns.length)}`); + summaryText = ` (${summary.join(', ')})`; + } + + console.log(`\nRan ${executedRuns.length}${summaryText} of ${runs.length} test${runs.length > 1 ? 's' : ''}`); + const milliseconds = Date.now() - this._timestamp; + const seconds = milliseconds / 1000; + console.log(`Finished in ${colors.yellow(seconds)} seconds`); + } + + onTestRunStarted(testRun) { + } + + onTestRunFinished(testRun) { + ++this._testCounter; + if (this._verbose) { + this._printVerboseTestRunResult(this._testCounter, testRun); + } else { + if (testRun.result() === 'ok') + process.stdout.write(colors.green('\u00B7')); + else if (testRun.result() === 'skipped') + process.stdout.write(colors.yellow('\u00B7')); + else if (testRun.result() === 'markedAsFailing') + process.stdout.write(colors.yellow('\u00D7')); + else if (testRun.result() === 'failed') + process.stdout.write(colors.red('F')); + else if (testRun.result() === 'crashed') + process.stdout.write(colors.red('C')); + else if (testRun.result() === 'terminated') + process.stdout.write(colors.magenta('.')); + else if (testRun.result() === 'timedout') + process.stdout.write(colors.red('T')); + if (this._lineBreak && !(this._testCounter % this._lineBreak)) + process.stdout.write('\n'); + } + } + + _printVerboseTestRunResult(resultIndex, testRun) { + const test = testRun.test(); + let prefix = `${resultIndex})`; + if (this._delegate.parallel() > 1) + prefix += ' ' + colors.gray(`[worker = ${testRun.workerId()}]`); + if (testRun.result() === 'ok') { + console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName()} (${formatLocation(test.location())})`); + } else if (testRun.result() === 'terminated') { + console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName()} (${formatLocation(test.location())})`); + } else if (testRun.result() === 'crashed') { + console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName()} (${formatLocation(test.location())})`); + } else if (testRun.result() === 'skipped') { + } else if (testRun.result() === 'markedAsFailing') { + console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName()} (${formatLocation(test.location())})`); + } else if (testRun.result() === 'timedout') { + console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout()}ms]`)} ${test.fullName()} (${formatLocation(test.location())})`); + const output = testRun.output(); + if (output.length) { + console.log(' Output:'); + for (const line of output) + console.log(' ' + line); + } + } else if (testRun.result() === 'failed') { + console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName()} (${formatLocation(test.location())})`); + if (testRun.error() instanceof MatchError) { + const location = testRun.error().location; + let lines = this._filePathToLines.get(location.filePath()); + if (!lines) { + try { + lines = fs.readFileSync(location.filePath(), 'utf8').split('\n'); + } catch (e) { + lines = []; + } + this._filePathToLines.set(location.filePath(), lines); + } + const lineNumber = location.lineNumber(); + if (lineNumber < lines.length) { + const lineNumberLength = (lineNumber + 1 + '').length; + const FROM = Math.max(0, lineNumber - 5); + const snippet = lines.slice(FROM, lineNumber).map((line, index) => ` ${(FROM + index + 1 + '').padStart(lineNumberLength, ' ')} | ${line}`).join('\n'); + const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(location.columnNumber() - 1) + '^'; + console.log('\n' + snippet + '\n' + colors.grey(pointer) + '\n'); + } + console.log(padLines(testRun.error().formatter(), 4)); + console.log(''); + } else { + console.log(' Message:'); + let message = '' + (testRun.error().message || testRun.error()); + if (testRun.error().stack && message.includes(testRun.error().stack)) + message = message.substring(0, message.indexOf(testRun.error().stack)); + if (message) + console.log(` ${colors.red(message)}`); + if (testRun.error().stack) { + console.log(' Stack:'); + let stack = testRun.error().stack; + // Highlight first test location, if any. + const match = stack.match(new RegExp(test.location().filePath() + ':(\\d+):(\\d+)')); + if (match) { + const [, line, column] = match; + const fileName = `${test.location().fileName()}:${line}:${column}`; + stack = stack.substring(0, match.index) + stack.substring(match.index).replace(fileName, colors.yellow(fileName)); + } + console.log(padLines(stack, 4)); + } + } + const output = testRun.output(); + if (output.length) { + console.log(' Output:'); + for (const line of output) + console.log(' ' + line); + } + } + } +} + +function formatLocation(location) { + if (!location) + return ''; + return colors.yellow(`${location.toDetailedString()}`); +} + +function padLines(text, spaces = 0) { + const indent = ' '.repeat(spaces); + return text.split('\n').map(line => indent + line).join('\n'); +} + +module.exports = Reporter; diff --git a/utils/testrunner/SourceMap.js b/utils/testrunner/SourceMap.js new file mode 100644 index 0000000000000..c919715198786 --- /dev/null +++ b/utils/testrunner/SourceMap.js @@ -0,0 +1,460 @@ +/* + * Copyright (C) 2012 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +const path = require('path'); + +function upperBound(array, object, comparator, left, right) { + function defaultComparator(a, b) { + return a < b ? -1 : (a > b ? 1 : 0); + } + comparator = comparator || defaultComparator; + let l = left || 0; + let r = right !== undefined ? right : array.length; + while (l < r) { + const m = (l + r) >> 1; + if (comparator(object, array[m]) >= 0) { + l = m + 1; + } else { + r = m; + } + } + return r; +} + +/** + * @interface + */ +class SourceMap { + /** + * @return {string} + */ + compiledURL() { + } + + /** + * @return {string} + */ + url() { + } + + /** + * @return {!Array} + */ + sourceURLs() { + } + + /** + * @param {string} sourceURL + * @return {?string} + */ + embeddedContentByURL(sourceURL) { + } + + /** + * @param {number} lineNumber in compiled resource + * @param {number} columnNumber in compiled resource + * @return {?SourceMapEntry} + */ + findEntry(lineNumber, columnNumber) { + } + + /** + * @param {string} sourceURL + * @param {number} lineNumber + * @param {number} columnNumber + * @return {?SourceMapEntry} + */ + sourceLineMapping(sourceURL, lineNumber, columnNumber) { + } + + /** + * @return {!Array} + */ + mappings() { + } + + dispose() { + } +} + +/** + * @unrestricted + */ +class SourceMapV3 { + constructor() { + /** @type {number} */ this.version; + /** @type {string|undefined} */ this.file; + /** @type {!Array.} */ this.sources; + /** @type {!Array.|undefined} */ this.sections; + /** @type {string} */ this.mappings; + /** @type {string|undefined} */ this.sourceRoot; + /** @type {!Array.|undefined} */ this.names; + } +} + +/** + * @unrestricted + */ +SourceMapV3.Section = class { + constructor() { + /** @type {!SourceMapV3} */ this.map; + /** @type {!SourceMapV3.Offset} */ this.offset; + } +}; + +/** + * @unrestricted + */ +SourceMapV3.Offset = class { + constructor() { + /** @type {number} */ this.line; + /** @type {number} */ this.column; + } +}; + +/** + * @unrestricted + */ +class SourceMapEntry { + /** + * @param {number} lineNumber + * @param {number} columnNumber + * @param {string=} sourceURL + * @param {number=} sourceLineNumber + * @param {number=} sourceColumnNumber + * @param {string=} name + */ + constructor(lineNumber, columnNumber, sourceURL, sourceLineNumber, sourceColumnNumber, name) { + this.lineNumber = lineNumber; + this.columnNumber = columnNumber; + this.sourceURL = sourceURL; + this.sourceLineNumber = sourceLineNumber; + this.sourceColumnNumber = sourceColumnNumber; + this.name = name; + } + + /** + * @param {!SourceMapEntry} entry1 + * @param {!SourceMapEntry} entry2 + * @return {number} + */ + static compare(entry1, entry2) { + if (entry1.lineNumber !== entry2.lineNumber) { + return entry1.lineNumber - entry2.lineNumber; + } + return entry1.columnNumber - entry2.columnNumber; + } +} + +/** + * @implements {SourceMap} + * @unrestricted + */ +class TextSourceMap { + /** + * Implements Source Map V3 model. See https://github.com/google/closure-compiler/wiki/Source-Maps + * for format description. + * @param {string} compiledURL + * @param {string} sourceMappingURL + * @param {!SourceMapV3} payload + */ + constructor(compiledURL, sourceMappingURL, payload) { + if (!TextSourceMap._base64Map) { + const base64Digits = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + TextSourceMap._base64Map = {}; + for (let i = 0; i < base64Digits.length; ++i) { + TextSourceMap._base64Map[base64Digits.charAt(i)] = i; + } + } + + this._json = payload; + this._compiledURL = compiledURL; + this._sourceMappingURL = sourceMappingURL; + this._baseURL = sourceMappingURL.startsWith('data:') ? compiledURL : sourceMappingURL; + + /** @type {?Array} */ + this._mappings = null; + /** @type {!Map} */ + this._sourceInfos = new Map(); + if (this._json.sections) { + const sectionWithURL = !!this._json.sections.find(section => !!section.url); + if (sectionWithURL) { + cosole.warn(`SourceMap "${sourceMappingURL}" contains unsupported "URL" field in one of its sections.`); + } + } + this._eachSection(this._parseSources.bind(this)); + } + + /** + * @override + * @return {string} + */ + compiledURL() { + return this._compiledURL; + } + + /** + * @override + * @return {string} + */ + url() { + return this._sourceMappingURL; + } + + /** + * @override + * @return {!Array.} + */ + sourceURLs() { + return Array.from(this._sourceInfos.keys()); + } + + /** + * @override + * @param {string} sourceURL + * @return {?string} + */ + embeddedContentByURL(sourceURL) { + if (!this._sourceInfos.has(sourceURL)) { + return null; + } + return this._sourceInfos.get(sourceURL).content; + } + + /** + * @override + * @param {number} lineNumber in compiled resource + * @param {number} columnNumber in compiled resource + * @return {?SourceMapEntry} + */ + findEntry(lineNumber, columnNumber) { + const mappings = this.mappings(); + const index = upperBound(mappings, undefined, (unused, entry) => lineNumber - entry.lineNumber || columnNumber - entry.columnNumber); + return index ? mappings[index - 1] : null; + } + + /** + * @override + * @return {!Array} + */ + mappings() { + if (this._mappings === null) { + this._mappings = []; + this._eachSection(this._parseMap.bind(this)); + this._json = null; + } + return /** @type {!Array} */ (this._mappings); + } + + /** + * @param {function(!SourceMapV3, number, number)} callback + */ + _eachSection(callback) { + if (!this._json.sections) { + callback(this._json, 0, 0); + return; + } + for (const section of this._json.sections) { + callback(section.map, section.offset.line, section.offset.column); + } + } + + /** + * @param {!SourceMapV3} sourceMap + */ + _parseSources(sourceMap) { + const sourcesList = []; + let sourceRoot = sourceMap.sourceRoot || ''; + if (sourceRoot && !sourceRoot.endsWith('/')) { + sourceRoot += '/'; + } + for (let i = 0; i < sourceMap.sources.length; ++i) { + const href = sourceRoot + sourceMap.sources[i]; + let url = path.resolve(path.dirname(this._baseURL), href); + const source = sourceMap.sourcesContent && sourceMap.sourcesContent[i]; + if (url === this._compiledURL && source) { + url += '? [sm]'; + } + this._sourceInfos.set(url, new TextSourceMap.SourceInfo(source, null)); + sourcesList.push(url); + } + sourceMap[TextSourceMap._sourcesListSymbol] = sourcesList; + } + + /** + * @param {!SourceMapV3} map + * @param {number} lineNumber + * @param {number} columnNumber + */ + _parseMap(map, lineNumber, columnNumber) { + let sourceIndex = 0; + let sourceLineNumber = 0; + let sourceColumnNumber = 0; + let nameIndex = 0; + const sources = map[TextSourceMap._sourcesListSymbol]; + const names = map.names || []; + const stringCharIterator = new TextSourceMap.StringCharIterator(map.mappings); + let sourceURL = sources[sourceIndex]; + + while (true) { + if (stringCharIterator.peek() === ',') { + stringCharIterator.next(); + } else { + while (stringCharIterator.peek() === ';') { + lineNumber += 1; + columnNumber = 0; + stringCharIterator.next(); + } + if (!stringCharIterator.hasNext()) { + break; + } + } + + columnNumber += this._decodeVLQ(stringCharIterator); + if (!stringCharIterator.hasNext() || this._isSeparator(stringCharIterator.peek())) { + this._mappings.push(new SourceMapEntry(lineNumber, columnNumber)); + continue; + } + + const sourceIndexDelta = this._decodeVLQ(stringCharIterator); + if (sourceIndexDelta) { + sourceIndex += sourceIndexDelta; + sourceURL = sources[sourceIndex]; + } + sourceLineNumber += this._decodeVLQ(stringCharIterator); + sourceColumnNumber += this._decodeVLQ(stringCharIterator); + + if (!stringCharIterator.hasNext() || this._isSeparator(stringCharIterator.peek())) { + this._mappings.push( + new SourceMapEntry(lineNumber, columnNumber, sourceURL, sourceLineNumber, sourceColumnNumber)); + continue; + } + + nameIndex += this._decodeVLQ(stringCharIterator); + this._mappings.push(new SourceMapEntry( + lineNumber, columnNumber, sourceURL, sourceLineNumber, sourceColumnNumber, names[nameIndex])); + } + + // As per spec, mappings are not necessarily sorted. + this._mappings.sort(SourceMapEntry.compare); + } + + /** + * @param {string} char + * @return {boolean} + */ + _isSeparator(char) { + return char === ',' || char === ';'; + } + + /** + * @param {!TextSourceMap.StringCharIterator} stringCharIterator + * @return {number} + */ + _decodeVLQ(stringCharIterator) { + // Read unsigned value. + let result = 0; + let shift = 0; + let digit; + do { + digit = TextSourceMap._base64Map[stringCharIterator.next()]; + result += (digit & TextSourceMap._VLQ_BASE_MASK) << shift; + shift += TextSourceMap._VLQ_BASE_SHIFT; + } while (digit & TextSourceMap._VLQ_CONTINUATION_MASK); + + // Fix the sign. + const negative = result & 1; + result >>= 1; + return negative ? -result : result; + } + + /** + * @override + */ + dispose() { + } +} + +TextSourceMap._VLQ_BASE_SHIFT = 5; +TextSourceMap._VLQ_BASE_MASK = (1 << 5) - 1; +TextSourceMap._VLQ_CONTINUATION_MASK = 1 << 5; + +/** + * @unrestricted + */ +TextSourceMap.StringCharIterator = class { + /** + * @param {string} string + */ + constructor(string) { + this._string = string; + this._position = 0; + } + + /** + * @return {string} + */ + next() { + return this._string.charAt(this._position++); + } + + /** + * @return {string} + */ + peek() { + return this._string.charAt(this._position); + } + + /** + * @return {boolean} + */ + hasNext() { + return this._position < this._string.length; + } +}; + +/** + * @unrestricted + */ +TextSourceMap.SourceInfo = class { + /** + * @param {?string} content + * @param {?Array} reverseMappings + */ + constructor(content, reverseMappings) { + this.content = content; + this.reverseMappings = reverseMappings; + } +}; + +TextSourceMap._sourcesListSymbol = Symbol('sourcesList'); + +module.exports = {TextSourceMap}; diff --git a/utils/testrunner/SourceMapSupport.js b/utils/testrunner/SourceMapSupport.js new file mode 100644 index 0000000000000..2cabccc005e90 --- /dev/null +++ b/utils/testrunner/SourceMapSupport.js @@ -0,0 +1,84 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fs = require('fs'); +const path = require('path'); +const {TextSourceMap} = require('./SourceMap'); +const util = require('util'); + +const readFileAsync = util.promisify(fs.readFile.bind(fs)); + +class SourceMapSupport { + constructor() { + this._sourceMapPromises = new Map(); + } + + async rewriteStackTraceWithSourceMaps(error) { + if (!error.stack || typeof error.stack !== 'string') + return; + const stackFrames = error.stack.split('\n'); + for (let i = 0; i < stackFrames.length; ++i) { + const stackFrame = stackFrames[i]; + + let match = stackFrame.match(/\((.*):(\d+):(\d+)\)$/); + if (!match) + match = stackFrame.match(/^\s*at (.*):(\d+):(\d+)$/); + if (!match) + continue; + const filePath = match[1]; + const sourceMap = await this._maybeLoadSourceMapForPath(filePath); + if (!sourceMap) + continue; + const compiledLineNumber = parseInt(match[2], 10); + const compiledColumnNumber = parseInt(match[3], 10); + if (isNaN(compiledLineNumber) || isNaN(compiledColumnNumber)) + continue; + const entry = sourceMap.findEntry(compiledLineNumber, compiledColumnNumber); + if (!entry) + continue; + stackFrames[i] = stackFrame.replace(filePath + ':' + compiledLineNumber + ':' + compiledColumnNumber, entry.sourceURL + ':' + entry.sourceLineNumber + ':' + entry.sourceColumnNumber); + } + error.stack = stackFrames.join('\n'); + } + + async _maybeLoadSourceMapForPath(filePath) { + let sourceMapPromise = this._sourceMapPromises.get(filePath); + if (sourceMapPromise === undefined) { + sourceMapPromise = this._loadSourceMapForPath(filePath); + this._sourceMapPromises.set(filePath, sourceMapPromise); + } + return sourceMapPromise; + } + + async _loadSourceMapForPath(filePath) { + try { + const fileContent = await readFileAsync(filePath, 'utf8'); + const magicCommentLine = fileContent.trim().split('\n').pop().trim(); + const magicCommentMatch = magicCommentLine.match('^//#\\s*sourceMappingURL\\s*=(.*)$'); + if (!magicCommentMatch) + return null; + const sourceMappingURL = magicCommentMatch[1].trim(); + + const sourceMapPath = path.resolve(path.dirname(filePath), sourceMappingURL); + const json = JSON.parse(await readFileAsync(sourceMapPath, 'utf8')); + return new TextSourceMap(filePath, sourceMapPath, json); + } catch(e) { + return null; + } + } +} + +module.exports = {SourceMapSupport}; diff --git a/utils/testrunner/Test.js b/utils/testrunner/Test.js new file mode 100644 index 0000000000000..bd528180b11e5 --- /dev/null +++ b/utils/testrunner/Test.js @@ -0,0 +1,179 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const Location = require('./Location'); + +const TestExpectation = { + Ok: 'ok', + Fail: 'fail', +}; + +class Test { + constructor(suite, name, callback, location) { + this._suite = suite; + this._name = name; + this._fullName = (suite.fullName() + ' ' + name).trim(); + this._skipped = false; + this._expectation = TestExpectation.Ok; + this._body = callback; + this._location = location; + this._timeout = 100000000; + this._environments = []; + this.Expectations = { ...TestExpectation }; + } + + titles() { + if (!this._name) + return this._suite.titles(); + return [...this._suite.titles(), this._name]; + } + + suite() { + return this._suite; + } + + name() { + return this._name; + } + + fullName() { + return this._fullName; + } + + location() { + return this._location; + } + + body() { + return this._body; + } + + skipped() { + return this._skipped; + } + + setSkipped(skipped) { + this._skipped = skipped; + return this; + } + + timeout() { + return this._timeout; + } + + setTimeout(timeout) { + this._timeout = timeout; + return this; + } + + expectation() { + return this._expectation; + } + + setExpectation(expectation) { + this._expectation = expectation; + return this; + } + + addEnvironment(environment) { + this._environments.push(environment); + return this; + } + + removeEnvironment(environment) { + const index = this._environments.indexOf(environment); + if (index === -1) + throw new Error(`Environment "${environment.name()}" cannot be removed because it was not added to the suite "${this.fullName()}"`); + this._environments.splice(index, 1); + return this; + } +} + +class Suite { + constructor(parentSuite, name, location) { + this._parentSuite = parentSuite; + this._name = name; + const fullName = (parentSuite ? parentSuite.fullName() + ' ' + name : name).trim(); + this._fullName = fullName; + this._location = location; + this._skipped = false; + this._expectation = TestExpectation.Ok; + + this._defaultEnvironment = { + name() { return fullName; }, + }; + + this._environments = [this._defaultEnvironment]; + this.Expectations = { ...TestExpectation }; + } + + _addHook(name, callback) { + if (this._defaultEnvironment[name]) + throw new Error(`ERROR: cannot re-assign hook "${name}" for suite "${this._fullName}"`); + this._defaultEnvironment[name] = callback; + } + + beforeEach(callback) { this._addHook('beforeEach', callback); } + afterEach(callback) { this._addHook('afterEach', callback); } + beforeAll(callback) { this._addHook('beforeAll', callback); } + afterAll(callback) { this._addHook('afterAll', callback); } + globalSetup(callback) { this._addHook('globalSetup', callback); } + globalTeardown(callback) { this._addHook('globalTeardown', callback); } + + titles() { + if (!this._parentSuite) + return this._name ? [this._name] : []; + return this._name ? [...this._parentSuite.titles(), this._name] : this._parentSuite.titles(); + } + + parentSuite() { return this._parentSuite; } + + name() { return this._name; } + + fullName() { return this._fullName; } + + skipped() { return this._skipped; } + + setSkipped(skipped) { + this._skipped = skipped; + return this; + } + + location() { return this._location; } + + expectation() { return this._expectation; } + + setExpectation(expectation) { + this._expectation = expectation; + return this; + } + + addEnvironment(environment) { + this._environments.push(environment); + return this; + } + + removeEnvironment(environment) { + const index = this._environments.indexOf(environment); + if (index === -1) + throw new Error(`Environment "${environment.name()}" cannot be removed because it was not added to the suite "${this.fullName()}"`); + this._environments.splice(index, 1); + return this; + } +} + +module.exports = { TestExpectation, Test, Suite }; diff --git a/utils/testrunner/TestCollector.js b/utils/testrunner/TestCollector.js new file mode 100644 index 0000000000000..ec522994030fb --- /dev/null +++ b/utils/testrunner/TestCollector.js @@ -0,0 +1,259 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const Location = require('./Location'); +const { Test, Suite } = require('./Test'); +const { TestRun } = require('./TestRunner'); + +class FocusedFilter { + constructor() { + this._focusedTests = new Set(); + this._focusedSuites = new Set(); + this._focusedFilePaths = new Set(); + } + + focusTest(test) { this._focusedTests.add(test); } + focusSuite(suite) { this._focusedSuites.add(suite); } + focusFilePath(filePath) { this._focusedFilePaths.add(filePath); } + + hasFocusedTestsOrSuitesOrFiles() { + return !!this._focusedTests.size || !!this._focusedSuites.size || !!this._focusedFilePaths.size; + } + + focusedTests(tests) { + return tests.filter(test => this._focusedTests.has(test)); + } + + focusedSuites(suites) { + return suites.filter(suite => this._focusedSuites.has(suite)); + } + + focusedFilePaths(filePaths) { + return filePaths.filter(filePath => this._focusedFilePaths.has(filePath)); + } + + filter(tests) { + if (!this.hasFocusedTestsOrSuitesOrFiles()) + return tests; + + const ignoredSuites = new Set(); + const ignoredFilePaths = new Set(); + for (const test of tests) { + if (this._focusedTests.has(test)) { + // Focused tests should be run even if skipped. + test.setSkipped(false); + // TODO: remove next line once we run failing tests. + test.setExpectation(test.Expectations.Ok); + ignoredFilePaths.add(test.location().filePath()); + } + for (let suite = test.suite(); suite; suite = suite.parentSuite()) { + if (this._focusedSuites.has(suite)) { + // Focused suites should be run even if skipped. + suite.setSkipped(false); + // TODO: remove next line once we run failing tests. + suite.setExpectation(suite.Expectations.Ok); + } + // Mark parent suites of focused tests as ignored. + if (this._focusedTests.has(test)) + ignoredSuites.add(suite); + } + } + // Pick all tests that are focused or belong to focused suites. + const result = []; + for (const test of tests) { + let focused = this._focusedTests.has(test) || (this._focusedFilePaths.has(test.location().filePath()) && !ignoredFilePaths.has(test.location().filePath())); + for (let suite = test.suite(); suite; suite = suite.parentSuite()) + focused = focused || (this._focusedSuites.has(suite) && !ignoredSuites.has(suite)); + if (focused) + result.push(test); + } + return result; + } +} + +class Repeater { + constructor() { + this._repeatCount = new Map(); + } + + repeat(testOrSuite, count) { + this._repeatCount.set(testOrSuite, count); + } + + _get(testOrSuite) { + const repeat = this._repeatCount.get(testOrSuite); + return repeat === undefined ? 1 : repeat; + } + + createTestRuns(tests) { + const suiteToChildren = new Map(); + const rootSuites = new Set(); + for (const test of tests) { + let children = suiteToChildren.get(test.suite()); + if (!children) { + children = new Set(); + suiteToChildren.set(test.suite(), children); + } + children.add(test); + for (let suite = test.suite(); suite; suite = suite.parentSuite()) { + let children = suiteToChildren.get(suite.parentSuite()); + if (!children) { + children = new Set(); + suiteToChildren.set(suite.parentSuite(), children); + } + children.add(suite); + // Add root suites. + if (!suite.parentSuite()) + rootSuites.add(suite); + } + } + + const collectTests = (testOrSuite) => { + const testOrder = []; + if (testOrSuite instanceof Test) { + testOrder.push(testOrSuite); + } else { + for (const child of suiteToChildren.get(testOrSuite)) + testOrder.push(...collectTests(child)); + } + const repeat = this._repeatCount.has(testOrSuite) ? this._repeatCount.get(testOrSuite) : 1; + const result = []; + for (let i = 0; i < repeat; ++i) + result.push(...testOrder); + return result; + } + + const testOrder = []; + for (const rootSuite of rootSuites) + testOrder.push(...collectTests(rootSuite)); + return testOrder.map(test => new TestRun(test)); + + } +} + +function specBuilder(modifiers, attributes, specCallback) { + function builder(specs) { + return new Proxy((...args) => specCallback(specs, ...args), { + get: (obj, prop) => { + if (modifiers.has(prop)) + return (...args) => builder([...specs, { callback: modifiers.get(prop), args }]); + if (attributes.has(prop)) + return builder([...specs, { callback: attributes.get(prop), args: [] }]); + return obj[prop]; + }, + }); + } + return builder([]); +} + +class TestCollector { + constructor(options = {}) { + let { timeout = 10 * 1000 } = options; + if (timeout === 0) + timeout = 100000000; // Inifinite timeout. + + this._tests = []; + this._suites = []; + this._suiteModifiers = new Map(); + this._suiteAttributes = new Map(); + this._testModifiers = new Map(); + this._testAttributes = new Map(); + this._testCallbackWrappers = []; + this._api = {}; + + this._currentSuite = new Suite(null, '', new Location()); + this._rootSuite = this._currentSuite; + + this._api.describe = specBuilder(this._suiteModifiers, this._suiteAttributes, (specs, name, suiteCallback, ...suiteArgs) => { + const location = Location.getCallerLocation(); + const suite = new Suite(this._currentSuite, name, location); + for (const { callback, args } of specs) + callback(suite, ...args); + this._currentSuite = suite; + suiteCallback(...suiteArgs); + this._suites.push(suite); + this._currentSuite = suite.parentSuite(); + }); + this._api.it = specBuilder(this._testModifiers, this._testAttributes, (specs, name, testCallback) => { + const location = Location.getCallerLocation(); + for (const wrapper of this._testCallbackWrappers) + testCallback = wrapper(testCallback); + const test = new Test(this._currentSuite, name, testCallback, location); + test.setTimeout(timeout); + for (const { callback, args } of specs) + callback(test, ...args); + this._tests.push(test); + }); + this._api.beforeAll = callback => this._currentSuite.beforeAll(callback); + this._api.beforeEach = callback => this._currentSuite.beforeEach(callback); + this._api.afterAll = callback => this._currentSuite.afterAll(callback); + this._api.afterEach = callback => this._currentSuite.afterEach(callback); + this._api.globalSetup = callback => this._currentSuite.globalSetup(callback); + this._api.globalTeardown = callback => this._currentSuite.globalTeardown(callback); + } + + useEnvironment(environment) { + return this._currentSuite.addEnvironment(environment); + } + + addTestCallbackWrapper(wrapper) { + this._testCallbackWrappers.push(wrapper); + } + + addTestModifier(name, callback) { + this._testModifiers.set(name, callback); + } + + addTestAttribute(name, callback) { + this._testAttributes.set(name, callback); + } + + addSuiteModifier(name, callback) { + this._suiteModifiers.set(name, callback); + } + + addSuiteAttribute(name, callback) { + this._suiteAttributes.set(name, callback); + } + + api() { + return this._api; + } + + tests() { + return this._tests; + } + + suites() { + return this._suites; + } + + filePaths() { + const filePaths = new Set(); + for (const test of this._tests) + filePaths.add(test.location().filePath()); + for (const suite of this._suites) + filePaths.add(suite.location().filePath()); + return [...filePaths]; + } + + rootSuite() { + return this._rootSuite; + } +} + +module.exports = { TestCollector, specBuilder, FocusedFilter, Repeater }; diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js new file mode 100644 index 0000000000000..f790d0231dc8b --- /dev/null +++ b/utils/testrunner/TestRunner.js @@ -0,0 +1,581 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { SourceMapSupport } = require('./SourceMapSupport'); +const debug = require('debug'); +const { TestExpectation } = require('./Test'); + +const TimeoutError = new Error('Timeout'); +const TerminatedError = new Error('Terminated'); + +function runUserCallback(callback, timeout, args) { + let terminateCallback; + let timeoutId; + const promise = Promise.race([ + Promise.resolve().then(callback.bind(null, ...args)).then(() => null).catch(e => e), + new Promise(resolve => { + timeoutId = setTimeout(resolve.bind(null, TimeoutError), timeout); + }), + new Promise(resolve => terminateCallback = resolve), + ]).catch(e => e).finally(() => clearTimeout(timeoutId)); + const terminate = () => terminateCallback(TerminatedError); + return { promise, terminate }; +} + +const TestResult = { + Ok: 'ok', + MarkedAsFailing: 'markedAsFailing', // User marked as failed + Skipped: 'skipped', // User marked as skipped + Failed: 'failed', // Exception happened during running + TimedOut: 'timedout', // Timeout Exceeded while running + Terminated: 'terminated', // Execution terminated + Crashed: 'crashed', // If testrunner crashed due to this test +}; + +function isEmptyEnvironment(env) { + return !env.afterEach && !env.afterAll && !env.beforeEach && !env.beforeAll && + !env.globalSetup && !env.globalTeardown; +} + +class TestRun { + constructor(test) { + this._test = test; + this._result = null; + this._error = null; + this._startTimestamp = 0; + this._endTimestamp = 0; + this._workerId = null; + this._output = []; + + this._environments = test._environments.filter(env => !isEmptyEnvironment(env)).reverse(); + for (let suite = test.suite(); suite; suite = suite.parentSuite()) + this._environments.push(...suite._environments.filter(env => !isEmptyEnvironment(env)).reverse()); + this._environments.reverse(); + } + + finished() { + return this._result !== null && this._result !== 'running'; + } + + isFailure() { + return this._result === TestResult.Failed || this._result === TestResult.TimedOut || this._result === TestResult.Crashed; + } + + ok() { + return this._result === TestResult.Ok; + } + + result() { + return this._result; + } + + error() { + return this._error; + } + + duration() { + return this._endTimestamp - this._startTimestamp; + } + + test() { + return this._test; + } + + workerId() { + return this._workerId; + } + + log(log) { + this._output.push(log); + } + + output() { + return this._output; + } +} + +class Result { + constructor() { + this.result = TestResult.Ok; + this.exitCode = 0; + this.message = ''; + this.errors = []; + this.runs = []; + } + + setResult(result, message) { + if (!this.ok()) + return; + this.result = result; + this.message = message || ''; + if (result === TestResult.Ok) + this.exitCode = 0; + else if (result === TestResult.Terminated) + this.exitCode = 130; + else if (result === TestResult.Crashed) + this.exitCode = 2; + else + this.exitCode = 1; + } + + addError(message, error, worker) { + const data = { message, error, runs: [] }; + if (worker) + data.runs = worker._runs.slice(); + this.errors.push(data); + } + + ok() { + return this.result === TestResult.Ok; + } +} + +class TestWorker { + constructor(testRunner, hookRunner, workerId, parallelIndex) { + this._testRunner = testRunner; + this._hookRunner = hookRunner; + this._state = { parallelIndex }; + this._environmentStack = []; + this._terminating = false; + this._workerId = workerId; + this._runningTestTerminate = null; + this._runs = []; + } + + terminate(terminateHooks) { + this._terminating = true; + if (this._runningTestTerminate) + this._runningTestTerminate(); + this._hookRunner.terminateWorker(this); + } + + _markTerminated(testRun) { + if (!this._terminating) + return false; + testRun._result = TestResult.Terminated; + return true; + } + + async run(testRun) { + this._runs.push(testRun); + + const test = testRun.test(); + let skipped = test.skipped(); + for (let suite = test.suite(); suite; suite = suite.parentSuite()) + skipped = skipped || suite.skipped(); + if (skipped) { + await this._willStartTestRun(testRun); + testRun._result = TestResult.Skipped; + await this._didFinishTestRun(testRun); + return; + } + + let expectedToFail = test.expectation() === TestExpectation.Fail; + for (let suite = test.suite(); suite; suite = suite.parentSuite()) + expectedToFail = expectedToFail || (suite.expectation() === TestExpectation.Fail); + if (expectedToFail) { + await this._willStartTestRun(testRun); + testRun._result = TestResult.MarkedAsFailing; + await this._didFinishTestRun(testRun); + return; + } + + const environmentStack = testRun._environments; + let common = 0; + while (common < environmentStack.length && this._environmentStack[common] === environmentStack[common]) + common++; + + while (this._environmentStack.length > common) { + if (this._markTerminated(testRun)) + return; + const environment = this._environmentStack.pop(); + if (!await this._hookRunner.runHook(environment, 'afterAll', [this._state], this, testRun)) + return; + if (!await this._hookRunner.maybeRunGlobalTeardown(environment)) + return; + } + while (this._environmentStack.length < environmentStack.length) { + if (this._markTerminated(testRun)) + return; + const environment = environmentStack[this._environmentStack.length]; + this._environmentStack.push(environment); + if (!await this._hookRunner.maybeRunGlobalSetup(environment)) + return; + if (!await this._hookRunner.runHook(environment, 'beforeAll', [this._state], this, testRun)) + return; + } + + if (this._markTerminated(testRun)) + return; + + // From this point till the end, we have to run all hooks + // no matter what happens. + + await this._willStartTestRun(testRun); + for (const environment of this._environmentStack) { + await this._hookRunner.runHook(environment, 'beforeEach', [this._state, testRun], this, testRun); + } + + if (!testRun._error && !this._markTerminated(testRun)) { + await this._willStartTestBody(testRun); + const { promise, terminate } = runUserCallback(test.body(), test.timeout(), [this._state, testRun]); + this._runningTestTerminate = terminate; + testRun._error = await promise; + this._runningTestTerminate = null; + if (testRun._error && testRun._error.stack) + await this._testRunner._sourceMapSupport.rewriteStackTraceWithSourceMaps(testRun._error); + if (!testRun._error) + testRun._result = TestResult.Ok; + else if (testRun._error === TimeoutError) + testRun._result = TestResult.TimedOut; + else if (testRun._error === TerminatedError) + testRun._result = TestResult.Terminated; + else + testRun._result = TestResult.Failed; + await this._didFinishTestBody(testRun); + } + + for (const environment of this._environmentStack.slice().reverse()) + await this._hookRunner.runHook(environment, 'afterEach', [this._state, testRun], this, testRun); + await this._didFinishTestRun(testRun); + } + + async _willStartTestRun(testRun) { + testRun._startTimestamp = Date.now(); + testRun._workerId = this._workerId; + await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunStarted, [testRun]); + } + + async _didFinishTestRun(testRun) { + testRun._endTimestamp = Date.now(); + testRun._workerId = this._workerId; + + this._hookRunner.markFinishedTestRun(testRun); + await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunFinished, [testRun]); + } + + async _willStartTestBody(testRun) { + debug('testrunner:test')(`[${this._workerId}] starting "${testRun.test().fullName()}" (${testRun.test().location()})`); + } + + async _didFinishTestBody(testRun) { + debug('testrunner:test')(`[${this._workerId}] ${testRun._result.toUpperCase()} "${testRun.test().fullName()}" (${testRun.test().location()})`); + } + + async shutdown() { + while (this._environmentStack.length > 0) { + const environment = this._environmentStack.pop(); + await this._hookRunner.runHook(environment, 'afterAll', [this._state], this, null); + await this._hookRunner.maybeRunGlobalTeardown(environment); + } + } +} + +class HookRunner { + constructor(testRunner, testRuns) { + this._testRunner = testRunner; + this._runningHookTerminations = new Map(); + + this._environmentToGlobalState = new Map(); + for (const testRun of testRuns) { + for (const env of testRun._environments) { + let globalState = this._environmentToGlobalState.get(env); + if (!globalState) { + globalState = { + pendingTestRuns: new Set(), + globalSetupPromise: null, + globalTeardownPromise: null, + }; + this._environmentToGlobalState.set(env, globalState); + } + globalState.pendingTestRuns.add(testRun); + } + } + } + + terminateWorker(worker) { + let termination = this._runningHookTerminations.get(worker); + this._runningHookTerminations.delete(worker); + if (termination) + termination(); + } + + terminateAll() { + for (const termination of this._runningHookTerminations.values()) + termination(); + this._runningHookTerminations.clear(); + } + + markFinishedTestRun(testRun) { + for (const environment of testRun._environments) { + const globalState = this._environmentToGlobalState.get(environment); + globalState.pendingTestRuns.delete(testRun); + } + } + + async _runHookInternal(worker, testRun, hook, fullName, hookArgs = []) { + await this._willStartHook(worker, testRun, hook, fullName); + const timeout = this._testRunner._hookTimeout; + const { promise, terminate } = runUserCallback(hook.body, timeout, hookArgs); + this._runningHookTerminations.set(worker, terminate); + let error = await promise; + this._runningHookTerminations.delete(worker); + + if (error) { + if (testRun && testRun._result !== TestResult.Terminated) { + // Prefer terminated result over any hook failures. + testRun._result = error === TerminatedError ? TestResult.Terminated : TestResult.Crashed; + } + let message; + if (error === TimeoutError) { + message = `Timeout Exceeded ${timeout}ms while running "${hook.name}" in "${fullName}"`; + error = null; + } else if (error === TerminatedError) { + // Do not report termination details - it's just noise. + message = ''; + error = null; + } else { + if (error.stack) + await this._testRunner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); + message = `FAILED while running "${hook.name}" in suite "${fullName}": `; + } + await this._didFailHook(worker, testRun, hook, fullName, message, error); + if (testRun) + testRun._error = error; + return false; + } + + await this._didCompleteHook(worker, testRun, hook, fullName); + return true; + } + + async runHook(environment, hookName, hookArgs, worker = null, testRun = null) { + const hookBody = environment[hookName]; + if (!hookBody) + return true; + const envName = environment.name ? environment.name() : environment.constructor.name; + return await this._runHookInternal(worker, testRun, {name: hookName, body: hookBody.bind(environment)}, envName, hookArgs); + } + + async maybeRunGlobalSetup(environment) { + const globalState = this._environmentToGlobalState.get(environment); + if (!globalState.globalSetupPromise) + globalState.globalSetupPromise = this.runHook(environment, 'globalSetup', []); + if (!await globalState.globalSetupPromise) { + await this._testRunner._terminate(TestResult.Crashed, 'Global setup failed!', false, null); + return false; + } + return true; + } + + async maybeRunGlobalTeardown(environment) { + const globalState = this._environmentToGlobalState.get(environment); + if (!globalState.globalTeardownPromise) { + if (!globalState.pendingTestRuns.size || (this._testRunner._terminating && globalState.globalSetupPromise)) + globalState.globalTeardownPromise = this.runHook(environment, 'globalTeardown', []); + } + if (!globalState.globalTeardownPromise) + return true; + if (!await globalState.globalTeardownPromise) { + await this._testRunner._terminate(TestResult.Crashed, 'Global teardown failed!', false, null); + return false; + } + return true; + } + + async _willStartHook(worker, testRun, hook, fullName) { + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" started for "${testRun ? testRun.test().fullName() : ''}"`); + } + + async _didFailHook(worker, testRun, hook, fullName, message, error) { + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" FAILED for "${testRun ? testRun.test().fullName() : ''}"`); + if (message) + this._testRunner._result.addError(message, error, worker); + this._testRunner._result.setResult(TestResult.Crashed, message); + } + + async _didCompleteHook(worker, testRun, hook, fullName) { + debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" OK for "${testRun ? testRun.test().fullName() : ''}"`); + } +} + +function workerName(worker) { + return worker ? `` : `<_global_>`; +} + +class TestRunner { + constructor() { + this._sourceMapSupport = new SourceMapSupport(); + this._nextWorkerId = 1; + this._workers = []; + this._terminating = false; + this._result = null; + this._hookRunner = null; + } + + async _runDelegateCallback(callback, args) { + let { promise, terminate } = runUserCallback(callback, this._hookTimeout, args); + // Note: we do not terminate the delegate to keep reporting even when terminating. + const e = await promise; + if (e) { + debug('testrunner')(`Error while running delegate method: ${e}`); + const { message, error } = this._toError('INTERNAL ERROR', e); + this._terminate(TestResult.Crashed, message, false, error); + } + } + + _toError(message, error) { + if (!(error instanceof Error)) { + message += ': ' + error; + error = new Error(); + error.stack = ''; + } + return { message, error }; + } + + async run(testRuns, options = {}) { + const { + parallel = 1, + breakOnFailure = false, + hookTimeout = 10 * 1000, + totalTimeout = 0, + onStarted = async (testRuns) => {}, + onFinished = async (result) => {}, + onTestRunStarted = async (testRun) => {}, + onTestRunFinished = async (testRun) => {}, + } = options; + this._breakOnFailure = breakOnFailure; + this._hookTimeout = hookTimeout === 0 ? 100000000 : hookTimeout; + this._delegate = { + onStarted, + onFinished, + onTestRunStarted, + onTestRunFinished + }; + + this._result = new Result(); + this._result.runs = testRuns; + + const terminationPromises = []; + const handleSIGINT = () => this._terminate(TestResult.Terminated, 'SIGINT received', false, null); + const handleSIGHUP = () => this._terminate(TestResult.Terminated, 'SIGHUP received', false, null); + const handleSIGTERM = () => this._terminate(TestResult.Terminated, 'SIGTERM received', true, null); + const handleRejection = e => { + const { message, error } = this._toError('UNHANDLED PROMISE REJECTION', e); + terminationPromises.push(this._terminate(TestResult.Crashed, message, false, error)); + }; + const handleException = e => { + const { message, error } = this._toError('UNHANDLED ERROR', e); + terminationPromises.push(this._terminate(TestResult.Crashed, message, false, error)); + }; + process.on('SIGINT', handleSIGINT); + process.on('SIGHUP', handleSIGHUP); + process.on('SIGTERM', handleSIGTERM); + process.on('unhandledRejection', handleRejection); + process.on('uncaughtException', handleException); + + let timeoutId; + if (totalTimeout) { + timeoutId = setTimeout(() => { + terminationPromises.push(this._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */)); + }, totalTimeout); + } + await this._runDelegateCallback(this._delegate.onStarted, [testRuns]); + + this._hookRunner = new HookRunner(this, testRuns); + + const workerCount = Math.min(parallel, testRuns.length); + const workerPromises = []; + for (let i = 0; i < workerCount; ++i) { + const initialTestRunIndex = i * Math.floor(testRuns.length / workerCount); + workerPromises.push(this._runWorker(initialTestRunIndex, testRuns, i)); + } + await Promise.all(workerPromises); + await Promise.all(terminationPromises); + + if (testRuns.some(run => run.isFailure())) + this._result.setResult(TestResult.Failed, ''); + + await this._runDelegateCallback(this._delegate.onFinished, [this._result]); + clearTimeout(timeoutId); + + process.removeListener('SIGINT', handleSIGINT); + process.removeListener('SIGHUP', handleSIGHUP); + process.removeListener('SIGTERM', handleSIGTERM); + process.removeListener('unhandledRejection', handleRejection); + process.removeListener('uncaughtException', handleException); + return this._result; + } + + async _runWorker(testRunIndex, testRuns, parallelIndex) { + let worker = new TestWorker(this, this._hookRunner, this._nextWorkerId++, parallelIndex); + this._workers[parallelIndex] = worker; + while (!this._terminating) { + let skipped = 0; + while (skipped < testRuns.length && testRuns[testRunIndex]._result !== null) { + testRunIndex = (testRunIndex + 1) % testRuns.length; + skipped++; + } + const testRun = testRuns[testRunIndex]; + if (testRun._result !== null) { + // All tests have been run. + break; + } + + // Mark as running so that other workers do not run it again. + testRun._result = 'running'; + await worker.run(testRun); + if (testRun.isFailure()) { + // Something went wrong during test run, let's use a fresh worker. + await worker.shutdown(); + if (this._breakOnFailure) { + const message = `Terminating because a test has failed and |testRunner.breakOnFailure| is enabled`; + await this._terminate(TestResult.Terminated, message, false /* force */, null /* error */); + return; + } + worker = new TestWorker(this, this._hookRunner, this._nextWorkerId++, parallelIndex); + this._workers[parallelIndex] = worker; + } + } + await worker.shutdown(); + } + + async _terminate(result, message, force, error) { + debug('testrunner')(`TERMINATED result = ${result}, message = ${message}`); + this._terminating = true; + for (const worker of this._workers) + worker.terminate(force /* terminateHooks */); + if (this._hookRunner) + this._hookRunner.terminateAll(); + this._result.setResult(result, message); + if (this._result.message === 'SIGINT received' && message === 'SIGTERM received') + this._result.message = message; + if (error) { + if (error.stack) + await this._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); + this._result.addError(message, error, this._workers.length === 1 ? this._workers[0] : null); + } + } + + async terminate() { + if (!this._result) + return; + await this._terminate(TestResult.Terminated, 'Terminated with |TestRunner.terminate()| call', true /* force */, null /* error */); + } +} + +module.exports = { TestRunner, TestRun, TestResult, Result }; diff --git a/utils/testrunner/diffstyle.css b/utils/testrunner/diffstyle.css new file mode 100644 index 0000000000000..c58f0e90a6a89 --- /dev/null +++ b/utils/testrunner/diffstyle.css @@ -0,0 +1,13 @@ +body { + font-family: monospace; + white-space: pre; +} + +ins { + background-color: #9cffa0; + text-decoration: none; +} + +del { + background-color: #ff9e9e; +} diff --git a/utils/testrunner/examples/fail.js b/utils/testrunner/examples/fail.js new file mode 100644 index 0000000000000..6e2cd75ac419f --- /dev/null +++ b/utils/testrunner/examples/fail.js @@ -0,0 +1,54 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {TestRunner, Reporter, Matchers} = require('..'); + +const runner = new TestRunner(); +const reporter = new Reporter(runner); +const {expect} = new Matchers(); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +describe('testsuite', () => { + it('toBe', async (state) => { + expect(2 + 2).toBe(5); + }); + it('toBeFalsy', async (state) => { + expect(true).toBeFalsy(); + }); + it('toBeTruthy', async (state) => { + expect(false).toBeTruthy(); + }); + it('toBeGreaterThan', async (state) => { + expect(2).toBeGreaterThan(3); + }); + it('toBeNull', async (state) => { + expect(2).toBeNull(); + }); + it('toContain', async (state) => { + expect('asdf').toContain('e'); + }); + it('not.toContain', async (state) => { + expect('asdf').not.toContain('a'); + }); + it('toEqual', async (state) => { + expect([1,2,3]).toEqual([1,2,3,4]); + }); +}); + +runner.run(); diff --git a/utils/testrunner/examples/hookfail.js b/utils/testrunner/examples/hookfail.js new file mode 100644 index 0000000000000..038f58c47f2fd --- /dev/null +++ b/utils/testrunner/examples/hookfail.js @@ -0,0 +1,35 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {TestRunner, Reporter, Matchers} = require('..'); + +const runner = new TestRunner(); +const reporter = new Reporter(runner); +const {expect} = new Matchers(); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +describe('testsuite', () => { + beforeAll(() => { + expect(false).toBeTruthy(); + }); + it('test', async () => { + }); +}); + +runner.run(); diff --git a/utils/testrunner/examples/hooktimeout.js b/utils/testrunner/examples/hooktimeout.js new file mode 100644 index 0000000000000..525f2b7c8adf0 --- /dev/null +++ b/utils/testrunner/examples/hooktimeout.js @@ -0,0 +1,35 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {TestRunner, Reporter, Matchers} = require('..'); + +const runner = new TestRunner({ timeout: 100 }); +const reporter = new Reporter(runner); +const {expect} = new Matchers(); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +describe('testsuite', () => { + beforeAll(async () => { + await new Promise(() => {}); + }); + it('something', async (state) => { + }); +}); + +runner.run(); diff --git a/utils/testrunner/examples/timeout.js b/utils/testrunner/examples/timeout.js new file mode 100644 index 0000000000000..0d451e971e889 --- /dev/null +++ b/utils/testrunner/examples/timeout.js @@ -0,0 +1,32 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {TestRunner, Reporter} = require('..'); + +const runner = new TestRunner({ timeout: 100 }); +const reporter = new Reporter(runner); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +describe('testsuite', () => { + it('timeout', async (state) => { + await new Promise(() => {}); + }); +}); + +runner.run(); diff --git a/utils/testrunner/examples/unhandledpromiserejection.js b/utils/testrunner/examples/unhandledpromiserejection.js new file mode 100644 index 0000000000000..270bfcc422171 --- /dev/null +++ b/utils/testrunner/examples/unhandledpromiserejection.js @@ -0,0 +1,35 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {TestRunner, Reporter} = require('..'); + +const runner = new TestRunner(); +const reporter = new Reporter(runner); + +const {describe, xdescribe, fdescribe} = runner; +const {it, fit, xit} = runner; +const {beforeAll, beforeEach, afterAll, afterEach} = runner; + +describe('testsuite', () => { + it('failure', async (state) => { + Promise.reject(new Error('fail!')); + }); + it('slow', async () => { + await new Promise(x => setTimeout(x, 1000)); + }); +}); + +runner.run(); diff --git a/utils/testrunner/index.js b/utils/testrunner/index.js new file mode 100644 index 0000000000000..1cf2e5f40bdcd --- /dev/null +++ b/utils/testrunner/index.js @@ -0,0 +1,168 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const path = require('path'); +const { TestRunner, Result, TestResult } = require('./TestRunner'); +const { TestCollector, FocusedFilter, Repeater } = require('./TestCollector'); +const Reporter = require('./Reporter'); +const { Matchers } = require('./Matchers'); + +class DefaultTestRunner { + constructor(options = {}) { + const { + // Our options. + crashIfTestsAreFocusedOnCI = true, + exit = true, + reporter = true, + // Collector options. + timeout, + // Runner options. + parallel = 1, + breakOnFailure, + totalTimeout, + hookTimeout = timeout, + // Reporting options. + showSlowTests, + showMarkedAsFailingTests, + verbose, + summary, + lineBreak, + goldenPath, + outputPath, + } = options; + + this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI; + this._exit = exit; + this._parallel = parallel; + this._breakOnFailure = breakOnFailure; + this._totalTimeout = totalTimeout; + this._hookTimeout = hookTimeout; + this._needReporter = reporter; + this._showSlowTests = showSlowTests; + this._showMarkedAsFailingTests = showMarkedAsFailingTests; + this._verbose = verbose; + this._summary = summary; + this._lineBreak = lineBreak; + + this._filter = new FocusedFilter(); + this._repeater = new Repeater(); + this._collector = new TestCollector({ timeout }); + + this._api = { + ...this._collector.api(), + expect: new Matchers({ goldenPath, outputPath }).expect, + }; + this._collector.addSuiteAttribute('only', s => this._filter.focusSuite(s)); + this._collector.addSuiteAttribute('skip', s => s.setSkipped(true)); + this._collector.addSuiteModifier('repeat', (s, count) => this._repeater.repeat(s, count)); + this._collector.addTestAttribute('only', t => this._filter.focusTest(t)); + this._collector.addTestAttribute('skip', t => t.setSkipped(true)); + this._collector.addTestAttribute('todo', t => t.setSkipped(true)); + this._collector.addTestAttribute('slow', t => t.setTimeout(t.timeout() * 3)); + this._collector.addTestModifier('repeat', (t, count) => this._repeater.repeat(t, count)); + this._api.fdescribe = this._api.describe.only; + this._api.xdescribe = this._api.describe.skip; + this._api.fit = this._api.it.only; + this._api.xit = this._api.it.skip; + } + + collector() { + return this._collector; + } + + api() { + return this._api; + } + + focusMatchingNameTests(fullNameRegex) { + const focusedTests = []; + for (const test of this._collector.tests()) { + if (fullNameRegex.test(test.fullName())) { + this._filter.focusTest(test); + focusedTests.push(test); + } + } + return focusedTests; + } + + focusMatchingFileName(filenameRegex) { + const focusedFilePaths = []; + for (const filePath of this._collector.filePaths()) { + if (filenameRegex.test(path.basename(filePath))) { + this._filter.focusFilePath(filePath); + focusedFilePaths.push(filePath); + } + } + return focusedFilePaths; + } + + repeatAll(repeatCount) { + this._repeater.repeat(this._collector.rootSuite(), repeatCount); + } + + async run() { + let reporter = null; + + if (this._needReporter) { + const reporterDelegate = { + focusedSuites: () => this._filter.focusedSuites(this._collector.suites()), + focusedTests: () => this._filter.focusedTests(this._collector.tests()), + focusedFilePaths: () => this._filter.focusedFilePaths(this._collector.filePaths()), + hasFocusedTestsOrSuitesOrFiles: () => this._filter.hasFocusedTestsOrSuitesOrFiles(), + parallel: () => this._parallel, + testCount: () => this._collector.tests().length, + }; + const reporterOptions = { + showSlowTests: this._showSlowTests, + showMarkedAsFailingTests: this._showMarkedAsFailingTests, + verbose: this._verbose, + summary: this._summary, + lineBreak: this._lineBreak, + }; + reporter = new Reporter(reporterDelegate, reporterOptions); + } + + if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this._filter.hasFocusedTestsOrSuitesOrFiles()) { + if (reporter) + await reporter.onStarted([]); + const result = new Result(); + result.setResult(TestResult.Crashed, '"focused" tests or suites are probitted on CI'); + if (reporter) + await reporter.onFinished(result); + if (this._exit) + process.exit(result.exitCode); + return result; + } + + const testRuns = this._repeater.createTestRuns(this._filter.filter(this._collector.tests())); + const testRunner = new TestRunner(); + const result = await testRunner.run(testRuns, { + parallel: this._parallel, + breakOnFailure: this._breakOnFailure, + totalTimeout: this._totalTimeout, + hookTimeout: this._hookTimeout, + onStarted: (...args) => reporter && reporter.onStarted(...args), + onFinished: (...args) => reporter && reporter.onFinished(...args), + onTestRunStarted: (...args) => reporter && reporter.onTestRunStarted(...args), + onTestRunFinished: (...args) => reporter && reporter.onTestRunFinished(...args), + }); + if (this._exit) + process.exit(result.exitCode); + return result; + } +} + +module.exports = DefaultTestRunner; diff --git a/utils/testrunner/test/test.js b/utils/testrunner/test/test.js new file mode 100644 index 0000000000000..304b847967828 --- /dev/null +++ b/utils/testrunner/test/test.js @@ -0,0 +1,4 @@ +const TestRunner = require('..'); +const testRunner = new TestRunner(); +require('./testrunner.spec.js').addTests(testRunner.api()); +testRunner.run(); diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js new file mode 100644 index 0000000000000..3655c292ea4f5 --- /dev/null +++ b/utils/testrunner/test/testrunner.spec.js @@ -0,0 +1,1052 @@ +const { TestRunner } = require('../TestRunner'); +const { TestCollector, FocusedFilter, Repeater } = require('../TestCollector'); +const { TestExpectation, Environment } = require('../Test'); + +class Runner { + constructor(options = {}) { + this._options = options; + this._filter = new FocusedFilter(); + this._repeater = new Repeater(); + this._collector = new TestCollector(options); + this._collector.addSuiteAttribute('only', s => this._filter.focusSuite(s)); + this._collector.addTestAttribute('only', t => this._filter.focusTest(t)); + this._collector.addSuiteAttribute('skip', s => s.setSkipped(true)); + this._collector.addTestAttribute('skip', t => t.setSkipped(true)); + this._collector.addTestAttribute('fail', t => t.setExpectation(t.Expectations.Fail)); + this._collector.addSuiteModifier('repeat', (s, count) => this._repeater.repeat(s, count)); + this._collector.addTestModifier('repeat', (t, count) => this._repeater.repeat(t, count)); + + const api = this._collector.api(); + for (const [key, value] of Object.entries(api)) + this[key] = value; + this.fdescribe = api.describe.only; + this.xdescribe = api.describe.skip; + this.fit = api.it.only; + this.xit = api.it.skip; + this.Expectations = { ...TestExpectation }; + } + + createTestRuns() { + return this._repeater.createTestRuns(this._filter.filter(this._collector.tests())); + } + + run() { + this._testRunner = new TestRunner(); + return this._testRunner.run(this.createTestRuns(), this._options); + } + + tests() { + return this._collector.tests(); + } + + parallel() { + return this._options.parallel || 1; + } + + focusedTests() { + return this._filter.focusedTests(this._collector.tests()); + } + + suites() { + return this._collector.suites(); + } + + focusedSuites() { + return this._filter.focusedSuites(this._collector.suites()); + } + + terminate() { + this._testRunner.terminate(); + } +} + +module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit, expect}) { + describe('TestRunner.it', () => { + it('should declare a test', async() => { + const t = new Runner(); + t.it('uno', () => {}); + expect(t.tests().length).toBe(1); + const test = t.tests()[0]; + expect(test.name()).toBe('uno'); + expect(test.fullName()).toBe('uno'); + expect(test.skipped()).toBe(false); + expect(test.location().filePath()).toEqual(__filename); + expect(test.location().fileName()).toEqual('testrunner.spec.js'); + expect(test.location().lineNumber()).toBeTruthy(); + expect(test.location().columnNumber()).toBeTruthy(); + }); + it('should run a test', async() => { + const t = new Runner(); + t.it('uno', () => {}); + const result = await t.run(); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('ok'); + }); + }); + + describe('TestRunner.xit', () => { + it('should declare a skipped test', async() => { + const t = new Runner(); + t.xit('uno', () => {}); + expect(t.tests().length).toBe(1); + const test = t.tests()[0]; + expect(test.name()).toBe('uno'); + expect(test.fullName()).toBe('uno'); + expect(test.skipped()).toBe(true); + }); + it('should not run a skipped test', async() => { + const t = new Runner(); + t.xit('uno', () => {}); + const result = await t.run(); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('skipped'); + }); + }); + + describe('TestRunner.fit', () => { + it('should declare a focused test', async() => { + const t = new Runner(); + t.fit('uno', () => {}); + expect(t.tests().length).toBe(1); + const test = t.tests()[0]; + expect(test.name()).toBe('uno'); + expect(test.fullName()).toBe('uno'); + expect(test.skipped()).toBe(false); + expect(t.focusedTests()[0]).toBe(test); + }); + it('should run a focused test', async() => { + const t = new Runner(); + t.fit('uno', () => {}); + const result = await t.run(); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('ok'); + }); + it('should run a failed focused test', async() => { + const t = new Runner(); + let run = false; + t.it.only.fail('uno', () => { + run = true; throw new Error('failure'); + }); + expect(t.focusedTests().length).toBe(1); + expect(t.tests()[0].expectation()).toBe(t.Expectations.Fail); + const result = await t.run(); + expect(run).toBe(true); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('failed'); + }); + }); + + describe('TestRunner.describe', () => { + it('should declare a suite', async() => { + const t = new Runner(); + t.describe('suite', () => { + t.it('uno', () => {}); + }); + expect(t.tests().length).toBe(1); + const test = t.tests()[0]; + expect(test.name()).toBe('uno'); + expect(test.fullName()).toBe('suite uno'); + expect(test.skipped()).toBe(false); + expect(test.suite().name()).toBe('suite'); + expect(test.suite().fullName()).toBe('suite'); + expect(test.suite().skipped()).toBe(false); + }); + }); + + describe('TestRunner.xdescribe', () => { + it('should declare a skipped suite', async() => { + const t = new Runner(); + t.xdescribe('suite', () => { + t.it('uno', () => {}); + }); + expect(t.tests().length).toBe(1); + const test = t.tests()[0]; + expect(test.skipped()).toBe(false); + expect(test.suite().skipped()).toBe(true); + }); + it('focused tests inside a skipped suite are not run', async() => { + const t = new Runner(); + let run = false; + t.xdescribe('suite', () => { + t.fit('uno', () => { run = true; }); + }); + const result = await t.run(); + expect(run).toBe(false); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('skipped'); + }); + }); + + describe('TestRunner.fdescribe', () => { + it('should declare a focused suite', async() => { + const t = new Runner(); + t.fdescribe('suite', () => { + t.it('uno', () => {}); + }); + expect(t.tests().length).toBe(1); + const test = t.tests()[0]; + expect(test.skipped()).toBe(false); + expect(t.focusedSuites()[0]).toBe(test.suite()); + expect(test.suite().skipped()).toBe(false); + }); + it('skipped tests inside a focused suite should not be run', async() => { + const t = new Runner(); + t.fdescribe('suite', () => { + t.xit('uno', () => {}); + }); + const result = await t.run(); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('skipped'); + }); + it('should run all "run" tests inside a focused suite', async() => { + const log = []; + const t = new Runner(); + t.it('uno', () => log.push(1)); + t.fdescribe('suite1', () => { + t.it('dos', () => log.push(2)); + t.it('tres', () => log.push(3)); + }); + t.it('cuatro', () => log.push(4)); + await t.run(); + expect(log.join()).toBe('2,3'); + }); + it('should run only "focus" tests inside a focused suite', async() => { + const log = []; + const t = new Runner(); + t.it('uno', () => log.push(1)); + t.fdescribe('suite1', () => { + t.fit('dos', () => log.push(2)); + t.it('tres', () => log.push(3)); + }); + t.it('cuatro', () => log.push(4)); + await t.run(); + expect(log.join()).toBe('2'); + }); + it('should run both "run" tests in focused suite and non-descendant focus tests', async() => { + const log = []; + const t = new Runner(); + t.it('uno', () => log.push(1)); + t.fdescribe('suite1', () => { + t.it('dos', () => log.push(2)); + t.it('tres', () => log.push(3)); + }); + t.fit('cuatro', () => log.push(4)); + await t.run(); + expect(log.join()).toBe('2,3,4'); + }); + }); + + describe('TestRunner attributes', () => { + it('should work', async() => { + const t = new Runner({timeout: 123}); + const log = []; + + t._collector.addTestModifier('foo', (t, ...args) => { + log.push('foo'); + + expect(t.skipped()).toBe(false); + expect(t.Expectations.Ok).toBeTruthy(); + expect(t.Expectations.Fail).toBeTruthy(); + expect(t.expectation()).toBe(t.Expectations.Ok); + expect(t.timeout()).toBe(123); + + expect(args.length).toBe(2); + expect(args[0]).toBe('uno'); + expect(args[1]).toBe('dos'); + + t.setExpectation(t.Expectations.Fail); + t.setTimeout(234); + }); + + t._collector.addTestAttribute('bar', t => { + log.push('bar'); + t.setSkipped(true); + expect(t.skipped()).toBe(true); + expect(t.expectation()).toBe(t.Expectations.Fail); + expect(t.timeout()).toBe(234); + }); + + t.it.foo('uno', 'dos').bar('test', () => { }); + expect(log).toEqual(['foo', 'bar']); + }); + }); + + describe('TestRunner hooks', () => { + it('should run all hooks in proper order', async() => { + const log = []; + const t = new Runner(); + const e = { + name() { return 'env'; }, + beforeAll() { log.push('env:beforeAll'); }, + afterAll() { log.push('env:afterAll'); }, + beforeEach() { log.push('env:beforeEach'); }, + afterEach() { log.push('env:afterEach'); }, + }; + const e2 = { + name() { return 'env2'; }, + beforeAll() { log.push('env2:beforeAll'); }, + afterAll() { log.push('env2:afterAll'); }, + }; + t.beforeAll(() => log.push('root:beforeAll')); + t.beforeEach(() => log.push('root:beforeEach')); + t.it('uno', () => log.push('test #1')); + t.describe('suite1', () => { + t.beforeAll(() => log.push('suite:beforeAll')); + t.beforeEach(() => log.push('suite:beforeEach')); + t.it('dos', () => log.push('test #2')); + t.it('tres', () => log.push('test #3')); + t.afterEach(() => log.push('suite:afterEach')); + t.afterAll(() => log.push('suite:afterAll')); + }); + t.it('cuatro', () => log.push('test #4')); + t.tests()[t.tests().length - 1].addEnvironment(e); + t.tests()[t.tests().length - 1].addEnvironment(e2); + t.describe('no hooks suite', () => { + t.describe('suite2', () => { + t.beforeAll(() => log.push('suite2:beforeAll')); + t.afterAll(() => log.push('suite2:afterAll')); + t.describe('no hooks suite 2', () => { + t.it('cinco', () => log.push('test #5')); + }); + }); + }); + t.suites()[t.suites().length - 1].addEnvironment(e); + t.suites()[t.suites().length - 1].addEnvironment(e2); + t.afterEach(() => log.push('root:afterEach')); + t.afterAll(() => log.push('root:afterAll')); + await t.run(); + expect(log).toEqual([ + 'root:beforeAll', + 'root:beforeEach', + 'test #1', + 'root:afterEach', + + 'suite:beforeAll', + + 'root:beforeEach', + 'suite:beforeEach', + 'test #2', + 'suite:afterEach', + 'root:afterEach', + + 'root:beforeEach', + 'suite:beforeEach', + 'test #3', + 'suite:afterEach', + 'root:afterEach', + + 'suite:afterAll', + + 'env:beforeAll', + 'env2:beforeAll', + + 'root:beforeEach', + 'env:beforeEach', + 'test #4', + 'env:afterEach', + 'root:afterEach', + + 'suite2:beforeAll', + 'root:beforeEach', + 'env:beforeEach', + 'test #5', + 'env:afterEach', + 'root:afterEach', + 'suite2:afterAll', + + 'env2:afterAll', + 'env:afterAll', + + 'root:afterAll', + ]); + }); + it('should remove environment', async() => { + const log = []; + const t = new Runner(); + const e = { + name() { return 'env'; }, + beforeAll() { log.push('env:beforeAll'); }, + afterAll() { log.push('env:afterAll'); }, + beforeEach() { log.push('env:beforeEach'); }, + afterEach() { log.push('env:afterEach'); }, + }; + const e2 = { + name() { return 'env2'; }, + beforeAll() { log.push('env2:beforeAll'); }, + afterAll() { log.push('env2:afterAll'); }, + beforeEach() { log.push('env2:beforeEach'); }, + afterEach() { log.push('env2:afterEach'); }, + }; + t.it('uno', () => log.push('test #1')); + t.tests()[0].addEnvironment(e).addEnvironment(e2).removeEnvironment(e); + await t.run(); + expect(log).toEqual([ + 'env2:beforeAll', + 'env2:beforeEach', + 'test #1', + 'env2:afterEach', + 'env2:afterAll', + ]); + }); + it('should have the same state object in hooks and test', async() => { + const states = []; + const t = new Runner(); + t.beforeEach(state => states.push(state)); + t.afterEach(state => states.push(state)); + t.beforeAll(state => states.push(state)); + t.afterAll(state => states.push(state)); + t.it('uno', state => states.push(state)); + await t.run(); + expect(states.length).toBe(5); + for (let i = 1; i < states.length; ++i) + expect(states[i]).toBe(states[0]); + }); + it('should unwind hooks properly when terminated', async() => { + const log = []; + const t = new Runner({timeout: 10000}); + t.beforeAll(() => log.push('beforeAll')); + t.beforeEach(() => log.push('beforeEach')); + t.afterEach(() => log.push('afterEach')); + t.afterAll(() => log.push('afterAll')); + t.it('uno', () => { + log.push('terminating...'); + t.terminate(); + }); + await t.run(); + + expect(log).toEqual([ + 'beforeAll', + 'beforeEach', + 'terminating...', + 'afterEach', + 'afterAll', + ]); + }); + it('should report as terminated even when hook crashes', async() => { + const t = new Runner({timeout: 10000}); + t.afterEach(() => { throw new Error('crash!'); }); + t.it('uno', () => { t.terminate(); }); + const result = await t.run(); + expect(result.runs[0].result()).toBe('terminated'); + }); + it('should report as terminated when terminated during hook', async() => { + const t = new Runner({timeout: 10000}); + t.afterEach(() => { t.terminate(); }); + t.it('uno', () => { }); + const result = await t.run(); + expect(result.runs[0].result()).toBe('terminated'); + }); + it('should unwind hooks properly when crashed', async() => { + const log = []; + const t = new Runner({timeout: 10000}); + t.beforeAll(() => log.push('root beforeAll')); + t.beforeEach(() => log.push('root beforeEach')); + t.describe('suite', () => { + t.beforeAll(() => log.push('suite beforeAll')); + t.beforeEach(() => log.push('suite beforeEach')); + t.it('uno', () => log.push('uno')); + t.afterEach(() => { + log.push('CRASH >> suite afterEach'); + throw new Error('crash!'); + }); + t.afterAll(() => log.push('suite afterAll')); + }); + t.afterEach(() => log.push('root afterEach')); + t.afterAll(() => log.push('root afterAll')); + await t.run(); + + expect(log).toEqual([ + 'root beforeAll', + 'suite beforeAll', + 'root beforeEach', + 'suite beforeEach', + 'uno', + 'CRASH >> suite afterEach', + 'root afterEach', + 'suite afterAll', + 'root afterAll' + ]); + }); + }); + + describe('globalSetup & globalTeardwon', () => { + it('should run globalSetup and globalTeardown in proper order', async() => { + const t = new Runner({timeout: 10000}); + const tracer = new TestTracer(t); + tracer.traceAllHooks(); + tracer.addTest('', 'test1'); + await t.run(); + + expect(tracer.trace()).toEqual([ + 'globalSetup', + 'beforeAll', + 'beforeEach', + 'test1', + 'afterEach', + 'afterAll', + 'globalTeardown', + ]); + }); + it('should run globalSetup and globalTeardown in proper order if parallel', async() => { + const t = new Runner({timeout: 10000, parallel: 2}); + const tracer = new TestTracer(t); + tracer.traceAllHooks('', async (hookName) => { + // slowdown globalsetup to see the rest hooks awaiting this one + if (hookName === 'globalSetup') + await new Promise(x => setTimeout(x, 50)); + }); + tracer.addTest('', 'test1'); + tracer.addTest('', 'test2'); + await t.run(); + + expect(tracer.trace()).toEqual([ + '<_global_> globalSetup', + ' beforeAll', + ' beforeAll', + ' beforeEach', + ' beforeEach', + ' test1', + ' test2', + ' afterEach', + ' afterEach', + ' afterAll', + ' afterAll', + '<_global_> globalTeardown', + ]); + }); + it('should support globalSetup/globalTeardown in nested suites', async() => { + const t = new Runner({timeout: 10000, parallel: 2}); + const tracer = new TestTracer(t); + tracer.traceAllHooks(''); + t.describe('suite', () => { + tracer.traceAllHooks(' '); + tracer.addTest(' ', 'test1'); + tracer.addTest(' ', 'test2'); + tracer.addTest(' ', 'test3'); + }); + await t.run(); + + expect(tracer.trace()).toEqual([ + '<_global_> globalSetup', + ' beforeAll', + ' beforeAll', + ' <_global_> globalSetup', + ' beforeAll', + ' beforeAll', + ' beforeEach', + ' beforeEach', + ' beforeEach', + ' beforeEach', + ' test1', + ' test2', + ' afterEach', + ' afterEach', + ' afterEach', + ' afterEach', + ' afterAll', + ' beforeEach', + ' afterAll', + ' beforeEach', + ' test3', + ' afterEach', + ' afterEach', + ' afterAll', + ' <_global_> globalTeardown', + ' afterAll', + '<_global_> globalTeardown', + ]); + }); + it('should report as crashed when global hook crashes', async() => { + const t = new Runner({timeout: 10000}); + t.globalSetup(() => { throw new Error('crash!'); }); + t.it('uno', () => { }); + const result = await t.run(); + expect(result.result).toBe('crashed'); + }); + it('should terminate and unwind hooks if globalSetup fails', async() => { + const t = new Runner({timeout: 10000}); + const tracer = new TestTracer(t); + tracer.traceAllHooks(); + t.describe('suite', () => { + tracer.traceAllHooks(' ', (hookName) => { + if (hookName === 'globalSetup') { + tracer.log(' !! CRASH !!'); + throw new Error('crash'); + } + }); + tracer.addTest(' ', 'test1'); + }); + await t.run(); + expect(tracer.trace()).toEqual([ + 'globalSetup', + 'beforeAll', + ' globalSetup', + ' !! CRASH !!', + ' afterAll', + ' globalTeardown', + 'afterAll', + 'globalTeardown', + ]); + }); + it('should not run globalSetup / globalTeardown if all tests are skipped', async() => { + const t = new Runner({timeout: 10000}); + const tracer = new TestTracer(t); + tracer.traceAllHooks(); + t.describe('suite', () => { + tracer.addSkippedTest(' ', 'test1'); + }); + await t.run(); + expect(tracer.trace()).toEqual([ + ]); + }); + it('should properly run globalTeardown if some tests are not run', async() => { + const t = new Runner({timeout: 10000}); + const tracer = new TestTracer(t); + tracer.traceAllHooks(); + t.describe('suite', () => { + tracer.addSkippedTest(' ', 'test1'); + tracer.addFailingTest(' ', 'test2'); + tracer.addTest(' ', 'test3'); + }); + await t.run(); + expect(tracer.trace()).toEqual([ + 'globalSetup', + 'beforeAll', + 'beforeEach', + ' test3', + 'afterEach', + 'afterAll', + 'globalTeardown', + ]); + }); + }); + + describe('TestRunner.run', () => { + it('should run a test', async() => { + const t = new Runner(); + let ran = false; + t.it('uno', () => ran = true); + await t.run(); + expect(ran).toBe(true); + }); + it('should handle repeat', async() => { + const t = new Runner(); + let suite = 0; + let test = 0; + let beforeAll = 0; + let beforeEach = 0; + t.describe.repeat(2)('suite', () => { + suite++; + t.beforeAll(() => beforeAll++); + t.beforeEach(() => beforeEach++); + t.it.repeat(3)('uno', () => test++); + }); + await t.run(); + expect(suite).toBe(1); + expect(beforeAll).toBe(1); + expect(beforeEach).toBe(6); + expect(test).toBe(6); + }); + it('should repeat without breaking test order', async() => { + const t = new Runner(); + const log = []; + t.describe.repeat(2)('suite', () => { + t.it('uno', () => log.push(1)); + t.it.repeat(2)('dos', () => log.push(2)); + }); + t.it('tres', () => log.push(3)); + await t.run(); + expect(log.join()).toBe('1,2,2,1,2,2,3'); + }); + it('should run tests if some fail', async() => { + const t = new Runner(); + const log = []; + t.it('uno', () => log.push(1)); + t.it('dos', () => { throw new Error('bad'); }); + t.it('tres', () => log.push(3)); + await t.run(); + expect(log.join()).toBe('1,3'); + }); + it('should run tests if some timeout', async() => { + const t = new Runner({timeout: 1}); + const log = []; + t.it('uno', () => log.push(1)); + t.it('dos', async() => new Promise(() => {})); + t.it('tres', () => log.push(3)); + await t.run(); + expect(log.join()).toBe('1,3'); + }); + it('should break on first failure if configured so', async() => { + const log = []; + const t = new Runner({breakOnFailure: true}); + t.it('test#1', () => log.push('test#1')); + t.it('test#2', () => log.push('test#2')); + t.it('test#3', () => { throw new Error('crash'); }); + t.it('test#4', () => log.push('test#4')); + await t.run(); + expect(log).toEqual([ + 'test#1', + 'test#2', + ]); + }); + it('should pass a state and a test as a test parameters', async() => { + const log = []; + const t = new Runner(); + t.beforeEach(state => state.FOO = 42); + t.it('uno', (state, testRun) => { + log.push('state.FOO=' + state.FOO); + log.push('test=' + testRun.test().name()); + }); + await t.run(); + expect(log.join()).toBe('state.FOO=42,test=uno'); + }); + it('should run async test', async() => { + const t = new Runner(); + let ran = false; + t.it('uno', async() => { + await new Promise(x => setTimeout(x, 10)); + ran = true; + }); + await t.run(); + expect(ran).toBe(true); + }); + it('should run async tests in order of their declaration', async() => { + const log = []; + const t = new Runner(); + t.it('uno', async() => { + await new Promise(x => setTimeout(x, 30)); + log.push(1); + }); + t.it('dos', async() => { + await new Promise(x => setTimeout(x, 20)); + log.push(2); + }); + t.it('tres', async() => { + await new Promise(x => setTimeout(x, 10)); + log.push(3); + }); + await t.run(); + expect(log.join()).toBe('1,2,3'); + }); + it('should run multiple tests', async() => { + const log = []; + const t = new Runner(); + t.it('uno', () => log.push(1)); + t.it('dos', () => log.push(2)); + await t.run(); + expect(log.join()).toBe('1,2'); + }); + it('should NOT run a skipped test', async() => { + const t = new Runner(); + let ran = false; + t.xit('uno', () => ran = true); + await t.run(); + expect(ran).toBe(false); + }); + it('should run ONLY non-skipped tests', async() => { + const log = []; + const t = new Runner(); + t.it('uno', () => log.push(1)); + t.xit('dos', () => log.push(2)); + t.it('tres', () => log.push(3)); + await t.run(); + expect(log.join()).toBe('1,3'); + }); + it('should run ONLY focused tests', async() => { + const log = []; + const t = new Runner(); + t.it('uno', () => log.push(1)); + t.xit('dos', () => log.push(2)); + t.fit('tres', () => log.push(3)); + await t.run(); + expect(log.join()).toBe('3'); + }); + it('should run tests in order of their declaration', async() => { + const log = []; + const t = new Runner(); + t.it('uno', () => log.push(1)); + t.describe('suite1', () => { + t.it('dos', () => log.push(2)); + t.it('tres', () => log.push(3)); + }); + t.it('cuatro', () => log.push(4)); + await t.run(); + expect(log.join()).toBe('1,2,3,4'); + }); + it('should respect total timeout', async() => { + const t = new Runner({timeout: 10000, totalTimeout: 1}); + t.it('uno', async () => { await new Promise(() => {}); }); + const result = await t.run(); + expect(result.runs[0].result()).toBe('terminated'); + expect(result.message).toContain('Total timeout'); + }); + }); + + describe('TestRunner.run result', () => { + it('should return OK if all tests pass', async() => { + const t = new Runner(); + t.it('uno', () => {}); + const result = await t.run(); + expect(result.result).toBe('ok'); + }); + it('should return FAIL if at least one test fails', async() => { + const t = new Runner(); + t.it('uno', () => { throw new Error('woof'); }); + const result = await t.run(); + expect(result.result).toBe('failed'); + }); + it('should return FAIL if at least one test times out', async() => { + const t = new Runner({timeout: 1}); + t.it('uno', async() => new Promise(() => {})); + const result = await t.run(); + expect(result.result).toBe('failed'); + }); + it('should return TERMINATED if it was terminated', async() => { + const t = new Runner({timeout: 1000000}); + t.it('uno', async() => new Promise(() => {})); + const [result] = await Promise.all([ + t.run(), + t.terminate(), + ]); + expect(result.result).toBe('terminated'); + }); + it('should return CRASHED if it crashed', async() => { + const t = new Runner({timeout: 1}); + t.it('uno', async() => new Promise(() => {})); + t.afterAll(() => { throw new Error('woof');}); + const result = await t.run(); + expect(result.result).toBe('crashed'); + }); + }); + + describe('TestRunner parallel', () => { + it('should run tests in parallel', async() => { + const log = []; + const t = new Runner({parallel: 2}); + t.it('uno', async state => { + log.push(`Worker #${state.parallelIndex} Starting: UNO`); + await Promise.resolve(); + log.push(`Worker #${state.parallelIndex} Ending: UNO`); + }); + t.it('dos', async state => { + log.push(`Worker #${state.parallelIndex} Starting: DOS`); + await Promise.resolve(); + log.push(`Worker #${state.parallelIndex} Ending: DOS`); + }); + await t.run(); + expect(log).toEqual([ + 'Worker #0 Starting: UNO', + 'Worker #1 Starting: DOS', + 'Worker #0 Ending: UNO', + 'Worker #1 Ending: DOS', + ]); + }); + }); + + describe('TestRunner.hasFocusedTestsOrSuitesOrFiles', () => { + it('should work', () => { + const t = new Runner(); + t.it('uno', () => {}); + expect(t._filter.hasFocusedTestsOrSuitesOrFiles()).toBe(false); + }); + it('should work #2', () => { + const t = new Runner(); + t.fit('uno', () => {}); + expect(t._filter.hasFocusedTestsOrSuitesOrFiles()).toBe(true); + }); + it('should work #3', () => { + const t = new Runner(); + t.describe('suite #1', () => { + t.fdescribe('suite #2', () => { + t.describe('suite #3', () => { + t.it('uno', () => {}); + }); + }); + }); + expect(t._filter.hasFocusedTestsOrSuitesOrFiles()).toBe(true); + }); + }); + + describe('TestRunner result', () => { + it('should work for both throwing and timeouting tests', async() => { + const t = new Runner({timeout: 1}); + t.it('uno', () => { throw new Error('boo');}); + t.it('dos', () => new Promise(() => {})); + const result = await t.run(); + expect(result.runs[0].result()).toBe('failed'); + expect(result.runs[1].result()).toBe('timedout'); + }); + it('should report crashed tests', async() => { + const t = new Runner(); + t.beforeEach(() => { throw new Error('woof');}); + t.it('uno', () => {}); + const result = await t.run(); + expect(result.runs[0].result()).toBe('crashed'); + }); + it('skipped should work for both throwing and timeouting tests', async() => { + const t = new Runner({timeout: 1}); + t.xit('uno', () => { throw new Error('boo');}); + const result = await t.run(); + expect(result.runs[0].result()).toBe('skipped'); + }); + it('should return OK', async() => { + const t = new Runner(); + t.it('uno', () => {}); + const result = await t.run(); + expect(result.runs[0].result()).toBe('ok'); + }); + it('should return TIMEDOUT', async() => { + const t = new Runner({timeout: 1}); + t.it('uno', async() => new Promise(() => {})); + const result = await t.run(); + expect(result.runs[0].result()).toBe('timedout'); + }); + it('should return SKIPPED', async() => { + const t = new Runner(); + t.xit('uno', () => {}); + const result = await t.run(); + expect(result.runs[0].result()).toBe('skipped'); + }); + it('should return FAILED', async() => { + const t = new Runner(); + t.it('uno', async() => Promise.reject('woof')); + const result = await t.run(); + expect(result.runs[0].result()).toBe('failed'); + }); + it('should return TERMINATED', async() => { + const t = new Runner(); + t.it('uno', async() => t.terminate()); + const result = await t.run(); + expect(result.runs[0].result()).toBe('terminated'); + }); + it('should return CRASHED', async() => { + const t = new Runner(); + t.it('uno', () => {}); + t.afterEach(() => {throw new Error('foo');}); + const result = await t.run(); + expect(result.runs[0].result()).toBe('crashed'); + }); + }); + + describe('TestRunner delegate', () => { + it('should call delegate methods in proper order', async() => { + const log = []; + const t = new Runner({ + onStarted: () => log.push('E:started'), + onTestRunStarted: () => log.push('E:teststarted'), + onTestRunFinished: () => log.push('E:testfinished'), + onFinished: () => log.push('E:finished'), + }); + t.beforeAll(() => log.push('beforeAll')); + t.beforeEach(() => log.push('beforeEach')); + t.it('test#1', () => log.push('test#1')); + t.afterEach(() => log.push('afterEach')); + t.afterAll(() => log.push('afterAll')); + await t.run(); + expect(log).toEqual([ + 'E:started', + 'beforeAll', + 'E:teststarted', + 'beforeEach', + 'test#1', + 'afterEach', + 'E:testfinished', + 'afterAll', + 'E:finished', + ]); + }); + it('should call onFinished with result', async() => { + let onFinished; + const finishedPromise = new Promise(f => onFinished = f); + const [result] = await Promise.all([ + finishedPromise, + new TestRunner().run([], { onFinished }), + ]); + expect(result.result).toBe('ok'); + }); + it('should crash when onStarted throws', async() => { + const t = new Runner({ + onStarted: () => { throw 42; }, + }); + const result = await t.run(); + expect(result.ok()).toBe(false); + expect(result.message).toBe('INTERNAL ERROR: 42'); + }); + it('should crash when onFinished throws', async() => { + const t = new Runner({ + onFinished: () => { throw new Error('42'); }, + }); + const result = await t.run(); + expect(result.ok()).toBe(false); + expect(result.message).toBe('INTERNAL ERROR'); + expect(result.result).toBe('crashed'); + }); + }); +}; + +class TestTracer { + constructor(testRunner) { + this._testRunner = testRunner; + this._trace = []; + } + + addSkippedTest(prefix, testName, callback) { + this._testRunner.it.skip(testName, async(...args) => { + if (callback) + await callback(...args); + this._trace.push(prefix + this._workerPrefix(args[0]) + testName); + }); + } + + addFailingTest(prefix, testName, callback) { + this._testRunner.it.fail(testName, async(...args) => { + if (callback) + await callback(...args); + this._trace.push(prefix + this._workerPrefix(args[0]) + testName); + }); + } + + addTest(prefix, testName, callback) { + this._testRunner.it(testName, async(...args) => { + if (callback) + await callback(...args); + this._trace.push(prefix + this._workerPrefix(args[0]) + testName); + }); + } + + traceHooks(hookNames, prefix = '', callback) { + for (const hookName of hookNames) { + this._testRunner[hookName].call(this._testRunner, async (state) => { + this._trace.push(prefix + this._workerPrefix(state) + hookName); + if (callback) + await callback(hookName); + }); + } + } + + _workerPrefix(state) { + if (this._testRunner.parallel() === 1) + return ''; + return state && (typeof state.parallelIndex !== 'undefined') ? ` ` : `<_global_> `; + + } + + traceAllHooks(prefix = '', callback) { + this.traceHooks(['globalSetup', 'globalTeardown', 'beforeAll', 'afterAll', 'beforeEach', 'afterEach'], prefix, callback); + } + + log(text) { + this._trace.push(text); + } + + trace() { + return this._trace; + } +} +