Skip to content

Commit

Permalink
chore(test-runner): move into its own folder and typescript project (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
JoelEinbinder authored Aug 20, 2020
1 parent 4c56354 commit 012f942
Show file tree
Hide file tree
Showing 34 changed files with 262 additions and 171 deletions.
1 change: 1 addition & 0 deletions .eslintignore
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,4 @@ src/webkit/protocol.ts
utils/generate_types/overrides.d.ts
utils/generate_types/test/test.ts
test/
test-runner/
2 changes: 1 addition & 1 deletion .github/workflows/auto_roll.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
env:
BROWSER: ${{ matrix.browser }}
DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000 && npm run coverage"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 && npm run coverage"
env:
BROWSER: ${{ matrix.browser }}
DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
Expand Down Expand Up @@ -68,7 +68,7 @@ jobs:
- uses: microsoft/playwright-github-action@v1
- run: npm ci
- run: npm run build
- run: node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000
- run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000
env:
BROWSER: ${{ matrix.browser }}
DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
Expand Down Expand Up @@ -102,7 +102,7 @@ jobs:
- uses: microsoft/playwright-github-action@v1
- run: npm ci
- run: npm run build
- run: node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000
- run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000
shell: bash
env:
BROWSER: ${{ matrix.browser }}
Expand Down Expand Up @@ -159,7 +159,7 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
if: ${{ always() }}
env:
BROWSER: ${{ matrix.browser }}
Expand Down Expand Up @@ -197,7 +197,7 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
env:
BROWSER: ${{ matrix.browser }}
DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
Expand Down
15 changes: 8 additions & 7 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,19 @@
"node": ">=10.15.0"
},
"scripts": {
"ctest": "cross-env BROWSER=chromium node test/runner/cli test/",
"ftest": "cross-env BROWSER=firefox node test/runner/cli test/",
"wtest": "cross-env BROWSER=webkit node test/runner/cli test/",
"test": "node test/runner/cli test/",
"ctest": "cross-env BROWSER=chromium node test-runner/cli test/",
"ftest": "cross-env BROWSER=firefox node test-runner/cli test/",
"wtest": "cross-env BROWSER=webkit node test-runner/cli test/",
"test": "node test-runner/cli test/",
"eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src",
"tsc": "tsc -p .",
"tsc-installer": "tsc -p ./src/install/tsconfig.json",
"doc": "node utils/doclint/cli.js",
"test-infra": "node test/runner/cli utils/doclint/check_public_api/test/test.js && node test/runner/cli utils/doclint/preprocessor/test.js",
"test-infra": "node test-runner/cli utils/doclint/check_public_api/test/test.js && node test-runner/cli utils/doclint/preprocessor/test.js",
"lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && npm run generate-channels && npm run test-types && npm run test-infra",
"clean": "rimraf lib && rimraf types",
"prepare": "node install-from-github.js",
"build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run generate-types",
"build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run build-testrunner && npm run generate-types",
"watch": "node utils/watch.js",
"test-types": "npm run generate-types && npx -p typescript@3.7.5 tsc -p utils/generate_types/test/tsconfig.json && npm run typecheck-tests",
"generate-types": "node utils/generate_types/",
Expand All @@ -30,7 +30,8 @@
"roll-browser": "node utils/roll_browser.js",
"coverage": "node test/checkCoverage.js",
"check-deps": "node utils/check_deps.js",
"build-driver": "pkg --public --targets node12-linux-x64,node12-macos-x64,node12-win-x64 --out-path=drivers packages/playwright-driver/main.js"
"build-driver": "pkg --public --targets node12-linux-x64,node12-macos-x64,node12-win-x64 --out-path=drivers packages/playwright-driver/main.js",
"build-testrunner": "tsc -p test-runner"
},
"author": {
"name": "Microsoft Corporation"
Expand Down
2 changes: 2 additions & 0 deletions test-runner/cli.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#!/usr/bin/env node
module.exports = require('./lib/cli')
8 changes: 8 additions & 0 deletions test-runner/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"name": "test-runner",
"version": "0.0.7",
"bin": {
"test-runner": "./cli.js"
},
"main": "./lib/index.js"
}
8 changes: 4 additions & 4 deletions test/runner/GoldenUtils.js → test-runner/src/GoldenUtils.js
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ const GoldenComparators = {
* @param {?Object} actualBuffer
* @param {!Buffer} expectedBuffer
* @param {!string} mimeType
* @return {?{diff: (!Object:undefined), errorMessage: (string|undefined)}}
* @return {?{diff?: Object, errorMessage?: string}}
*/
function compareImages(actualBuffer, expectedBuffer, mimeType, config = {}) {
if (!actualBuffer || !(actualBuffer instanceof Buffer))
Expand All @@ -63,7 +63,7 @@ function compareImages(actualBuffer, expectedBuffer, mimeType, config = {}) {
/**
* @param {?Object} actual
* @param {!Buffer} expectedBuffer
* @return {?{diff: (!Object:undefined), errorMessage: (string|undefined)}}
* @return {?{diff?: Object, errorMessage?: string, diffExtension?: string}}
*/
function compareText(actual, expectedBuffer) {
if (typeof actual !== 'string')
Expand All @@ -85,8 +85,8 @@ function compareText(actual, expectedBuffer) {

/**
* @param {?Object} actual
* @param {string} path
* @return {!{pass: boolean, message: (undefined|string)}}
* @param {string} name
* @return {!{pass: boolean, message?: string}}
*/
function compare(actual, name, options) {
const { relativeTestFile, snapshotDir, outputDir, updateSnapshots } = options;
Expand Down
72 changes: 72 additions & 0 deletions test-runner/src/builtin.fixtures.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import os from 'os';
import path from 'path';
import {promisify} from 'util';
import fs from 'fs';
import rimraf from 'rimraf';
import {registerFixture} from './fixtures';


declare global {
type DescribeFunction = ((name: string, inner: () => void) => void) & {
fail(condition: boolean): DescribeFunction;
skip(condition: boolean): DescribeFunction;
slow(): DescribeFunction;
repeat(n: number): DescribeFunction;
};

type ItFunction<STATE> = ((name: string, inner: (state: STATE) => Promise<void>) => void) & {
fail(condition: boolean): ItFunction<STATE>;
skip(condition: boolean): ItFunction<STATE>;
slow(): ItFunction<STATE>;
repeat(n: number): ItFunction<STATE>;
};

const describe: DescribeFunction;
const fdescribe: DescribeFunction;
const xdescribe: DescribeFunction;
const it: ItFunction<TestState & WorkerState & FixtureParameters>;
const fit: ItFunction<TestState & WorkerState & FixtureParameters>;
const dit: ItFunction<TestState & WorkerState & FixtureParameters>;
const xit: ItFunction<TestState & WorkerState & FixtureParameters>;

const beforeEach: (inner: (state: TestState & WorkerState & FixtureParameters) => Promise<void>) => void;
const afterEach: (inner: (state: TestState & WorkerState & FixtureParameters) => Promise<void>) => void;
const beforeAll: (inner: (state: WorkerState & FixtureParameters) => Promise<void>) => void;
const afterAll: (inner: (state: WorkerState & FixtureParameters) => Promise<void>) => void;
}

const mkdtempAsync = promisify(fs.mkdtemp);
const removeFolderAsync = promisify(rimraf);

declare global {
interface FixtureParameters {
parallelIndex: number;
}
interface TestState {
tmpDir: string;
}
}

export {parameters, registerFixture, registerWorkerFixture, registerParameter} from './fixtures';

registerFixture('tmpDir', async ({}, test) => {
const tmpDir = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-'));
await test(tmpDir);
await removeFolderAsync(tmpDir).catch(e => {});
});
14 changes: 7 additions & 7 deletions test/runner/cli.js → test-runner/src/cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,21 @@ let beforeFunction;
let afterFunction;
let matrix = {};

global.before = (fn => beforeFunction = fn);
global.after = (fn => afterFunction = fn);
global.matrix = (m => matrix = m);
global['before'] = (fn => beforeFunction = fn);
global['after'] = (fn => afterFunction = fn);
global['matrix'] = (m => matrix = m);

program
.version('Version ' + require('../../package.json').version)
.version('Version ' + /** @type {any} */ (require)('../package.json').version)
.option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
.option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
.option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2))
.option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2).toString())
.option('--reporter <reporter>', 'Specify reporter to use', '')
.option('--trial-run', 'Only collect the matching tests and report them as passing')
.option('--quiet', 'Suppress stdio', false)
.option('--debug', 'Run tests in-process for debugging', false)
.option('--output <outputDir>', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results'))
.option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', 10000)
.option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
.option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
.action(async (command) => {
// Collect files]
Expand Down Expand Up @@ -101,7 +101,7 @@ program
try {
if (beforeFunction)
await beforeFunction();
await runner.run(files);
await runner.run();
await runner.stop();
} finally {
if (afterFunction)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class DotReporter extends Base {
this.failures.forEach((failure, index) => {
const relativePath = path.relative(process.cwd(), failure.file);
const header = ` ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)}${failure.title}`;
console.log(colors.bold.red(header));
console.log(colors.bold(colors.red(header)));
const stack = failure.err.stack;
if (stack) {
console.log('');
Expand Down
Loading

0 comments on commit 012f942

Please sign in to comment.