From f5acaa81058cc2b654ba91827f8a0d5f08642764 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 28 Aug 2020 16:24:45 -0700 Subject: [PATCH] fix(testrunner): include fixture teardown into timeout, add global timeout --- .github/workflows/auto_roll.yml | 2 +- .github/workflows/tests.yml | 12 +++--- test-runner/src/cli.ts | 2 + test-runner/src/fixtures.ts | 41 ++++++++++++-------- test-runner/src/index.ts | 10 +++++ test-runner/src/reporter.ts | 7 ++-- test-runner/src/reporters/base.ts | 28 +++++++++++--- test-runner/src/reporters/dot.ts | 5 +++ test-runner/src/reporters/json.ts | 5 +++ test-runner/src/reporters/multiplexer.ts | 5 +++ test-runner/src/runnerConfig.ts | 11 +++--- test-runner/src/test.ts | 6 +-- test-runner/src/testRunner.ts | 4 +- test-runner/src/util.ts | 45 ++++++++++++++++++++++ test-runner/test/assets/fixture-timeout.js | 30 +++++++++++++++ test-runner/test/exit-code.spec.ts | 18 ++++++++- test/defaultbrowsercontext-2.spec.ts | 6 ++- test/frame-evaluate.spec.ts | 3 +- test/page-evaluate.spec.ts | 4 +- test/page-event-pageerror.spec.ts | 4 +- test/permissions.spec.ts | 2 +- test/proxy.spec.ts | 6 ++- test/screencast.spec.ts | 4 +- 23 files changed, 206 insertions(+), 54 deletions(-) create mode 100644 test-runner/src/util.ts create mode 100644 test-runner/test/assets/fixture-timeout.js diff --git a/.github/workflows/auto_roll.yml b/.github/workflows/auto_roll.yml index fdce9a1031949..021da9758db3f 100644 --- a/.github/workflows/auto_roll.yml +++ b/.github/workflows/auto_roll.yml @@ -27,7 +27,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --retries=3 --timeout=30000 --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --retries=3 --timeout=30000 --global-timeout=5400000 --reporter=dot,json" env: BROWSER: ${{ matrix.browser }} DEBUG: "pw:*,-pw:wrapped*,-pw:test*" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 80129485bba50..81b4c2b561e1f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,7 +37,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json && npm run coverage" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json && npm run coverage" env: BROWSER: ${{ matrix.browser }} PWRUNNER_JSON_REPORT: "test-results/report.json" @@ -62,7 +62,7 @@ jobs: - uses: microsoft/playwright-github-action@v1 - run: npm ci - run: npm run build - - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json + - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json env: BROWSER: ${{ matrix.browser }} PWRUNNER_JSON_REPORT: "test-results/report.json" @@ -90,7 +90,7 @@ jobs: - uses: microsoft/playwright-github-action@v1 - run: npm ci - run: npm run build - - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json + - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json shell: bash env: BROWSER: ${{ matrix.browser }} @@ -141,7 +141,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json" if: ${{ always() }} env: BROWSER: ${{ matrix.browser }} @@ -174,7 +174,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json" env: BROWSER: ${{ matrix.browser }} PWWIRE: true @@ -206,7 +206,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json" env: BROWSER: ${{ matrix.browser }} TRACING: true diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts index 6f102ea9507d5..c07e089bfbe0d 100644 --- a/test-runner/src/cli.ts +++ b/test-runner/src/cli.ts @@ -36,6 +36,7 @@ program .version('Version ' + /** @type {any} */ (require)('../package.json').version) .option('--forbid-only', 'Fail if exclusive test(s) encountered', false) .option('-g, --grep ', 'Only run tests matching this string or regexp', '.*') + .option('--global-timeout ', 'Specify maximum time this test suite can run (in milliseconds), default: 0 for unlimited', '0') .option('-j, --jobs ', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', String(Math.ceil(require('os').cpus().length / 2))) .option('--reporter ', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot') .option('--repeat-each ', 'Specify how many times to run the tests', '1') @@ -60,6 +61,7 @@ program snapshotDir: path.join(testDir, '__snapshots__'), testDir, timeout: parseInt(command.timeout, 10), + globalTimeout: parseInt(command.globalTimeout, 10), trialRun: command.trialRun, updateSnapshots: command.updateSnapshots }; diff --git a/test-runner/src/fixtures.ts b/test-runner/src/fixtures.ts index decc68fccf24f..52918d09d077b 100644 --- a/test-runner/src/fixtures.ts +++ b/test-runner/src/fixtures.ts @@ -17,6 +17,7 @@ import debug from 'debug'; import { RunnerConfig } from './runnerConfig'; import { serializeError, Test, TestResult } from './test'; +import { raceAgainstTimeout } from './util'; type Scope = 'test' | 'worker'; @@ -152,24 +153,32 @@ export class FixturePool { return fn(params); } - async runTestWithFixtures(fn: Function, timeout: number, info: TestInfo) { - let timer: NodeJS.Timer; - const timerPromise = new Promise(f => timer = setTimeout(f, timeout)); + async runTestWithFixturesAndTimeout(fn: Function, timeout: number, info: TestInfo) { + const { timedOut } = await raceAgainstTimeout(this._runTestWithFixtures(fn, info), timeout); + // Do not overwrite test failure upon timeout in fixture. + if (timedOut && info.result.status === 'passed') + info.result.status = 'timedOut'; + } + + async _runTestWithFixtures(fn: Function, info: TestInfo) { + try { + await this.resolveParametersAndRun(fn, info.config, info); + info.result.status = 'passed'; + } catch (error) { + // Prefer original error to the fixture teardown error or timeout. + if (info.result.status === 'passed') { + info.result.status = 'failed'; + info.result.error = serializeError(error); + } + } try { - await Promise.race([ - this.resolveParametersAndRun(fn, info.config, info).then(() => { - info.result.status = 'passed'; - clearTimeout(timer); - }).catch(e => { - info.result.status = 'failed'; - info.result.error = serializeError(e); - }), - timerPromise.then(() => { - info.result.status = 'timedOut'; - }) - ]); - } finally { await this.teardownScope('test'); + } catch (error) { + // Prefer original error to the fixture teardown error or timeout. + if (info.result.status === 'passed') { + info.result.status = 'failed'; + info.result.error = serializeError(error); + } } } } diff --git a/test-runner/src/index.ts b/test-runner/src/index.ts index fc8aa45929fb4..b318e0942e483 100644 --- a/test-runner/src/index.ts +++ b/test-runner/src/index.ts @@ -25,8 +25,10 @@ import { registerFixture as registerFixtureT, registerWorkerFixture as registerW import { Reporter } from './reporter'; import { Runner } from './runner'; import { RunnerConfig } from './runnerConfig'; +import { Suite } from './test'; import { Matrix, TestCollector } from './testCollector'; import { installTransform } from './transform'; +import { raceAgainstTimeout } from './util'; export { parameters, registerParameter } from './fixtures'; export { Reporter } from './reporter'; export { RunnerConfig } from './runnerConfig'; @@ -93,7 +95,15 @@ export async function run(config: RunnerConfig, files: string[], reporter: Repor const total = suite.total(); if (!total) return 'no-tests'; + const { result, timedOut } = await raceAgainstTimeout(runTests(config, suite, reporter), config.globalTimeout); + if (timedOut) { + reporter.onTimeout(config.globalTimeout); + process.exit(1); + } + return result; +} +async function runTests(config: RunnerConfig, suite: Suite, reporter: Reporter) { // Trial run does not need many workers, use one. const jobs = (config.trialRun || config.debug) ? 1 : config.jobs; const runner = new Runner(suite, { ...config, jobs }, reporter); diff --git a/test-runner/src/reporter.ts b/test-runner/src/reporter.ts index 36106fe24cd69..4e5fe34919c83 100644 --- a/test-runner/src/reporter.ts +++ b/test-runner/src/reporter.ts @@ -20,8 +20,9 @@ import { Suite, Test, TestResult } from './test'; export interface Reporter { onBegin(config: RunnerConfig, suite: Suite): void; onTestBegin(test: Test): void; - onTestStdOut(test: Test, chunk: string | Buffer); - onTestStdErr(test: Test, chunk: string | Buffer); - onTestEnd(test: Test, result: TestResult); + onTestStdOut(test: Test, chunk: string | Buffer): void; + onTestStdErr(test: Test, chunk: string | Buffer): void; + onTestEnd(test: Test, result: TestResult): void; + onTimeout(timeout: number): void; onEnd(): void; } diff --git a/test-runner/src/reporters/base.ts b/test-runner/src/reporters/base.ts index 84d75db1ef278..9f37a496a6bd6 100644 --- a/test-runner/src/reporters/base.ts +++ b/test-runner/src/reporters/base.ts @@ -32,11 +32,13 @@ export class BaseReporter implements Reporter { skipped: Test[] = []; asExpected: Test[] = []; unexpected = new Set(); - flaky: Test[] = []; + expectedFlaky: Test[] = []; + unexpectedFlaky: Test[] = []; duration = 0; startTime: number; config: RunnerConfig; suite: Suite; + timeout: number; constructor() { process.on('SIGINT', async () => { @@ -76,7 +78,10 @@ export class BaseReporter implements Reporter { this.asExpected.push(test); } else { // as expected after unexpected -> flaky. - this.flaky.push(test); + if (test.isFlaky()) + this.expectedFlaky.push(test); + else + this.unexpectedFlaky.push(test); } return; } @@ -86,6 +91,10 @@ export class BaseReporter implements Reporter { } } + onTimeout(timeout: number) { + this.timeout = timeout; + } + onEnd() { this.duration = Date.now() - this.startTime; } @@ -105,10 +114,13 @@ export class BaseReporter implements Reporter { this._printFailures(filteredUnexpected); } - if (this.flaky.length) { - console.log(colors.red(` ${this.flaky.length} flaky`)); - console.log(''); - this._printFailures(this.flaky); + const allFlaky = this.expectedFlaky.length + this.unexpectedFlaky.length; + if (allFlaky) { + console.log(colors.red(` ${allFlaky} flaky`)); + if (this.unexpectedFlaky.length) { + console.log(''); + this._printFailures(this.unexpectedFlaky); + } } const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut')); @@ -118,6 +130,10 @@ export class BaseReporter implements Reporter { this._printFailures(timedOut); } console.log(''); + if (this.timeout) { + console.log(colors.red(` Timed out waiting ${this.timeout / 1000}s for the entire test run`)); + console.log(''); + } } private _printFailures(failures: Test[]) { diff --git a/test-runner/src/reporters/dot.ts b/test-runner/src/reporters/dot.ts index b065ec1e1ed07..36eb25d08abf4 100644 --- a/test-runner/src/reporters/dot.ts +++ b/test-runner/src/reporters/dot.ts @@ -29,6 +29,11 @@ class DotReporter extends BaseReporter { } } + onTimeout(timeout) { + super.onTimeout(timeout); + this.onEnd(); + } + onEnd() { super.onEnd(); process.stdout.write('\n'); diff --git a/test-runner/src/reporters/json.ts b/test-runner/src/reporters/json.ts index 4745351b2b1a4..e6fcc7e809dee 100644 --- a/test-runner/src/reporters/json.ts +++ b/test-runner/src/reporters/json.ts @@ -19,6 +19,11 @@ import { Suite, Test, TestResult } from '../test'; import * as fs from 'fs'; class JSONReporter extends BaseReporter { + onTimeout(timeout) { + super.onTimeout(timeout); + this.onEnd(); + } + onEnd() { super.onEnd(); const result = { diff --git a/test-runner/src/reporters/multiplexer.ts b/test-runner/src/reporters/multiplexer.ts index 6d068fb7a569d..0b939b68adbfc 100644 --- a/test-runner/src/reporters/multiplexer.ts +++ b/test-runner/src/reporters/multiplexer.ts @@ -50,6 +50,11 @@ export class Multiplexer implements Reporter { reporter.onTestEnd(test, result); } + onTimeout(timeout: number) { + for (const reporter of this._reporters) + reporter.onTimeout(timeout); + } + onEnd() { for (const reporter of this._reporters) reporter.onEnd(); diff --git a/test-runner/src/runnerConfig.ts b/test-runner/src/runnerConfig.ts index 6d585067e0fdc..fab686f2f7cf5 100644 --- a/test-runner/src/runnerConfig.ts +++ b/test-runner/src/runnerConfig.ts @@ -15,17 +15,18 @@ */ export type RunnerConfig = { + debug?: boolean; forbidOnly?: boolean; + globalTimeout: number; + grep?: string; jobs: number; outputDir: string; - snapshotDir: string; - testDir: string; - timeout: number; - debug?: boolean; quiet?: boolean; - grep?: string; repeatEach: number; retries: number, + snapshotDir: string; + testDir: string; + timeout: number; trialRun?: boolean; updateSnapshots?: boolean; }; diff --git a/test-runner/src/test.ts b/test-runner/src/test.ts index 6a033586199a1..9e58eb0dc2d86 100644 --- a/test-runner/src/test.ts +++ b/test-runner/src/test.ts @@ -103,8 +103,8 @@ export class Runnable { return this._slow || (this.parent && this.parent._isSlow()); } - _isFlaky(): boolean { - return this._flaky || (this.parent && this.parent._isFlaky()); + isFlaky(): boolean { + return this._flaky || (this.parent && this.parent.isFlaky()); } titlePath(): string[] { @@ -162,7 +162,7 @@ export class Test extends Runnable { const hasFailedResults = !!this.results.find(r => r.status !== r.expectedStatus); if (!hasFailedResults) return true; - if (!this._isFlaky()) + if (!this.isFlaky()) return false; const hasPassedResults = !!this.results.find(r => r.status === r.expectedStatus); return hasPassedResults; diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts index 61ab2034da78a..c8247d203831e 100644 --- a/test-runner/src/testRunner.ts +++ b/test-runner/src/testRunner.ts @@ -158,7 +158,7 @@ export class TestRunner extends EventEmitter { // We only know resolved skipped/flaky value in the worker, // send it to the runner. test._skipped = test._isSkipped(); - test._flaky = test._isFlaky(); + test._flaky = test.isFlaky(); test._slow = test._isSlow(); this.emit('testBegin', { id, @@ -189,7 +189,7 @@ export class TestRunner extends EventEmitter { if (!this._trialRun) { await this._runHooks(test.parent, 'beforeEach', 'before', testInfo); const timeout = test._isSlow() ? this._timeout * 3 : this._timeout; - await fixturePool.runTestWithFixtures(test.fn, timeout, testInfo); + await fixturePool.runTestWithFixturesAndTimeout(test.fn, timeout, testInfo); await this._runHooks(test.parent, 'afterEach', 'after', testInfo); } else { result.status = result.expectedStatus; diff --git a/test-runner/src/util.ts b/test-runner/src/util.ts new file mode 100644 index 0000000000000..0e9c862088489 --- /dev/null +++ b/test-runner/src/util.ts @@ -0,0 +1,45 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export async function raceAgainstTimeout(promise: Promise, timeout: number): Promise<{ result?: T, timedOut?: boolean }> { + if (!timeout) + return { result: await promise }; + + let timer: NodeJS.Timer; + let done = false; + let fulfill: (t: { result?: T, timedOut?: boolean }) => void; + let reject: (e: Error) => void; + const result = new Promise((f, r) => { + fulfill = f; + reject = r; + }); + setTimeout(() => { + done = true; + fulfill({ timedOut: true }); + }, timeout); + promise.then(result => { + clearTimeout(timer); + if (!done) { + done = true; + fulfill({ result }); + } + }).catch(e => { + clearTimeout(timer); + if (!done) + reject(e); + }); + return result; +} diff --git a/test-runner/test/assets/fixture-timeout.js b/test-runner/test/assets/fixture-timeout.js new file mode 100644 index 0000000000000..8836bd0dd9d42 --- /dev/null +++ b/test-runner/test/assets/fixture-timeout.js @@ -0,0 +1,30 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { registerFixture } = require('../../'); + +registerFixture('timeout', async ({}, runTest) => { + await runTest(); + await new Promise(f => setTimeout(f, 100000)); +}); + +it('fixture timeout', async({timeout}) => { + expect(1).toBe(1); +}); + +it('failing fixture timeout', async({timeout}) => { + expect(1).toBe(2); +}); diff --git a/test-runner/test/exit-code.spec.ts b/test-runner/test/exit-code.spec.ts index b6567a28e323c..a243b0440ac4f 100644 --- a/test-runner/test/exit-code.spec.ts +++ b/test-runner/test/exit-code.spec.ts @@ -62,10 +62,18 @@ it('should access data in fixture', async () => { expect(testResult.stderr).toEqual([{ text: 'console.error\n' }]); }); +it('should handle fixture timeout', async () => { + const { exitCode, output, failed, timedOut } = await runTest('fixture-timeout.js', { timeout: 500 }); + expect(exitCode).toBe(1); + expect(output).toContain('Timeout of 500ms'); + expect(failed).toBe(1); + expect(timedOut).toBe(1); +}); + it('should handle worker fixture timeout', async () => { - const result = await runTest('worker-fixture-timeout.js', { timeout: 1000 }); + const result = await runTest('worker-fixture-timeout.js', { timeout: 500 }); expect(result.exitCode).toBe(1); - expect(result.output).toContain('Timeout of 1000ms'); + expect(result.output).toContain('Timeout of 500ms'); }); it('should handle worker fixture error', async () => { @@ -171,6 +179,12 @@ it('should retry unhandled rejection', async () => { expect(result.output).toContain('Unhandled rejection'); }); +it('should respect global timeout', async () => { + const { exitCode, output } = await runTest('one-timeout.js', { 'timeout': 100000, 'global-timeout': 500 }); + expect(exitCode).toBe(1); + expect(output).toContain('Timed out waiting 0.5s for the entire test run'); +}); + async function runTest(filePath: string, params: any = {}) { const outputDir = path.join(__dirname, 'test-results'); const reportFile = path.join(outputDir, 'results.json'); diff --git a/test/defaultbrowsercontext-2.spec.ts b/test/defaultbrowsercontext-2.spec.ts index cc85f7463381e..b41056b0f4f97 100644 --- a/test/defaultbrowsercontext-2.spec.ts +++ b/test/defaultbrowsercontext-2.spec.ts @@ -68,7 +68,9 @@ it('should support ignoreHTTPSErrors option', async ({httpsServer, launchPersist expect(response.ok()).toBe(true); }); -it('should support extraHTTPHeaders option', async ({server, launchPersistent}) => { +it('should support extraHTTPHeaders option', test => { + test.flaky(options.FIREFOX && !options.HEADLESS && LINUX, 'Intermittent timeout on bots'); +}, async ({server, launchPersistent}) => { const {page} = await launchPersistent({extraHTTPHeaders: { foo: 'bar' }}); const [request] = await Promise.all([ server.waitForRequest('/empty.html'), @@ -119,7 +121,7 @@ it('should restore state from userDataDir', test => { it('should restore cookies from userDataDir', test => { test.slow(); - test.flaky(options.CHROMIUM && WIN); + test.flaky(options.CHROMIUM); }, async ({browserType, defaultBrowserOptions, server, launchPersistent}) => { const userDataDir = await makeUserDataDir(); const browserContext = await browserType.launchPersistentContext(userDataDir, defaultBrowserOptions); diff --git a/test/frame-evaluate.spec.ts b/test/frame-evaluate.spec.ts index 3d8ef602864fd..a35cce6a745ec 100644 --- a/test/frame-evaluate.spec.ts +++ b/test/frame-evaluate.spec.ts @@ -131,7 +131,8 @@ it('should be isolated between frames', async ({page, server}) => { }); it('should work in iframes that failed initial navigation', test => { - test.fail(options.CHROMIUM || options.FIREFOX); + test.fail(options.CHROMIUM); + test.fixme(options.FIREFOX); }, async ({page}) => { // - Firefox does not report domcontentloaded for the iframe. // - Chromium and Firefox report empty url. diff --git a/test/page-evaluate.spec.ts b/test/page-evaluate.spec.ts index 8ccd4b75f7747..f8a619154f0d9 100644 --- a/test/page-evaluate.spec.ts +++ b/test/page-evaluate.spec.ts @@ -435,7 +435,9 @@ it('should not throw an error when evaluation does a synchronous navigation and expect(result).toBe(undefined); }); -it('should transfer 100Mb of data from page to node.js', async ({ page }) => { +it('should transfer 100Mb of data from page to node.js', test => { + test.skip(options.WIRE); +}, async ({ page }) => { // This is too slow with wire. const a = await page.evaluate(() => Array(100 * 1024 * 1024 + 1).join('a')); expect(a.length).toBe(100 * 1024 * 1024); diff --git a/test/page-event-pageerror.spec.ts b/test/page-event-pageerror.spec.ts index d17bb8131597c..4e7ff6dde56e5 100644 --- a/test/page-event-pageerror.spec.ts +++ b/test/page-event-pageerror.spec.ts @@ -58,7 +58,7 @@ it('should handle odd values', async ({page}) => { }); it('should handle object', test => { - test.fail(options.FIREFOX); + test.fixme(options.FIREFOX); }, async ({page}) => { // Firefox just does not report this error. const [error] = await Promise.all([ @@ -69,7 +69,7 @@ it('should handle object', test => { }); it('should handle window', test => { - test.fail(options.FIREFOX); + test.fixme(options.FIREFOX); }, async ({page}) => { // Firefox just does not report this error. const [error] = await Promise.all([ diff --git a/test/permissions.spec.ts b/test/permissions.spec.ts index 741666300df68..1089bd496e162 100644 --- a/test/permissions.spec.ts +++ b/test/permissions.spec.ts @@ -149,7 +149,7 @@ describe('permissions', suite => { it('should support clipboard read', test => { test.fail(options.WEBKIT); test.fail(options.FIREFOX, 'No such permissions (requires flag) in Firefox'); - test.fail(options.CHROMIUM && !options.HEADLESS); + test.fixme(options.CHROMIUM && !options.HEADLESS); }, async ({page, server, context}) => { await page.goto(server.EMPTY_PAGE); expect(await getPermission(page, 'clipboard-read')).toBe('prompt'); diff --git a/test/proxy.spec.ts b/test/proxy.spec.ts index 65eb4fefbd990..389bf40c1df30 100644 --- a/test/proxy.spec.ts +++ b/test/proxy.spec.ts @@ -78,7 +78,7 @@ it('should authenticate', async ({browserType, defaultBrowserOptions, server}) = }); it('should exclude patterns', test => { - test.fail(options.CHROMIUM && !options.HEADLESS, 'Chromium headful crashes with CHECK(!in_frame_tree_) in RenderFrameImpl::OnDeleteFrame.'); + test.flaky(options.CHROMIUM && !options.HEADLESS, 'Chromium headful crashes with CHECK(!in_frame_tree_) in RenderFrameImpl::OnDeleteFrame.'); }, async ({browserType, defaultBrowserOptions, server}) => { server.setRoute('/target.html', async (req, res) => { res.end('Served by the proxy'); @@ -119,7 +119,9 @@ it('should exclude patterns', test => { await browser.close(); }); -it('should use socks proxy', async ({ browserType, defaultBrowserOptions }) => { +it('should use socks proxy', test => { + test.flaky(MAC && options.WEBKIT, 'Intermittent page.goto: The network connection was lost error on bots'); +}, async ({ browserType, defaultBrowserOptions }) => { const server = socks.createServer((info, accept, deny) => { let socket; if ((socket = accept(true))) { diff --git a/test/screencast.spec.ts b/test/screencast.spec.ts index 2a99d488afe72..05455304e63b9 100644 --- a/test/screencast.spec.ts +++ b/test/screencast.spec.ts @@ -204,6 +204,7 @@ describe('screencast', suite => { it('should capture navigation', test => { test.flaky(options.WEBKIT); + test.flaky(options.FIREFOX); }, async ({page, tmpDir, server, videoPlayer, toImpl}) => { const videoFile = path.join(tmpDir, 'v.webm'); await page.goto(server.PREFIX + '/background-color.html#rgb(0,0,0)'); @@ -263,7 +264,8 @@ describe('screencast', suite => { it('should fire start/stop events when page created/closed', test => { test.slow(); - }, async ({browser, tmpDir, server, toImpl}) => { + test.flaky(options.FIREFOX, 'Even slow is not slow enough'); + }, async ({browser, tmpDir, toImpl}) => { // Use server side of the context. All the code below also uses server side APIs. const context = toImpl(await browser.newContext()); await context._enableScreencast({width: 640, height: 480, dir: tmpDir});