From f8750cf58b0adcd2d64ee2830c0f33e87b8e445a Mon Sep 17 00:00:00 2001 From: Adam Raine <6752989+adamraine@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:21:22 -0700 Subject: [PATCH] tests(unit): retry failures and upload failure artifacts (#15378) --- .github/workflows/unit.yml | 7 +++++ core/test/scenarios/api-test-pptr.js | 7 +++++ core/test/scenarios/pptr-test-utils.js | 28 +++++++++++++++++++ .../start-end-navigation-test-pptr.js | 2 ++ core/test/scripts/run-mocha-tests.js | 14 ++++++++++ package.json | 2 +- 6 files changed, 59 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 9f07711e16ff..80a45b24926d 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -84,6 +84,13 @@ jobs: - name: yarn test-viewer run: yarn build-viewer && xvfb-run --auto-servernum bash $GITHUB_WORKSPACE/.github/scripts/test-retry.sh yarn test-viewer + - name: Upload failures + if: failure() + uses: actions/upload-artifact@v1 + with: + name: Unit (ubuntu; node ${{ matrix.node }}) + path: .tmp/unit-failures/ + # For windows, just test the potentially platform-specific code. unit-windows: runs-on: windows-latest diff --git a/core/test/scenarios/api-test-pptr.js b/core/test/scenarios/api-test-pptr.js index b293ee81d286..811141e86467 100644 --- a/core/test/scenarios/api-test-pptr.js +++ b/core/test/scenarios/api-test-pptr.js @@ -75,6 +75,7 @@ describe('Individual modes API', function() { if (!result) throw new Error('Lighthouse failed to produce a result'); const {lhr, artifacts} = result; + state.saveTrace(artifacts.Trace); expect(artifacts.URL).toEqual({ finalDisplayedUrl: `${state.serverBaseUrl}/onclick.html#done`, }); @@ -133,6 +134,8 @@ describe('Individual modes API', function() { if (!result) throw new Error('Lighthouse failed to produce a result'); + state.saveTrace(result.artifacts.Trace); + expect(result.artifacts.URL).toEqual({ finalDisplayedUrl: `${serverBaseUrl}/onclick.html#done`, }); @@ -165,6 +168,8 @@ describe('Individual modes API', function() { if (!result) throw new Error('Lighthouse failed to produce a result'); + state.saveTrace(result.artifacts.Trace); + const networkRequestsDetails = /** @type {LH.Audit.Details.Table} */ ( result.lhr.audits['network-requests'].details); const networkRequests = networkRequestsDetails?.items @@ -222,6 +227,7 @@ Array [ if (!result) throw new Error('Lighthouse failed to produce a result'); const {lhr, artifacts} = result; + state.saveTrace(artifacts.Trace); expect(artifacts.URL).toEqual({ requestedUrl: url, mainDocumentUrl: url, @@ -263,6 +269,7 @@ Array [ expect(requestor).toHaveBeenCalled(); const {lhr, artifacts} = result; + state.saveTrace(artifacts.Trace); expect(lhr.requestedUrl).toEqual(requestedUrl); expect(lhr.finalDisplayedUrl).toEqual(mainDocumentUrl); expect(artifacts.URL).toEqual({ diff --git a/core/test/scenarios/pptr-test-utils.js b/core/test/scenarios/pptr-test-utils.js index 5bfe6e681653..dd8a4beff855 100644 --- a/core/test/scenarios/pptr-test-utils.js +++ b/core/test/scenarios/pptr-test-utils.js @@ -4,11 +4,14 @@ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +import fs from 'fs'; + import {before, beforeEach, after, afterEach} from 'mocha'; import * as puppeteer from 'puppeteer-core'; import {getChromePath} from 'chrome-launcher'; import {Server} from '../../../cli/test/fixtures/static-server.js'; +import {LH_ROOT} from '../../../root.js'; /** @typedef {InstanceType} StaticServer */ @@ -22,12 +25,17 @@ const FLAKY_AUDIT_IDS_APPLICABILITY = new Set([ 'layout-shift-elements', // Depends on if the JS takes too long after input to be ignored for layout shift. ]); +const UNIT_OUTPUT_DIR = `${LH_ROOT}/.tmp/unit-failures`; + function createTestState() { /** @param {string} name @return {any} */ const any = name => new Proxy({}, {get: () => { throw new Error(`${name} used without invoking \`state.before\``); }}); + /** @type {LH.Trace|undefined} */ + let trace; + return { browser: /** @type {puppeteer.Browser} */ (any('browser')), page: /** @type {puppeteer.Page} */ (any('page')), @@ -68,6 +76,8 @@ function createTestState() { }); beforeEach(async () => { + trace = undefined; + console.log('########'); this.page = await this.browser.newPage(); }); @@ -75,10 +85,28 @@ function createTestState() { await this.page.close(); }); + afterEach(function() { + // eslint-disable-next-line no-invalid-this + const currentTest = this.currentTest; + if (currentTest?.state === 'failed' && trace) { + const dirname = currentTest.fullTitle().replace(/[^a-z0-9]/gi, '_').toLowerCase(); + const testOutputDir = `${UNIT_OUTPUT_DIR}/${dirname}`; + fs.mkdirSync(testOutputDir, {recursive: true}); + fs.writeFileSync(`${testOutputDir}/trace.json`, JSON.stringify(trace, null, 2)); + } + }); + after(async () => { await this.browser.close(); }); }, + + /** + * @param {LH.Trace} testTrace + */ + saveTrace(testTrace) { + trace = testTrace; + }, }; } diff --git a/core/test/scenarios/start-end-navigation-test-pptr.js b/core/test/scenarios/start-end-navigation-test-pptr.js index 294489bf5a9a..efa50d1e1b2e 100644 --- a/core/test/scenarios/start-end-navigation-test-pptr.js +++ b/core/test/scenarios/start-end-navigation-test-pptr.js @@ -37,6 +37,8 @@ describe('Start/End navigation', function() { const lhr = flowResult.steps[0].lhr; const artifacts = flowArtifacts.gatherSteps[0].artifacts; + state.saveTrace(artifacts.Trace); + expect(artifacts.URL).toEqual({ requestedUrl: `${state.serverBaseUrl}/?redirect=/index.html`, mainDocumentUrl: `${state.serverBaseUrl}/index.html`, diff --git a/core/test/scripts/run-mocha-tests.js b/core/test/scripts/run-mocha-tests.js index eb237a07567f..d1a872ca710c 100644 --- a/core/test/scripts/run-mocha-tests.js +++ b/core/test/scripts/run-mocha-tests.js @@ -146,6 +146,14 @@ const rawArgv = y 'require': { type: 'string', }, + 'retries': { + type: 'number', + default: process.env.CI ? 5 : undefined, + }, + 'forbidOnly': { + type: 'boolean', + default: Boolean(process.env.CI), + }, }) .wrap(y.terminalWidth()) .argv; @@ -257,8 +265,10 @@ function exit({numberFailures, numberMochaInvocations}) { * @typedef OurMochaArgs * @property {RegExp | string | undefined} grep * @property {boolean} bail + * @property {boolean} forbidOnly * @property {boolean} parallel * @property {string | undefined} require + * @property {number | undefined} retries */ /** @@ -278,9 +288,11 @@ async function runMocha(tests, mochaArgs, invocationNumber) { timeout: 20_000, bail: mochaArgs.bail, grep: mochaArgs.grep, + forbidOnly: mochaArgs.forbidOnly, // TODO: not working // parallel: tests.length > 1 && mochaArgs.parallel, parallel: false, + retries: mochaArgs.retries, }); // @ts-expect-error - not in types. @@ -325,6 +337,8 @@ async function main() { bail: argv.bail, parallel: argv.parallel, require: argv.require, + retries: argv.retries, + forbidOnly: argv.forbidOnly, }; mochaGlobalSetup(); diff --git a/package.json b/package.json index 95af379a4736..f10e84630d61 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "unit-viewer": "yarn mocha --testMatch viewer/**/*-test.js", "unit-flow": "bash flow-report/test/run-flow-report-tests.sh", "unit": "yarn unit-flow && yarn mocha", - "unit:ci": "NODE_OPTIONS=--max-old-space-size=8192 npm run unit --forbid-only", + "unit:ci": "NODE_OPTIONS=--max-old-space-size=8192 npm run unit", "core-unit": "yarn unit-core", "cli-unit": "yarn unit-cli", "viewer-unit": "yarn unit-viewer",