From ab9be18fe0c9fe34a6a44336cb3c0898a389ade7 Mon Sep 17 00:00:00 2001 From: Adam Raine <6752989+adamraine@users.noreply.github.com> Date: Fri, 8 Mar 2024 11:54:57 -0800 Subject: [PATCH 1/2] tests(devtools): use `is_debug = true` for local builds (#15860) --- core/scripts/build-devtools.sh | 4 ++-- core/test/devtools-tests/download-devtools.sh | 4 ++-- core/test/devtools-tests/roll-devtools.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/scripts/build-devtools.sh b/core/scripts/build-devtools.sh index 37453dcd5828..d523a4378e4b 100644 --- a/core/scripts/build-devtools.sh +++ b/core/scripts/build-devtools.sh @@ -45,9 +45,9 @@ yarn devtools "$DEVTOOLS_PATH" cd "$DEVTOOLS_PATH" if [[ "$CI" ]]; then - gn gen "out/$BUILD_FOLDER" --args='devtools_dcheck_always_on=true is_debug=false' + gn gen "out/$BUILD_FOLDER" --args='is_debug=false' else - gn gen "out/$BUILD_FOLDER" --args='devtools_dcheck_always_on=true is_debug=false devtools_skip_typecheck=true' + gn gen "out/$BUILD_FOLDER" --args='is_debug=true devtools_skip_typecheck=true' fi gclient sync autoninja -C "out/$BUILD_FOLDER" diff --git a/core/test/devtools-tests/download-devtools.sh b/core/test/devtools-tests/download-devtools.sh index b1559b923148..5ea3ddc31a3d 100644 --- a/core/test/devtools-tests/download-devtools.sh +++ b/core/test/devtools-tests/download-devtools.sh @@ -36,7 +36,7 @@ fetch --nohooks --no-history devtools-frontend cd devtools-frontend gclient sync if [[ "$CI" ]]; then - gn gen "out/$BUILD_FOLDER" --args='devtools_dcheck_always_on=true is_debug=false' + gn gen "out/$BUILD_FOLDER" --args='is_debug=false' else - gn gen "out/$BUILD_FOLDER" --args='devtools_dcheck_always_on=true is_debug=false devtools_skip_typecheck=true' + gn gen "out/$BUILD_FOLDER" --args='is_debug=true devtools_skip_typecheck=true' fi diff --git a/core/test/devtools-tests/roll-devtools.sh b/core/test/devtools-tests/roll-devtools.sh index cb9ef0d73612..a39946f2b9c3 100644 --- a/core/test/devtools-tests/roll-devtools.sh +++ b/core/test/devtools-tests/roll-devtools.sh @@ -31,9 +31,9 @@ roll_devtools gclient sync --delete_unversioned_trees --reset if [[ "$CI" ]]; then - gn gen "out/$BUILD_FOLDER" --args='devtools_dcheck_always_on=true is_debug=false' + gn gen "out/$BUILD_FOLDER" --args='is_debug=false' else - gn gen "out/$BUILD_FOLDER" --args='devtools_dcheck_always_on=true is_debug=false devtools_skip_typecheck=true' + gn gen "out/$BUILD_FOLDER" --args='is_debug=true devtools_skip_typecheck=true' fi # Build devtools. By default, this creates `out/LighthouseIntegration/gen/front_end`. From 62bcd987fdb7ef38182e7a5e7450e6823a4207d6 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Fri, 8 Mar 2024 13:37:18 -0800 Subject: [PATCH 2/2] tests(full-page-screenshot): add node verification and debug tool (#15324) --- cli/test/fixtures/screenshot-nodes.html | 59 ++++ core/gather/gatherers/full-page-screenshot.js | 11 +- core/scripts/full-page-screenshot-debug.js | 95 ++++++ .../gatherers/full-page-screenshot-test.js | 321 +++++++++++++++++- core/test/scripts/run-mocha-tests.js | 3 + types/lhr/lhr.d.ts | 2 +- 6 files changed, 485 insertions(+), 6 deletions(-) create mode 100644 cli/test/fixtures/screenshot-nodes.html create mode 100644 core/scripts/full-page-screenshot-debug.js diff --git a/cli/test/fixtures/screenshot-nodes.html b/cli/test/fixtures/screenshot-nodes.html new file mode 100644 index 000000000000..82c1f195776d --- /dev/null +++ b/cli/test/fixtures/screenshot-nodes.html @@ -0,0 +1,59 @@ + + + + + + + + +

+ + + +
+
+ +
+ diff --git a/core/gather/gatherers/full-page-screenshot.js b/core/gather/gatherers/full-page-screenshot.js index dd368cb7d1c6..33fbcfb76b7e 100644 --- a/core/gather/gatherers/full-page-screenshot.js +++ b/core/gather/gatherers/full-page-screenshot.js @@ -13,7 +13,10 @@ import {waitForNetworkIdle} from '../driver/wait-for-condition.js'; // JPEG quality setting // Exploration and examples of reports using different quality settings: https://docs.google.com/document/d/1ZSffucIca9XDW2eEwfoevrk-OTl7WQFeMf0CgeJAA8M/edit# // Note: this analysis was done for JPEG, but now we use WEBP. -const FULL_PAGE_SCREENSHOT_QUALITY = 30; +const FULL_PAGE_SCREENSHOT_QUALITY = process.env.LH_FPS_TEST ? 100 : 30; +// webp currently cant do lossless encoding, so to help tests switch to png +// Remove when this is resolved: https://bugs.chromium.org/p/chromium/issues/detail?id=1469183 +const FULL_PAGE_SCREENSHOT_FORMAT = process.env.LH_FPS_TEST ? 'png' : 'webp'; // https://developers.google.com/speed/webp/faq#what_is_the_maximum_size_a_webp_image_can_be const MAX_WEBP_SIZE = 16383; @@ -121,10 +124,10 @@ class FullPageScreenshot extends BaseGatherer { */ async _takeScreenshot(context) { const result = await context.driver.defaultSession.sendCommand('Page.captureScreenshot', { - format: 'webp', + format: FULL_PAGE_SCREENSHOT_FORMAT, quality: FULL_PAGE_SCREENSHOT_QUALITY, }); - const data = 'data:image/webp;base64,' + result.data; + const data = `data:image/${FULL_PAGE_SCREENSHOT_FORMAT};base64,` + result.data; const screenshotAreaSize = await context.driver.executionContext.evaluate(getScreenshotAreaSize, { @@ -159,7 +162,7 @@ class FullPageScreenshot extends BaseGatherer { for (const [node, id] of lhIdToElements.entries()) { // @ts-expect-error - getBoundingClientRect put into scope via stringification const rect = getBoundingClientRect(node); - nodes[id] = rect; + nodes[id] = {id: node.id, ...rect}; } return nodes; diff --git a/core/scripts/full-page-screenshot-debug.js b/core/scripts/full-page-screenshot-debug.js new file mode 100644 index 000000000000..8d19a7adbcd5 --- /dev/null +++ b/core/scripts/full-page-screenshot-debug.js @@ -0,0 +1,95 @@ +/** + * @license Copyright 2023 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +// To open result in chrome: +// node core/scripts/full-page-screenshot-debug.js latest-run/lhr.report.json | xargs "$CHROME_PATH" + +import * as fs from 'fs'; + +import esMain from 'es-main'; +import * as puppeteer from 'puppeteer-core'; +import {getChromePath} from 'chrome-launcher'; + +import {LH_ROOT} from '../../shared/root.js'; + +/** + * @param {LH.Result} lhr + * @return {Promise} + */ +async function getDebugImage(lhr) { + if (!lhr.fullPageScreenshot) { + return ''; + } + + const browser = await puppeteer.launch({ + headless: true, + executablePath: getChromePath(), + ignoreDefaultArgs: ['--enable-automation'], + }); + const page = await browser.newPage(); + + const debugDataUrl = await page.evaluate(async (fullPageScreenshot) => { + const img = await new Promise((resolve, reject) => { + // eslint-disable-next-line no-undef + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = reject; + img.src = fullPageScreenshot.screenshot.data; + }); + + // eslint-disable-next-line no-undef + const canvasEl = document.createElement('canvas'); + canvasEl.width = img.width; + canvasEl.height = img.height; + const ctx = canvasEl.getContext('2d'); + if (!ctx) return ''; + + ctx.drawImage(img, 0, 0); + for (const [lhId, node] of Object.entries(fullPageScreenshot.nodes)) { + if (!node.width && !node.height) continue; + + ctx.strokeStyle = '#D3E156'; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.rect(node.left, node.top, node.width, node.height); + ctx.stroke(); + + const txt = node.id || lhId; + const txtWidth = Math.min(ctx.measureText(txt).width, node.width); + const txtHeight = 10; + const txtTop = node.top - 3; + const txtLeft = node.left; + ctx.fillStyle = '#FFFFFF88'; + ctx.fillRect(txtLeft, txtTop, txtWidth, txtHeight); + ctx.fillStyle = '#000000'; + ctx.lineWidth = 1; + ctx.textBaseline = 'top'; + ctx.fillText(txt, txtLeft, txtTop); + } + + return canvasEl.toDataURL(); + }, lhr.fullPageScreenshot); + + await browser.close(); + + if (!debugDataUrl.startsWith('data:image/')) { + throw new Error('invalid data url'); + } + + return debugDataUrl; +} + +if (esMain(import.meta)) { + const lhr = JSON.parse(fs.readFileSync(process.argv[2], 'utf-8')); + const imageUrl = await getDebugImage(lhr); + const [type, base64Data] = imageUrl.split(','); + const ext = type.replace('data:image/', ''); + const dest = `${LH_ROOT}/.tmp/fps-debug.${ext}`; + fs.writeFileSync(dest, base64Data, 'base64'); + console.log(dest); +} + +export {getDebugImage}; diff --git a/core/test/gather/gatherers/full-page-screenshot-test.js b/core/test/gather/gatherers/full-page-screenshot-test.js index f1e464d0fec2..1c1b459c4a0c 100644 --- a/core/test/gather/gatherers/full-page-screenshot-test.js +++ b/core/test/gather/gatherers/full-page-screenshot-test.js @@ -3,8 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ +import * as puppeteer from 'puppeteer-core'; +import {getChromePath} from 'chrome-launcher'; + +import * as LH from '../../../../types/lh.js'; import {createMockContext} from '../../gather/mock-driver.js'; import FullPageScreenshotGatherer from '../../../gather/gatherers/full-page-screenshot.js'; +import lighthouse from '../../../index.js'; +import {Server} from '../../../../cli/test/fixtures/static-server.js'; /** @type {{width: number, height: number}} */ let contentSize; @@ -84,7 +90,7 @@ describe('FullPageScreenshot gatherer', () => { expect(artifact).toEqual({ screenshot: { - data: 'data:image/webp;base64,abc', + data: 'data:image/png;base64,abc', height: 2000, width: 412, }, @@ -201,4 +207,317 @@ describe('FullPageScreenshot gatherer', () => { } ); }); + + // Tests that our node rects line up with content in the screenshot image data. + // This uses "screenshot-nodes.html", which has elements of solid colors to make verification simple. + // To verify a node rect is correct, each pixel in its area is looked at in the screenshot data, and is checked + // for the expected color. Due to compression artifacts, there are thresholds involved instead of exact matches. + describe('end-to-end integration test', () => { + const port = 10503; + let serverBaseUrl = '' + /** @type {StaticServer} */; + let server; + /** @type {puppeteer.Browser} */ + let browser; + /** @type {puppeteer.Page} */ + let page; + + before(async () => { + browser = await puppeteer.launch({ + headless: true, + executablePath: getChromePath(), + ignoreDefaultArgs: ['--enable-automation'], + }); + + server = new Server(port); + await server.listen(port, '127.0.0.1'); + serverBaseUrl = `http://localhost:${server.getPort()}`; + }); + + after(async () => { + await browser.close(); + await server.close(); + }); + + beforeEach(async () => { + page = await browser.newPage(); + }); + + afterEach(async () => { + await page.close(); + }); + + /** + * @typedef NodeAnalysisResult + * @property {string} id + * @property {boolean} success + * @property {number[][]} debugData + */ + + /** + * @param {puppeteer.Page} page + * @param {LH.Result} lhr + * @param {DebugFormat} debugFormat + * @return {Promise} + */ + function analyzeScreenshotNodes(page, lhr, debugFormat) { + const options = { + fullPageScreenshot: lhr.fullPageScreenshot, + debugFormat, + }; + + return page.evaluate(async (options) => { + function hexToRgb(hex) { + const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex); + return { + r: parseInt(result[1], 16), + g: parseInt(result[2], 16), + b: parseInt(result[3], 16), + }; + } + + // https://www.compuphase.com/cmetric.htm + function colorDistance(e1, e2) { + const rmean = (e1.r + e2.r) / 2; + const r = e1.r - e2.r; + const g = e1.g - e2.g; + const b = e1.b - e2.b; + return Math.sqrt( + (((512 + rmean) * r * r) >> 8) + 4 * g * g + (((767 - rmean) * b * b) >> 8)); + } + + const img = await new Promise((resolve, reject) => { + // eslint-disable-next-line no-undef + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = reject; + img.src = options.fullPageScreenshot.screenshot.data; + }); + + // eslint-disable-next-line no-undef + const canvasEl = document.createElement('canvas'); + canvasEl.width = img.width; + canvasEl.height = img.height; + const ctx = canvasEl.getContext('2d'); + ctx.drawImage(img, 0, 0); + + const results = []; + for (const node of Object.values(options.fullPageScreenshot.nodes)) { + if (!(node.id.includes('green') || node.id.includes('red'))) { + continue; + } + if (!node.id.includes('el-')) { + continue; + } + + const expectedColor = hexToRgb(node.id.includes('green') ? '#427f36' : '#8a4343'); + const result = {id: node.id, success: true}; + results.push(result); + + const right = Math.min(node.right, canvasEl.width - 1); + const bottom = Math.min(node.bottom, canvasEl.height - 1); + + const debugData = []; + for (let y = node.top; y <= bottom; y++) { + const row = []; + debugData.push(row); + + for (let x = node.left; x <= right; x++) { + const [r, g, b] = ctx.getImageData(x, y, 1, 1).data; + const delta = colorDistance(expectedColor, {r, g, b}); + const pass = delta > 0; + + if (options.debugFormat === 'color') { + row.push((r << 16) | (g << 8) | b); + } else if (options.debugFormat === 'delta') { + row.push(delta); + } else if (options.debugFormat === 'pass') { + row.push(pass); + } + } + } + + if (options.debugFormat) result.debugData = debugData; + } + + return results; + }, options); + } + + /** + * @param {NodeAnalysisResult[]} results + * @param {DebugFormat} debugFormat + */ + function visualizeDebugData(results, debugFormat) { + for (const result of results) { + console.log(`\n=== ${result.id} ${result.success ? 'success' : 'failure'} ===\n`); + + const columns = result.debugData; + for (let y = 0; y < columns.length; y++) { + let line = ''; + for (let x = 0; x < columns[0].length; x++) { + if (debugFormat === 'color') { + line += columns[y][x].toString(16).padStart(6, '0') + ' '; + } else if (debugFormat === 'delta') { + line += columns[y][x].toFixed(1) + ' '; + } else if (debugFormat === 'pass') { + line += columns[y][x] ? 'O' : 'X'; + } + } + console.log(line); + } + } + } + + /** + * @param {LH.Result} lhr + * @param {Array} rectExpectations + */ + async function verifyNodeRectsAlignWithImage(lhr, rectExpectations) { + if (!lhr.fullPageScreenshot) throw new Error('no screenshot'); + + // First check we recieved all the expected nodes. + const nodes = Object.values(lhr.fullPageScreenshot.nodes); + for (const expectation of rectExpectations) { + const nodeSeen = nodes.find(node => node.id === expectation.id); + if (!nodeSeen) throw new Error(`did not find node for id ${expectation.id}`); + + const {id, left, top, right, bottom} = nodeSeen; + expect({id, left, top, right, bottom}).toEqual(expectation); + } + + // Now check that the image contents line up with what we think the nodes are. + + /** @type {DebugFormat} */ + const debugFormat = process.env.LH_FPS_DEBUG ?? false; + const results = await analyzeScreenshotNodes(page, lhr, debugFormat); + + // Very helpful for debugging. Set env LH_FPS_DEBUG to one of the valid debug formats. + if (debugFormat) { + console.log(lhr.fullPageScreenshot.screenshot.data); + visualizeDebugData(results, debugFormat); + } + + const failingIds = results.filter(r => !r.success).map(r => r.id); + expect(failingIds).toEqual([]); + + expect(results.length).toBeGreaterThan(0); + } + + it('mobile dpr 1', async () => { + const rectExpectations = [ + { + id: 'el-1-red', + top: 10, + bottom: 20, + left: 18, + right: 178, + }, + { + id: 'el-2-green', + top: 60, + bottom: 120, + left: 48, + right: 108, + }, + ]; + const runnerResult = await lighthouse(`${serverBaseUrl}/screenshot-nodes.html`, { + onlyAudits: ['largest-contentful-paint-element', 'aria-required-parent'], + screenEmulation: {mobile: true, width: 600, height: 900, deviceScaleFactor: 1.0}, + }, undefined, page); + if (!runnerResult) throw new Error('no runner result'); + + await verifyNodeRectsAlignWithImage(runnerResult.lhr, rectExpectations); + }); + + it('mobile dpr 1 tiny viewport', async () => { + const rectExpectations = [ + { + id: 'el-1-red', + top: 10, + bottom: 20, + left: 18, + right: 178, + }, + { + id: 'el-2-green', + top: 60, + bottom: 120, + left: 48, + right: 108, + }, + ]; + const runnerResult = await lighthouse(`${serverBaseUrl}/screenshot-nodes.html`, { + onlyAudits: ['largest-contentful-paint-element', 'aria-required-parent'], + screenEmulation: {mobile: true, width: 100, height: 200, deviceScaleFactor: 1.0}, + }, undefined, page); + if (!runnerResult) throw new Error('no runner result'); + + await verifyNodeRectsAlignWithImage(runnerResult.lhr, rectExpectations); + }); + + it('mobile dpr 1.75', async () => { + const rectExpectations = [ + { + id: 'el-1-red', + top: 10, + bottom: 20, + left: 18, + right: 178, + }, + { + id: 'el-2-green', + top: 60, + bottom: 120, + left: 48, + right: 108, + }, + ]; + const runnerResult = await lighthouse(`${serverBaseUrl}/screenshot-nodes.html`, { + onlyAudits: ['largest-contentful-paint-element', 'aria-required-parent'], + screenEmulation: {mobile: true, width: 600, height: 900, deviceScaleFactor: 1.75}, + }, undefined, page); + if (!runnerResult) throw new Error('no runner result'); + + await verifyNodeRectsAlignWithImage(runnerResult.lhr, rectExpectations); + }); + + it('desktop', async () => { + const rectExpectations = [ + { + id: 'el-1-red', + top: 10, + bottom: 20, + left: 18, + right: 178, + }, + { + id: 'el-2-green', + top: 60, + bottom: 120, + left: 48, + right: 108, + }, + ]; + const runnerResult = await lighthouse(`${serverBaseUrl}/screenshot-nodes.html`, { + onlyAudits: ['largest-contentful-paint-element', 'aria-required-parent'], + screenEmulation: {mobile: false, width: 1350, height: 940, deviceScaleFactor: 1}, + formFactor: 'desktop', + }, undefined, page); + if (!runnerResult) throw new Error('no runner result'); + + await verifyNodeRectsAlignWithImage(runnerResult.lhr, rectExpectations); + }); + + it('grow', async () => { + const runnerResult = await lighthouse(`${serverBaseUrl}/screenshot-nodes.html?grow`, { + onlyAudits: ['largest-contentful-paint-element', 'aria-required-parent'], + screenEmulation: {mobile: true, width: 600, height: 900, deviceScaleFactor: 1.0}, + }, undefined, page); + if (!runnerResult) throw new Error('no runner result'); + + // No rect expectations, because we can't know what they'll be ahead of time. + await verifyNodeRectsAlignWithImage(runnerResult.lhr, []); + }); + }); }); diff --git a/core/test/scripts/run-mocha-tests.js b/core/test/scripts/run-mocha-tests.js index 1c99b5de2509..fc5bd253ceeb 100644 --- a/core/test/scripts/run-mocha-tests.js +++ b/core/test/scripts/run-mocha-tests.js @@ -22,6 +22,9 @@ import glob from 'glob'; import {LH_ROOT} from '../../../shared/root.js'; import {mochaGlobalSetup, mochaGlobalTeardown} from '../test-env/mocha-setup.js'; +// Tell gatherer to use 100 quality for FPS tests. +process.env.LH_FPS_TEST = '1'; + const failedTestsDir = `${LH_ROOT}/.tmp/failing-tests`; if (!isMainThread && parentPort) { diff --git a/types/lhr/lhr.d.ts b/types/lhr/lhr.d.ts index df4edf201ab1..0248d10a2c5b 100644 --- a/types/lhr/lhr.d.ts +++ b/types/lhr/lhr.d.ts @@ -150,7 +150,7 @@ declare module Result { width: number; height: number; }; - nodes: Record; + nodes: Record; } /**