Improve test flakiness

test/e2e/onCLS-test.js

@@ -16,6 +16,7 @@
 
 import assert from 'assert';
 import {beaconCountIs, clearBeacons, getBeacons} from '../utils/beacons.js';
+import {browserSupportsActivationStart} from '../utils/browserSupportsActivationStart.js';
 import {browserSupportsEntry} from '../utils/browserSupportsEntry.js';
 import {firstContentfulPaint} from '../utils/firstContentfulPaint.js';
 import {imagesPainted} from '../utils/imagesPainted.js';
@@ -29,8 +30,10 @@ describe('onCLS()', async function () {
   this.retries(2);
 
   let browserSupportsCLS;
+  let browserSupportsPrerender;
   before(async function () {
     browserSupportsCLS = await browserSupportsEntry('layout-shift');
+    browserSupportsPrerender = await browserSupportsActivationStart();
 
     // Set a standard screen size so thresholds are the same
     browser.setWindowSize(1280, 1024);
@@ -686,6 +689,7 @@ describe('onCLS()', async function () {
 
   it('reports prerender as nav type for prerender', async function () {
     if (!browserSupportsCLS) this.skip();
+    if (!browserSupportsPrerender) this.skip();
 
     await navigateTo('/test/cls?prerender=1');
 

test/e2e/onFCP-test.js

@@ -16,36 +16,21 @@
 
 import assert from 'assert';
 import {beaconCountIs, clearBeacons, getBeacons} from '../utils/beacons.js';
+import {browserSupportsActivationStart} from '../utils/browserSupportsActivationStart.js';
 import {browserSupportsEntry} from '../utils/browserSupportsEntry.js';
 import {navigateTo} from '../utils/navigateTo.js';
 import {stubForwardBack} from '../utils/stubForwardBack.js';
 import {stubVisibilityChange} from '../utils/stubVisibilityChange.js';
 
-// Temp fix to address Firefox flakiness.
-// See https://github.com/GoogleChrome/web-vitals/issues/472
-const originalStrictEqual = assert.strictEqual;
-assert.strictEqual = function (actual, expected, message) {
-  if (
-    process.env.GITHUB_ACTIONS &&
-    browser.capabilities.browserName === 'firefox' &&
-    (expected === 'good' || expected === 'needs-improvement') &&
-    actual !== expected
-  ) {
-    console.error(
-      `Override assert for Firefox (actual: ${actual}, expected: ${expected})`,
-    );
-    return true;
-  }
-  return originalStrictEqual(actual, expected, message);
-};
-
 describe('onFCP()', async function () {
   // Retry all tests in this suite up to 2 times.
   this.retries(2);
 
   let browserSupportsFCP;
+  let browserSupportsPrerender;
   before(async function () {
     browserSupportsFCP = await browserSupportsEntry('paint');
+    browserSupportsPrerender = await browserSupportsActivationStart();
   });
 
   beforeEach(async function () {
@@ -89,6 +74,7 @@ describe('onFCP()', async function () {
 
   it('accounts for time prerendering the page', async function () {
     if (!browserSupportsFCP) this.skip();
+    if (!browserSupportsPrerender) this.skip();
 
     await navigateTo('/test/fcp?prerender=1');
 
@@ -136,6 +122,10 @@ describe('onFCP()', async function () {
 
     await navigateTo('/test/fcp?hidden=1', {readyState: 'interactive'});
 
+    // Wait a bit to ensure an initial "frame" happens to avoid Safari
+    // flakiness with this test if switching visibility as soon as interactive.
+    await browser.pause(100);
+
     await stubVisibilityChange('visible');
 
     // Wait a bit to ensure no beacons were sent.
@@ -335,6 +325,7 @@ describe('onFCP()', async function () {
 
     it('accounts for time prerendering the page', async function () {
       if (!browserSupportsFCP) this.skip();
+      if (!browserSupportsPrerender) this.skip();
 
       await navigateTo(`/test/fcp?attribution=1&prerender=1`, {
         readyState: 'complete',

test/e2e/onFID-test.js

@@ -16,6 +16,7 @@
 
 import assert from 'assert';
 import {beaconCountIs, clearBeacons, getBeacons} from '../utils/beacons.js';
+import {browserSupportsActivationStart} from '../utils/browserSupportsActivationStart.js';
 import {browserSupportsEntry} from '../utils/browserSupportsEntry.js';
 import {navigateTo} from '../utils/navigateTo.js';
 import {stubForwardBack} from '../utils/stubForwardBack.js';
@@ -26,8 +27,10 @@ describe('onFID()', async function () {
   this.retries(2);
 
   let browserSupportsFID;
+  let browserSupportsPrerender;
   before(async function () {
     browserSupportsFID = await browserSupportsEntry('first-input');
+    browserSupportsPrerender = await browserSupportsActivationStart();
   });
 
   beforeEach(async function () {
@@ -188,6 +191,7 @@ describe('onFID()', async function () {
 
   it('reports prerender as nav type for prerender', async function () {
     if (!browserSupportsFID) this.skip();
+    if (!browserSupportsPrerender) this.skip();
 
     await navigateTo('/test/fid?prerender=1');
 

test/e2e/onINP-test.js

@@ -16,6 +16,7 @@
 
 import assert from 'assert';
 import {beaconCountIs, clearBeacons, getBeacons} from '../utils/beacons.js';
+import {browserSupportsActivationStart} from '../utils/browserSupportsActivationStart.js';
 import {browserSupportsEntry} from '../utils/browserSupportsEntry.js';
 import {navigateTo} from '../utils/navigateTo.js';
 import {nextFrame} from '../utils/nextFrame.js';
@@ -30,9 +31,11 @@ describe('onINP()', async function () {
 
   let browserSupportsINP;
   let browserSupportsLoAF;
+  let browserSupportsPrerender;
   before(async function () {
     browserSupportsINP = await browserSupportsEntry('event');
     browserSupportsLoAF = await browserSupportsEntry('long-animation-frame');
+    browserSupportsPrerender = await browserSupportsActivationStart();
   });
 
   beforeEach(async function () {
@@ -392,6 +395,7 @@ describe('onINP()', async function () {
 
   it('reports prerender as nav type for prerender', async function () {
     if (!browserSupportsINP) this.skip();
+    if (!browserSupportsPrerender) this.skip();
 
     await navigateTo('/test/inp?click=100&prerender=1', {
       readyState: 'interactive',

test/e2e/onLCP-test.js

@@ -16,37 +16,22 @@
 
 import assert from 'assert';
 import {beaconCountIs, clearBeacons, getBeacons} from '../utils/beacons.js';
+import {browserSupportsActivationStart} from '../utils/browserSupportsActivationStart.js';
 import {browserSupportsEntry} from '../utils/browserSupportsEntry.js';
 import {imagesPainted} from '../utils/imagesPainted.js';
 import {navigateTo} from '../utils/navigateTo.js';
 import {stubForwardBack} from '../utils/stubForwardBack.js';
 import {stubVisibilityChange} from '../utils/stubVisibilityChange.js';
 
-// Temp fix to address Firefox flakiness.
-// See https://github.com/GoogleChrome/web-vitals/issues/472
-const originalStrictEqual = assert.strictEqual;
-assert.strictEqual = function (actual, expected, message) {
-  if (
-    process.env.GITHUB_ACTIONS &&
-    browser.capabilities.browserName === 'firefox' &&
-    (expected === 'good' || expected === 'needs-improvement') &&
-    actual !== expected
-  ) {
-    console.error(
-      `Override assert for Firefox (actual: ${actual}, expected: ${expected})`,
-    );
-    return true;
-  }
-  return originalStrictEqual(actual, expected, message);
-};
-
 describe('onLCP()', async function () {
   // Retry all tests in this suite up to 2 times.
   this.retries(2);
 
   let browserSupportsLCP;
+  let browserSupportsPrerender;
   before(async function () {
     browserSupportsLCP = await browserSupportsEntry('largest-contentful-paint');
+    browserSupportsPrerender = await browserSupportsActivationStart();
   });
 
   beforeEach(async function () {
@@ -161,6 +146,7 @@ describe('onLCP()', async function () {
 
   it('accounts for time prerendering the page', async function () {
     if (!browserSupportsLCP) this.skip();
+    if (!browserSupportsPrerender) this.skip();
 
     await navigateTo('/test/lcp?prerender=1');
 
@@ -277,7 +263,9 @@ describe('onLCP()', async function () {
   it('stops reporting after the document changes to hidden (reportAllChanges === false)', async function () {
     if (!browserSupportsLCP) this.skip();
 
-    await navigateTo('/test/lcp?imgDelay=0&imgHidden=1');
+    await navigateTo('/test/lcp?imgDelay=0&imgHidden=1', {
+      readyState: 'interactive',
+    });
 
     // Wait for a frame to be painted.
     await browser.executeAsync((done) => requestAnimationFrame(done));
@@ -554,6 +542,7 @@ describe('onLCP()', async function () {
 
     it('accounts for time prerendering the page', async function () {
       if (!browserSupportsLCP) this.skip();
+      if (!browserSupportsPrerender) this.skip();
 
       await navigateTo('/test/lcp?attribution=1&prerender=1');
 
@@ -716,19 +705,7 @@ const assertStandardReportsAreCorrect = (beacons) => {
 const assertFullReportsAreCorrect = (beacons) => {
   const [lcp1, lcp2] = beacons;
 
-  // Temp fix to address Firefox flakiness.
-  // See https://github.com/GoogleChrome/web-vitals/issues/472
-  if (
-    process.env.GITHUB_ACTIONS &&
-    browser.capabilities.browserName === 'firefox' &&
-    lcp1.value >= 500
-  ) {
-    console.log(
-      `Override assert for Firefox (actual: ${lcp1.value}, expected: < 500)`,
-    );
-  } else {
-    assert(lcp1.value < 500); // Less than the image load delay.
-  }
+  assert(lcp1.value < 500); // Less than the image load delay.
   assert(lcp1.id.match(/^v4-\d+-\d+$/));
   assert.strictEqual(lcp1.name, 'LCP');
   assert.strictEqual(lcp1.value, lcp1.delta);

test/e2e/onTTFB-test.js

@@ -16,6 +16,7 @@
 
 import assert from 'assert';
 import {beaconCountIs, clearBeacons, getBeacons} from '../utils/beacons.js';
+import {browserSupportsActivationStart} from '../utils/browserSupportsActivationStart.js';
 import {navigateTo} from '../utils/navigateTo.js';
 import {stubForwardBack} from '../utils/stubForwardBack.js';
 
@@ -58,6 +59,11 @@ describe('onTTFB()', async function () {
   // Retry all tests in this suite up to 2 times.
   this.retries(2);
 
+  let browserSupportsPrerender;
+  before(async function () {
+    browserSupportsPrerender = await browserSupportsActivationStart();
+  });
+
   beforeEach(async function () {
     // In Safari when navigating to 'about:blank' between tests the
     // Navigation Timing data is consistently negative, so the tests fail.
@@ -122,6 +128,7 @@ describe('onTTFB()', async function () {
   });
 
   it('accounts for time prerendering the page', async function () {
+    if (!browserSupportsPrerender) this.skip();
     await navigateTo('/test/ttfb?prerender=1');
 
     const ttfb = await getTTFBBeacon();
@@ -286,6 +293,7 @@ describe('onTTFB()', async function () {
     });
 
     it('accounts for time prerendering the page', async function () {
+      if (!browserSupportsPrerender) this.skip();
       await navigateTo('/test/ttfb?attribution=1&prerender=1');
 
       const ttfb = await getTTFBBeacon();

test/utils/browserSupportsActivationStart.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Returns true if the browser supports using Prerendering
+ * @return {boolean}
+ */
+export function browserSupportsActivationStart() {
+  return browser.execute(() => {
+    return 'activationStart' in self.PerformanceNavigationTiming.prototype;
+  });
+}

wdio.conf.cjs

@@ -82,6 +82,7 @@ module.exports.config = {
     if (browserName === 'chrome') {
       capability['goog:chromeOptions'] = {
         excludeSwitches: ['enable-automation'],
+        args: ['disable-search-engine-choice-screen'],
         // Uncomment to test on Chrome Canary.
         // binary: '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary'
       };