tests(devtools): ensure Lighthouse starts in smoke tests

.github/workflows/devtools.yml

@@ -168,7 +168,7 @@ jobs:
     - run: mkdir latest-run
       working-directory: ${{ github.workspace }}/lighthouse
     - name: yarn smoke --runner devtools
-      run: xvfb-run --auto-servernum yarn smoke --runner devtools --shard=${{ matrix.smoke-test-shard }}/${{ strategy.job-total }} --jobs=3 --retries=2
+      run: xvfb-run --auto-servernum yarn smoke --runner devtools --shard=${{ matrix.smoke-test-shard }}/${{ strategy.job-total }} --jobs=3 --retries=2 --debug
       working-directory: ${{ github.workspace }}/lighthouse
 
     - name: Upload failures

cli/test/smokehouse/lighthouse-runners/devtools.js

@@ -40,15 +40,17 @@ async function setup() {
  * CHROME_PATH determines which Chrome is used–otherwise the default is puppeteer's chrome binary.
  * @param {string} url
  * @param {LH.Config=} config
+ * @param {{isDebug?: boolean}=} options
  * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
  */
-async function runLighthouse(url, config) {
+async function runLighthouse(url, config, options) {
   const chromeFlags = [
     `--custom-devtools-frontend=file://${devtoolsDir}/out/LighthouseIntegration/gen/front_end`,
   ];
   const {lhr, artifacts, logs} = await testUrlFromDevtools(url, {
     config,
     chromeFlags,
+    includedScreenshotInError: options?.isDebug,
   });
 
   const log = logs.join('') + '\n';

cli/test/smokehouse/smokehouse.js

@@ -12,8 +12,6 @@
 
 /* eslint-disable no-console */
 
-/** @typedef {import('./lib/child-process-error.js').ChildProcessError} ChildProcessError */
-
 /**
  * @typedef Run
  * @property {string[] | undefined} networkRequests
@@ -39,6 +37,7 @@ import {runLighthouse as cliLighthouseRunner} from './lighthouse-runners/cli.js'
 import {getAssertionReport} from './report-assert.js';
 import {LocalConsole} from './lib/local-console.js';
 import {ConcurrentMapper} from './lib/concurrent-mapper.js';
+import {ChildProcessError} from './lib/child-process-error.js';
 
 // The number of concurrent (`!runSerially`) tests to run if `jobs` isn't set.
 const DEFAULT_CONCURRENT_RUNS = 5;
@@ -239,11 +238,18 @@ async function runSmokeTest(smokeTestDefn, testOptions) {
  * @param {ChildProcessError|Error} err
  */
 function logChildProcessError(localConsole, err) {
-  if ('stdout' in err && 'stderr' in err) {
+  if (err instanceof ChildProcessError) {
     localConsole.adoptStdStrings(err);
   }
 
   localConsole.log(log.redify(err.stack || err.message));
+  if (err.cause) {
+    if (err.cause instanceof Error) {
+      localConsole.log(log.redify(`[cause] ${err.cause.stack || err.cause.message}`));
+    } else {
+      localConsole.log(log.redify(`[cause] ${err.cause}`));
+    }
+  }
 }
 
 /**

core/scripts/pptr-run-devtools.js

@@ -217,8 +217,19 @@ async function runLighthouse() {
     );
   });
 
-  const button = panel.contentElement.querySelector('button');
-  button.click();
+  // In CI there ins't a 100% guarantee that clicking the start button once
+  // will start the run.
+  // Therefore, keep clicking the button until we detect that the run started.
+  const intervalHandle = setInterval(() => {
+    const button = panel.contentElement.querySelector('button');
+    button.click();
+  }, 100);
+
+  addSniffer(
+    panel.__proto__,
+    'handleCompleteRun',
+    () => clearInterval(intervalHandle),
+  );
 
   return resultPromise;
 }
@@ -286,7 +297,7 @@ function dismissDialog(dialog) {
 
 /**
  * @param {string} url
- * @param {{config?: LH.Config, chromeFlags?: string[]}} [options]
+ * @param {{config?: LH.Config, chromeFlags?: string[], includedScreenshotInError?: boolean}} [options]
  * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, logs: string[]}>}
  */
 async function testUrlFromDevtools(url, options = {}) {
@@ -299,6 +310,9 @@ async function testUrlFromDevtools(url, options = {}) {
     defaultViewport: null,
   });
 
+  /** @type {puppeteer.CDPSession|undefined} */
+  let inspectorSession;
+
   try {
     if ((await browser.version()).startsWith('Headless')) {
       throw new Error('You cannot use headless');
@@ -307,7 +321,7 @@ async function testUrlFromDevtools(url, options = {}) {
     const page = (await browser.pages())[0];
 
     const inspectorTarget = await browser.waitForTarget(t => t.url().includes('devtools'));
-    const inspectorSession = await inspectorTarget.createCDPSession();
+    inspectorSession = await inspectorTarget.createCDPSession();
 
     /** @type {string[]} */
     const logs = [];
@@ -328,6 +342,16 @@ async function testUrlFromDevtools(url, options = {}) {
     const result = await evaluateInSession(inspectorSession, runLighthouse, [addSniffer]);
 
     return {...result, logs};
+  } catch (err) {
+    if (options.includedScreenshotInError && inspectorSession) {
+      const {data} = await inspectorSession.send('Page.captureScreenshot', {format: 'webp'});
+      const image = `data:image/webp;base64,${data}`;
+      throw new Error(
+        `Lighthouse in DevTool failed. DevTools screenshot:\n${image}`,
+        {cause: err}
+      );
+    }
+    throw err;
   } finally {
     await browser.close();
   }