test(runner): preserve browser tracing if test fails

vercel · Dec 10, 2023 · b4c854f · b4c854f
1 parent e02fe31
commit b4c854f
Show file tree

Hide file tree

Showing 4 changed files with 94 additions and 39 deletions.
diff --git a/.github/workflows/build_reusable.yml b/.github/workflows/build_reusable.yml
@@ -178,6 +178,7 @@ jobs:
         with:
           name: test-reports
           path: |
+            test/traces
             test/test-junit-report
             test/turbopack-test-junit-report
           if-no-files-found: ignore
diff --git a/run-tests.js b/run-tests.js
@@ -508,7 +508,8 @@ ${ENDGROUP}`)
 
       child.on('exit', async (code, signal) => {
         children.delete(child)
-        if (code !== 0 || signal !== null) {
+        const isChildExitWithNonZero = code !== 0 || signal !== null
+        if (isChildExitWithNonZero) {
           if (hideOutput) {
             await outputSema.acquire()
             const isExpanded =
@@ -550,18 +551,26 @@ ${ENDGROUP}`)
 
           return reject(err)
         }
-        await fsp
-          .rm(
-            path.join(
-              __dirname,
-              'test/traces',
-              path
-                .relative(path.join(__dirname, 'test'), test.file)
-                .replace(/\//g, '-')
-            ),
-            { recursive: true, force: true }
-          )
-          .catch(() => {})
+
+        // If environment is CI and if this test execution is failed after retry, preserve test traces
+        // to upload into github actions artifacts for debugging purpose
+        const shouldPreserveTracesOutput =
+          process.env.CI && isRetry && isChildExitWithNonZero
+        if (!shouldPreserveTracesOutput) {
+          await fsp
+            .rm(
+              path.join(
+                __dirname,
+                'test/traces',
+                path
+                  .relative(path.join(__dirname, 'test'), test.file)
+                  .replace(/\//g, '-')
+              ),
+              { recursive: true, force: true }
+            )
+            .catch(() => {})
+        }
+
         resolve(new Date().getTime() - start)
       })
     })

diff --git a/test/lib/browsers/playwright.ts b/test/lib/browsers/playwright.ts
@@ -39,6 +39,65 @@ export class Playwright extends BrowserInterface {
     request: new Set(),
   }
 
+  private async initContextTracing(
+    url: string,
+    page: Page,
+    context: BrowserContext
+  ) {
+    if (!tracePlaywright) {
+      return
+    }
+
+    try {
+      await this.teardownTracing()
+      await context.tracing.start({
+        screenshots: true,
+        snapshots: true,
+      })
+      this.activeTrace = encodeURIComponent(url)
+
+      page.on('close', async () => {
+        await this.teardownTracing()
+      })
+    } catch (e) {
+      require('console').warn('failed to start playwright trace', e)
+      this.activeTrace = undefined
+    }
+  }
+
+  private async teardownTracing() {
+    if (!tracePlaywright || !this.activeTrace) {
+      return
+    }
+
+    try {
+      const traceDir = path.join(__dirname, '../../traces')
+      const traceOutputPath = path.join(
+        traceDir,
+        `${path
+          .relative(path.join(__dirname, '../../'), process.env.TEST_FILE_PATH)
+          .replace(/\//g, '-')}`,
+        `playwright-${this.activeTrace}-${Date.now()}.zip`
+      )
+
+      await fs.remove(traceOutputPath)
+      await context.tracing
+        .stop({
+          path: traceOutputPath,
+        })
+        .catch((err) => console.error('failed to write playwright trace', err))
+
+      require('console').error('activeTrace', {
+        activeTrace: this.activeTrace,
+        traceOutputPath,
+      })
+    } catch (e) {
+      require('console').warn(e)
+    } finally {
+      this.activeTrace = undefined
+    }
+  }
+
   on(event: Event, cb: (...args: any[]) => void) {
     if (!this.eventCallbacks[event]) {
       throw new Error(
@@ -82,6 +141,9 @@ export class Playwright extends BrowserInterface {
           ignoreHTTPSErrors,
           ...device,
         })
+        context.once('close', async () => {
+          await this.teardownTracing()
+        })
         contextHasJSEnabled = javaScriptEnabled
       }
       return
@@ -94,6 +156,9 @@ export class Playwright extends BrowserInterface {
       ignoreHTTPSErrors,
       ...device,
     })
+    context.once('close', async () => {
+      await this.teardownTracing()
+    })
     contextHasJSEnabled = javaScriptEnabled
   }
 
@@ -134,30 +199,15 @@ export class Playwright extends BrowserInterface {
       beforePageLoad?: (...args: any[]) => void
     }
   ) {
-    if (this.activeTrace) {
-      const traceDir = path.join(__dirname, '../../traces')
-      const traceOutputPath = path.join(
-        traceDir,
-        `${path
-          .relative(path.join(__dirname, '../../'), process.env.TEST_FILE_PATH)
-          .replace(/\//g, '-')}`,
-        `playwright-${this.activeTrace}-${Date.now()}.zip`
-      )
-
-      await fs.remove(traceOutputPath)
-      await context.tracing
-        .stop({
-          path: traceOutputPath,
-        })
-        .catch((err) => console.error('failed to write playwright trace', err))
-    }
-
     // clean-up existing pages
     for (const oldPage of context.pages()) {
       await oldPage.close()
     }
+
     page = await context.newPage()
 
+    this.initContextTracing(url, page, context)
+
     // in development compilation can take longer due to
     // lower CPU availability in GH actions
     page.setDefaultTimeout(60 * 1000)
@@ -219,13 +269,6 @@ export class Playwright extends BrowserInterface {
 
     opts?.beforePageLoad?.(page)
 
-    if (tracePlaywright) {
-      await context.tracing.start({
-        screenshots: true,
-        snapshots: true,
-      })
-      this.activeTrace = encodeURIComponent(url)
-    }
     await page.goto(url, { waitUntil: 'load' })
   }
 

diff --git a/test/lib/next-modes/base.ts b/test/lib/next-modes/base.ts
@@ -390,7 +390,9 @@ export class NextInstance {
               `next-trace`
             )
           )
-          .catch(() => {})
+          .catch((e) => {
+            require('console').error(e)
+          })
       }
 
       if (!process.env.NEXT_TEST_SKIP_CLEANUP) {