Fix isolated tests on windows and update azure config

azure-pipelines.yml

@@ -109,7 +109,7 @@ stages:
             condition: eq(variables['isDocsOnly'], 'No')
             displayName: 'Run tests'
 
-      - job: test_integration_app_dir
+      - job: test_e2e_dev
         pool:
           vmImage: 'windows-2019'
         steps:
@@ -139,6 +139,44 @@ stages:
             condition: eq(variables['isDocsOnly'], 'No')
 
           - script: |
-              node run-tests.js -c 1 test/integration/app-dir-basic/test/index.test.js
+              node run-tests.js -c 1 --debug test/e2e/app-dir/app/index.test.ts
             condition: eq(variables['isDocsOnly'], 'No')
-            displayName: 'Run tests'
+            displayName: 'Run tests (E2E Development)'
+            env:
+              NEXT_TEST_MODE: 'dev'
+
+      - job: test_e2e_prod
+        pool:
+          vmImage: 'windows-2019'
+        steps:
+          - task: NodeTool@0
+            inputs:
+              versionSpec: $(node_16_version)
+            displayName: 'Install Node.js'
+
+          - bash: |
+              node scripts/run-for-change.js --not --type docs --exec echo "##vso[task.setvariable variable=isDocsOnly]No"
+            displayName: 'Check Docs Only Change'
+
+          - script: npm i -g pnpm@$(PNPM_VERSION)
+            condition: eq(variables['isDocsOnly'], 'No')
+
+          - script: pnpm config set store-dir $(PNPM_CACHE_FOLDER)
+            condition: eq(variables['isDocsOnly'], 'No')
+
+          - script: pnpm store path
+            condition: eq(variables['isDocsOnly'], 'No')
+
+          - script: pnpm install && pnpm run build
+            condition: eq(variables['isDocsOnly'], 'No')
+            displayName: 'Install and build'
+
+          - script: npx playwright install chromium
+            condition: eq(variables['isDocsOnly'], 'No')
+
+          - script: |
+              node run-tests.js -c 1 --debug test/e2e/app-dir/app/index.test.ts
+            condition: eq(variables['isDocsOnly'], 'No')
+            displayName: 'Run tests (E2E Production)'
+            env:
+              NEXT_TEST_MODE: 'start'

run-tests.js

@@ -49,11 +49,22 @@ const cleanUpAndExit = async (code) => {
 }
 
 async function getTestTimings() {
-  const timingsRes = await fetch(TIMINGS_API, {
-    headers: {
-      ...TIMINGS_API_HEADERS,
-    },
-  })
+  let timingsRes
+
+  const doFetch = () =>
+    fetch(TIMINGS_API, {
+      headers: {
+        ...TIMINGS_API_HEADERS,
+      },
+    })
+  timingsRes = await doFetch()
+
+  if (timingsRes.status === 403) {
+    const delay = 15
+    console.log(`Got 403 response waiting ${delay} seconds before retry`)
+    await new Promise((resolve) => setTimeout(resolve, delay * 1000))
+    timingsRes = await doFetch()
+  }
 
   if (!timingsRes.ok) {
     throw new Error(`request status: ${timingsRes.status}`)
@@ -219,6 +230,7 @@ async function main() {
   })
 
   if (
+    process.platform !== 'win32' &&
     process.env.NEXT_TEST_MODE !== 'deploy' &&
     ((testType && testType !== 'unit') || hasIsolatedTests)
   ) {

test/lib/next-modes/next-dev.ts

@@ -1,4 +1,4 @@
-import { spawn } from 'child_process'
+import { spawn } from 'cross-spawn'
 import { Span } from 'next/trace'
 import { NextInstance } from './base'
 
@@ -36,65 +36,70 @@ export class NextDevInstance extends NextInstance {
     }
 
     await new Promise<void>((resolve, reject) => {
-      this.childProcess = spawn(startArgs[0], startArgs.slice(1), {
-        cwd: useDirArg ? process.cwd() : this.testDir,
-        stdio: ['ignore', 'pipe', 'pipe'],
-        shell: false,
-        env: {
-          ...process.env,
-          ...this.env,
-          NODE_ENV: '' as any,
-          PORT: this.forcedPort || '0',
-          __NEXT_TEST_MODE: '1',
-          __NEXT_TEST_WITH_DEVTOOL: '1',
-        },
-      })
+      try {
+        this.childProcess = spawn(startArgs[0], startArgs.slice(1), {
+          cwd: useDirArg ? process.cwd() : this.testDir,
+          stdio: ['ignore', 'pipe', 'pipe'],
+          shell: false,
+          env: {
+            ...process.env,
+            ...this.env,
+            NODE_ENV: '' as any,
+            PORT: this.forcedPort || '0',
+            __NEXT_TEST_MODE: '1',
+            __NEXT_TEST_WITH_DEVTOOL: '1',
+          },
+        })
 
-      this._cliOutput = ''
+        this._cliOutput = ''
 
-      this.childProcess.stdout.on('data', (chunk) => {
-        const msg = chunk.toString()
-        process.stdout.write(chunk)
-        this._cliOutput += msg
-        this.emit('stdout', [msg])
-      })
-      this.childProcess.stderr.on('data', (chunk) => {
-        const msg = chunk.toString()
-        process.stderr.write(chunk)
-        this._cliOutput += msg
-        this.emit('stderr', [msg])
-      })
+        this.childProcess.stdout.on('data', (chunk) => {
+          const msg = chunk.toString()
+          process.stdout.write(chunk)
+          this._cliOutput += msg
+          this.emit('stdout', [msg])
+        })
+        this.childProcess.stderr.on('data', (chunk) => {
+          const msg = chunk.toString()
+          process.stderr.write(chunk)
+          this._cliOutput += msg
+          this.emit('stderr', [msg])
+        })
 
-      this.childProcess.on('close', (code, signal) => {
-        if (this.isStopping) return
-        if (code || signal) {
-          throw new Error(
-            `next dev exited unexpectedly with code/signal ${code || signal}`
-          )
-        }
-      })
-      const readyCb = (msg) => {
-        if (msg.includes('started server on') && msg.includes('url:')) {
-          // turbo devserver emits stdout in rust directly, can contain unexpected chars with color codes
-          // strip out again for the safety
-          this._url = msg
-            .split('url: ')
-            .pop()
-            .trim()
-            .split(require('os').EOL)[0]
-          try {
-            this._parsedUrl = new URL(this._url)
-          } catch (err) {
-            reject({
-              err,
-              msg,
-            })
+        this.childProcess.on('close', (code, signal) => {
+          if (this.isStopping) return
+          if (code || signal) {
+            throw new Error(
+              `next dev exited unexpectedly with code/signal ${code || signal}`
+            )
+          }
+        })
+        const readyCb = (msg) => {
+          if (msg.includes('started server on') && msg.includes('url:')) {
+            // turbo devserver emits stdout in rust directly, can contain unexpected chars with color codes
+            // strip out again for the safety
+            this._url = msg
+              .split('url: ')
+              .pop()
+              .trim()
+              .split(require('os').EOL)[0]
+            try {
+              this._parsedUrl = new URL(this._url)
+            } catch (err) {
+              reject({
+                err,
+                msg,
+              })
+            }
+            this.off('stdout', readyCb)
+            resolve()
           }
-          this.off('stdout', readyCb)
-          resolve()
         }
+        this.on('stdout', readyCb)
+      } catch (err) {
+        require('console').error(`Failed to run ${startArgs.join(' ')}`, err)
+        setTimeout(() => process.exit(1), 0)
       }
-      this.on('stdout', readyCb)
     })
   }
 }

test/lib/next-modes/next-start.ts

@@ -1,7 +1,7 @@
 import path from 'path'
 import fs from 'fs-extra'
 import { NextInstance } from './base'
-import { spawn, SpawnOptions } from 'child_process'
+import { spawn, SpawnOptions } from 'cross-spawn'
 import { Span } from 'next/trace'
 
 export class NextStartInstance extends NextInstance {
@@ -65,20 +65,26 @@ export class NextStartInstance extends NextInstance {
 
     await new Promise<void>((resolve, reject) => {
       console.log('running', buildArgs.join(' '))
-      this.childProcess = spawn(
-        buildArgs[0],
-        buildArgs.slice(1),
-        this.spawnOpts
-      )
-      this.handleStdio(this.childProcess)
-      this.childProcess.on('exit', (code, signal) => {
-        this.childProcess = null
-        if (code || signal)
-          reject(
-            new Error(`next build failed with code/signal ${code || signal}`)
-          )
-        else resolve()
-      })
+
+      try {
+        this.childProcess = spawn(
+          buildArgs[0],
+          buildArgs.slice(1),
+          this.spawnOpts
+        )
+        this.handleStdio(this.childProcess)
+        this.childProcess.on('exit', (code, signal) => {
+          this.childProcess = null
+          if (code || signal)
+            reject(
+              new Error(`next build failed with code/signal ${code || signal}`)
+            )
+          else resolve()
+        })
+      } catch (err) {
+        require('console').error(`Failed to run ${buildArgs.join(' ')}`, err)
+        setTimeout(() => process.exit(1), 0)
+      }
     })
 
     this._buildId = (
@@ -95,31 +101,38 @@ export class NextStartInstance extends NextInstance {
     console.log('running', startArgs.join(' '))
 
     await new Promise<void>((resolve) => {
-      this.childProcess = spawn(
-        startArgs[0],
-        startArgs.slice(1),
-        this.spawnOpts
-      )
-      this.handleStdio(this.childProcess)
-
-      this.childProcess.on('close', (code, signal) => {
-        if (this.isStopping) return
-        if (code || signal) {
-          throw new Error(
-            `next start exited unexpectedly with code/signal ${code || signal}`
-          )
-        }
-      })
+      try {
+        this.childProcess = spawn(
+          startArgs[0],
+          startArgs.slice(1),
+          this.spawnOpts
+        )
+        this.handleStdio(this.childProcess)
+
+        this.childProcess.on('close', (code, signal) => {
+          if (this.isStopping) return
+          if (code || signal) {
+            throw new Error(
+              `next start exited unexpectedly with code/signal ${
+                code || signal
+              }`
+            )
+          }
+        })
 
-      const readyCb = (msg) => {
-        if (msg.includes('started server on') && msg.includes('url:')) {
-          this._url = msg.split('url: ').pop().trim()
-          this._parsedUrl = new URL(this._url)
-          this.off('stdout', readyCb)
-          resolve()
+        const readyCb = (msg) => {
+          if (msg.includes('started server on') && msg.includes('url:')) {
+            this._url = msg.split('url: ').pop().trim()
+            this._parsedUrl = new URL(this._url)
+            this.off('stdout', readyCb)
+            resolve()
+          }
         }
+        this.on('stdout', readyCb)
+      } catch (err) {
+        require('console').error(`Failed to run ${startArgs.join(' ')}`, err)
+        setTimeout(() => process.exit(1), 0)
       }
-      this.on('stdout', readyCb)
     })
   }