misc: fix benchmark script

bench/vercel/bench.js

@@ -4,16 +4,24 @@ import console from 'console'
 import chalk from 'chalk'
 
 import PQueue from 'p-queue'
-import { generateProjects, cleanupProjectFolders } from './project-utils.js'
+import {
+  generateProjects,
+  cleanupProjectFolders,
+  TEST_PROJECT_NAME,
+} from './project-utils.js'
 import { printBenchmarkResults } from './chart.js'
 import { genRetryableRequest } from './gen-request.js'
 
 const program = new Command()
 
-const queue = new PQueue({ concurrency: 25 })
-const TTFB_OUTLIERS_THRESHOLD = 250
+const queue = new PQueue({ concurrency: 50 })
+const TTFB_OUTLIERS_THRESHOLD = 1500
+
+let progress = 0
 
 program.option('-p, --path <path>')
+program.option('-s, --skip-build', 'Skip build step')
+program.option('-f, --force-crash', 'Force function crash')
 
 program.parse(process.argv)
 
@@ -23,8 +31,23 @@ if (options.path) {
   console.log('Running benchmark for path: ', options.path)
 }
 
+if (options.skipBuild) {
+  console.log('Skipping build step')
+}
+
+if (options.forceCrash) {
+  console.log('Forcing function crash')
+}
+
+export const forceCrash = options.forceCrash
+
 try {
-  const [originDeploymentURL, headDeploymentURL] = await generateProjects()
+  const [originDeploymentURL, headDeploymentURL] = options.skipBuild
+    ? [
+        `https://${TEST_PROJECT_NAME}-origin.vercel.app`,
+        `https://${TEST_PROJECT_NAME}-head.vercel.app`,
+      ]
+    : await generateProjects()
 
   const originBenchmarkURL = `${originDeploymentURL}${options.path || ''}`
   const headBenchmarkURL = `${headDeploymentURL}${options.path || ''}`
@@ -60,11 +83,22 @@ try {
 }
 
 async function runBenchmark(url) {
+  progress = 0
+  process.stdout.write(`Sending requests to ${url} ...\n`)
+  process.stdout.write(`Progress: ${++progress}/500`)
   return (
     await Promise.all(
-      Array.from({ length: 500 }).map(() =>
-        queue.add(() => genRetryableRequest(url))
-      )
+      Array.from({ length: 500 }).map(async () => {
+        const p = await queue.add(() => genRetryableRequest(url))
+        refreshProgress()
+        return p
+      })
     )
   ).filter(Boolean)
 }
+
+function refreshProgress() {
+  process.stdout.clearLine()
+  process.stdout.cursorTo(0)
+  process.stdout.write(`Requests sent: ${++progress}/500`)
+}

bench/vercel/benchmark-app/app/layout.js

@@ -8,7 +8,3 @@ export default function Root({ children }) {
     </html>
   )
 }
-
-export const config = {
-  runtime: 'experimental-edge',
-}

bench/vercel/benchmark-app/app/rsc/page.js

@@ -1,14 +1,20 @@
 import * as React from 'react'
+import { cookies } from 'next/headers'
 
-// if (!('hot' in Math)) Math.hot = false
+if (!('hot' in Math)) Math.hot = false
 
 export default function page() {
-  // const previous = Math.hot
-  // Math.hot = true
-  // return <div>{previous ? 'HOT' : 'COLD'}</div>
-  return <div>hello</div>
-}
+  // make the page dynamic
+  cookies()
+  const previous = Math.hot
+  Math.hot = true
+
+  // crash the server after responding
+  if (process.env.CRASH_FUNCTION) {
+    setTimeout(() => {
+      throw new Error('crash')
+    }, 500)
+  }
 
-export const config = {
-  runtime: 'experimental-edge',
+  return <div>{previous ? 'HOT' : 'COLD'}</div>
 }

bench/vercel/benchmark-app/pages/index.js

@@ -8,13 +8,16 @@ export async function getServerSideProps() {
   const wasHot = Math.hot
   Math.hot = true
 
+  // crash the server after responding
+  if (process.env.CRASH_FUNCTION) {
+    setTimeout(() => {
+      throw new Error('crash')
+    }, 700)
+  }
+
   return {
     props: {
       hot: wasHot,
     },
   }
 }
-
-export const config = {
-  runtime: 'experimental-edge',
-}

bench/vercel/chart.js

@@ -56,6 +56,11 @@ export function printBenchmarkResults({ origin, head }, metricSelector) {
     (results) => results.map(metricSelector).filter(Boolean)
   )
 
+  if (processedHeadData.length === 0 || processedOriginData.length === 0) {
+    console.log('No data to compare, skipping')
+    return
+  }
+
   const [originMetrics, headMetrics] = [
     processedOriginData,
     processedHeadData,

bench/vercel/gen-request.js

@@ -4,12 +4,11 @@ import timer from '@szmarczak/http-timer'
 // a wrapper around genAsyncRequest that will retry the request 5 times if it fails
 export async function genRetryableRequest(url) {
   let retries = 0
-  while (retries < 5) {
+  while (retries < 10) {
     try {
       return await genAsyncRequest(url)
     } catch (err) {}
     retries++
-    await new Promise((r) => setTimeout(r, 1000))
   }
   throw new Error(`Failed to fetch ${url}, too many retries`)
 }
@@ -25,6 +24,9 @@ async function genAsyncRequest(url) {
         body += data
       })
       response.on('end', () => {
+        if (response.statusCode !== 200) {
+          reject(new Error(`Failed to fetch ${url}`))
+        }
         resolve({
           ...response.timings.phases,
           cold: !body.includes('HOT'),

bench/vercel/generate-package-json.js

@@ -46,7 +46,7 @@ async function getCurrentRootReactPackagesVersions() {
     await fs.readFile('../../package.json', 'utf8')
   )
   return {
-    react: packageJson.devDependencies['react-exp'],
-    'react-dom': packageJson.devDependencies['react-dom-exp'],
+    react: packageJson.devDependencies['react'],
+    'react-dom': packageJson.devDependencies['react-dom'],
   }
 }

bench/vercel/project-utils.js

@@ -7,10 +7,11 @@ import path from 'path'
 import url from 'url'
 import { generatePackageJson } from './generate-package-json.js'
 import { Listr } from 'listr2'
+import { forceCrash } from './bench.js'
 
 config()
 
-const TEST_PROJECT_NAME = process.env.VERCEL_TEST_PROJECT_NAME
+export const TEST_PROJECT_NAME = process.env.VERCEL_TEST_PROJECT_NAME
 const ORIGIN_PROJECT_NAME = TEST_PROJECT_NAME + '-origin'
 const HEAD_PROJECT_NAME = TEST_PROJECT_NAME + '-head'
 
@@ -197,6 +198,7 @@ export async function deployProject(projectName, appFolder) {
           : []),
         '--force',
         ...vercelFlags,
+        ...(forceCrash ? ['--env', 'CRASH_FUNCTION=1'] : []),
       ],
       {
         cwd: appFolder,