Turbopack: limit build concurrency, show progress bar

packages/next/src/build/index.ts

@@ -19,6 +19,7 @@ import { defaultConfig } from '../server/config-shared'
 import devalue from 'next/dist/compiled/devalue'
 import findUp from 'next/dist/compiled/find-up'
 import { nanoid } from 'next/dist/compiled/nanoid/index.cjs'
+import { Sema } from 'next/dist/compiled/async-sema'
 import path from 'path'
 import {
   STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR,
@@ -177,6 +178,7 @@ import {
 import { TurbopackManifestLoader } from '../server/dev/turbopack/manifest-loader'
 import type { Entrypoints } from '../server/dev/turbopack/types'
 import { buildCustomRoute } from '../lib/build-custom-route'
+import { createProgress } from './progress'
 
 interface ExperimentalBypassForInfo {
   experimentalBypassFor?: RouteHas[]
@@ -1416,9 +1418,28 @@ export default async function build(
           rewrites: emptyRewritesObjToBeImplemented,
         })
 
-        const promises = []
-        for (const [page, route] of currentEntrypoints.page) {
+        const progress = createProgress(
+          currentEntrypoints.page.size + currentEntrypoints.app.size + 1,
+          'Building'
+        )
+        const promises: Promise<any>[] = []
+        const sema = new Sema(10)
+        const enqueue = (fn: () => Promise<void>) => {
           promises.push(
+            (async () => {
+              await sema.acquire()
+              try {
+                await fn()
+              } finally {
+                sema.release()
+                progress()
+              }
+            })()
+          )
+        }
+
+        for (const [page, route] of currentEntrypoints.page) {
+          enqueue(() =>
             handleRouteType({
               dev,
               page,
@@ -1434,7 +1455,7 @@ export default async function build(
         }
 
         for (const [page, route] of currentEntrypoints.app) {
-          promises.push(
+          enqueue(() =>
             handleRouteType({
               page,
               dev: false,
@@ -1448,7 +1469,7 @@ export default async function build(
           )
         }
 
-        promises.push(
+        enqueue(() =>
           handlePagesErrorRoute({
             currentIssues,
             entrypoints: currentEntrypoints,

packages/next/src/build/progress.ts

@@ -0,0 +1,86 @@
+import * as Log from '../build/output/log'
+import createSpinner from './spinner'
+
+function divideSegments(number: number, segments: number): number[] {
+  const result = []
+  while (number > 0 && segments > 0) {
+    const dividedNumber =
+      number < segments ? number : Math.floor(number / segments)
+
+    number -= dividedNumber
+    segments--
+    result.push(dividedNumber)
+  }
+  return result
+}
+
+export const createProgress = (total: number, label: string) => {
+  const segments = divideSegments(total, 4)
+
+  if (total === 0) {
+    throw new Error('invariant: progress total can not be zero')
+  }
+  let currentSegmentTotal = segments.shift()
+  let currentSegmentCount = 0
+  let lastProgressOutput = Date.now()
+  let curProgress = 0
+  let progressSpinner = createSpinner(`${label} (${curProgress}/${total})`, {
+    spinner: {
+      frames: [
+        '[    ]',
+        '[=   ]',
+        '[==  ]',
+        '[=== ]',
+        '[ ===]',
+        '[  ==]',
+        '[   =]',
+        '[    ]',
+        '[   =]',
+        '[  ==]',
+        '[ ===]',
+        '[====]',
+        '[=== ]',
+        '[==  ]',
+        '[=   ]',
+      ],
+      interval: 200,
+    },
+  })
+
+  return () => {
+    curProgress++
+
+    // Make sure we only log once
+    // - per fully generated segment, or
+    // - per minute
+    // when not showing the spinner
+    if (!progressSpinner) {
+      currentSegmentCount++
+
+      if (currentSegmentCount === currentSegmentTotal) {
+        currentSegmentTotal = segments.shift()
+        currentSegmentCount = 0
+      } else if (lastProgressOutput + 60000 > Date.now()) {
+        return
+      }
+
+      lastProgressOutput = Date.now()
+    }
+
+    const isFinished = curProgress === total
+    if (progressSpinner && !isFinished) {
+      progressSpinner.setText(`${label} (${curProgress}/${total})`)
+    } else {
+      if (progressSpinner) {
+        progressSpinner.stop()
+      }
+      console.log(
+        ` ${
+          isFinished ? Log.prefixes.event : Log.prefixes.info
+        } ${label} (${curProgress}/${total}) ${
+          isFinished ? '' : process.stdout.isTTY ? '\n' : '\r'
+        }`
+      )
+    }
+  }
+}

packages/next/src/build/spinner.ts

@@ -11,11 +11,9 @@ export default function createSpinner(
   options: ora.Options = {},
   logFn: (...data: any[]) => void = console.log
 ) {
-  let spinner: undefined | ora.Ora
+  let spinner: undefined | (ora.Ora & { setText: (text: string) => void })
 
-  const prefixText = ` ${Log.prefixes.info} ${text} `
-  // Add \r at beginning to reset the current line of loading status text
-  const suffixText = `\r ${Log.prefixes.event} ${text} `
+  let prefixText = ` ${Log.prefixes.info} ${text} `
 
   if (process.stdout.isTTY) {
     spinner = ora({
@@ -24,7 +22,7 @@ export default function createSpinner(
       spinner: dotsSpinner,
       stream: process.stdout,
       ...options,
-    }).start()
+    }).start() as ora.Ora & { setText: (text: string) => void }
 
     // Add capturing of console.log/warn/error to allow pausing
     // the spinner before logging and then restarting spinner after
@@ -49,18 +47,24 @@ export default function createSpinner(
       console.warn = origWarn
       console.error = origError
     }
+    spinner.setText = (newText: string) => {
+      text = newText
+      prefixText = ` ${Log.prefixes.info} ${newText} `
+      spinner!.prefixText = prefixText
+      return spinner!
+    }
     spinner.stop = (): ora.Ora => {
       origStop()
       resetLog()
       return spinner!
     }
     spinner.stopAndPersist = (): ora.Ora => {
-      if (suffixText) {
-        if (spinner) {
-          spinner.text = suffixText
-        } else {
-          logFn(suffixText)
-        }
+      // Add \r at beginning to reset the current line of loading status text
+      const suffixText = `\r ${Log.prefixes.event} ${text} `
+      if (spinner) {
+        spinner.text = suffixText
+      } else {
+        logFn(suffixText)
       }
       origStopAndPersist()
       resetLog()

packages/next/src/export/index.ts

@@ -18,7 +18,6 @@ import { dirname, join, resolve, sep } from 'path'
 import { formatAmpMessages } from '../build/output/index'
 import type { AmpPageStatus } from '../build/output/index'
 import * as Log from '../build/output/log'
-import createSpinner from '../build/spinner'
 import { RSC_SUFFIX, SSG_FALLBACK_EXPORT_ERROR } from '../lib/constants'
 import { recursiveCopy } from '../lib/recursive-copy'
 import {
@@ -56,93 +55,7 @@ import { needsExperimentalReact } from '../lib/needs-experimental-react'
 import { formatManifest } from '../build/manifests/formatter/format-manifest'
 import { validateRevalidate } from '../server/lib/patch-fetch'
 import { TurborepoAccessTraceResult } from '../build/turborepo-access-trace'
-
-function divideSegments(number: number, segments: number): number[] {
-  const result = []
-  while (number > 0 && segments > 0) {
-    const dividedNumber =
-      number < segments ? number : Math.floor(number / segments)
-
-    number -= dividedNumber
-    segments--
-    result.push(dividedNumber)
-  }
-  return result
-}
-
-const createProgress = (total: number, label: string) => {
-  const segments = divideSegments(total, 4)
-
-  if (total === 0) {
-    throw new Error('invariant: progress total can not be zero')
-  }
-  let currentSegmentTotal = segments.shift()
-  let currentSegmentCount = 0
-  let lastProgressOutput = Date.now()
-  let curProgress = 0
-  let progressSpinner = createSpinner(`${label} (${curProgress}/${total})`, {
-    spinner: {
-      frames: [
-        '[    ]',
-        '[=   ]',
-        '[==  ]',
-        '[=== ]',
-        '[ ===]',
-        '[  ==]',
-        '[   =]',
-        '[    ]',
-        '[   =]',
-        '[  ==]',
-        '[ ===]',
-        '[====]',
-        '[=== ]',
-        '[==  ]',
-        '[=   ]',
-      ],
-      interval: 200,
-    },
-  })
-
-  return () => {
-    curProgress++
-
-    // Make sure we only log once
-    // - per fully generated segment, or
-    // - per minute
-    // when not showing the spinner
-    if (!progressSpinner) {
-      currentSegmentCount++
-
-      if (currentSegmentCount === currentSegmentTotal) {
-        currentSegmentTotal = segments.shift()
-        currentSegmentCount = 0
-      } else if (lastProgressOutput + 60000 > Date.now()) {
-        return
-      }
-
-      lastProgressOutput = Date.now()
-    }
-
-    const isFinished = curProgress === total
-    // Use \r to reset current line with spinner.
-    // If it's 100% progressed, then we don't need to break a new line to avoid logging from routes while building.
-    const newText = `\r ${
-      isFinished ? Log.prefixes.event : Log.prefixes.info
-    } ${label} (${curProgress}/${total}) ${
-      isFinished ? '' : process.stdout.isTTY ? '\n' : '\r'
-    }`
-    if (progressSpinner) {
-      progressSpinner.text = newText
-    } else {
-      console.log(newText)
-    }
-
-    if (isFinished && progressSpinner) {
-      progressSpinner.stop()
-      console.log(newText)
-    }
-  }
-}
+import { createProgress } from '../build/progress'
 
 export class ExportError extends Error {
   code = 'NEXT_EXPORT_ERROR'