Add initial separated route resolving (#47208)

This updates to have a separate routing process and separate rendering
processes for `pages` and `app` so that we can properly isolate the two
since they rely on different react versions.

Besides allowing the above mentioned isolation this also helps us
control recovering from process crashes easier as pieces are more
isolated from one another e.g. an infinite loop during rendering will no
longer block the compiler and can be stopped/restarted as needed.

In follow-up PRs we will continue to separate out the routing logic from
the rendering logic so that each process only loads what is relevant to
it helping simplify the flow for requests regardless of type.

---------

Co-authored-by: Shu Ding <g@shud.in>
Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

packages/next-env/index.ts

@@ -10,7 +10,7 @@ export type LoadedEnvFiles = Array<{
   contents: string
 }>
 
-let initialEnv: Env | undefined = undefined
+export let initialEnv: Env | undefined = undefined
 let combinedEnv: Env | undefined = undefined
 let cachedLoadedEnvFiles: LoadedEnvFiles = []
 let previousLoadedEnvFiles: LoadedEnvFiles = []

packages/next/src/build/index.ts

@@ -1783,79 +1783,7 @@ export default async function build(
         'next-server.js.nft.json'
       )
 
-      if (config.experimental.preCompiledNextServer) {
-        if (!ciEnvironment.hasNextSupport) {
-          Log.warn(
-            `"experimental.preCompiledNextServer" is currently optimized for environments with Next.js support, some features may not be supported`
-          )
-        }
-        const nextServerPath = require.resolve('next/dist/server/next-server')
-        await promises.rename(nextServerPath, `${nextServerPath}.bak`)
-
-        await promises.writeFile(
-          nextServerPath,
-          `module.exports = require('next/dist/compiled/next-server/next-server.js')`
-        )
-        const glob =
-          require('next/dist/compiled/glob') as typeof import('next/dist/compiled/glob')
-        const compiledFiles: string[] = []
-        const compiledNextServerFolder = path.dirname(
-          require.resolve('next/dist/compiled/next-server/next-server.js')
-        )
-
-        for (const compiledFolder of [
-          compiledNextServerFolder,
-          path.join(compiledNextServerFolder, '../react'),
-          path.join(compiledNextServerFolder, '../react-dom'),
-          path.join(compiledNextServerFolder, '../scheduler'),
-        ]) {
-          const globResult = glob.sync('**/*', {
-            cwd: compiledFolder,
-            dot: true,
-          })
-
-          await Promise.all(
-            globResult.map(async (file) => {
-              const absolutePath = path.join(compiledFolder, file)
-              const statResult = await promises.stat(absolutePath)
-
-              if (statResult.isFile()) {
-                compiledFiles.push(absolutePath)
-              }
-            })
-          )
-        }
-
-        const externalLibFiles = [
-          'next/dist/shared/lib/server-inserted-html.js',
-          'next/dist/shared/lib/router-context.js',
-          'next/dist/shared/lib/loadable-context.js',
-          'next/dist/shared/lib/image-config-context.js',
-          'next/dist/shared/lib/image-config.js',
-          'next/dist/shared/lib/head-manager-context.js',
-          'next/dist/shared/lib/app-router-context.js',
-          'next/dist/shared/lib/amp-context.js',
-          'next/dist/shared/lib/hooks-client-context.js',
-          'next/dist/shared/lib/html-context.js',
-        ]
-        for (const file of externalLibFiles) {
-          compiledFiles.push(require.resolve(file))
-        }
-        compiledFiles.push(nextServerPath)
-
-        await promises.writeFile(
-          nextServerTraceOutput,
-          JSON.stringify({
-            version: 1,
-            files: [...new Set(compiledFiles)].map((file) =>
-              path.relative(distDir, file)
-            ),
-          } as {
-            version: number
-            files: string[]
-          })
-        )
-      } else if (config.outputFileTracing) {
+      if (config.outputFileTracing) {
         let nodeFileTrace: any
         if (config.experimental.turbotrace) {
           if (!binding?.isWasm) {
@@ -2064,6 +1992,7 @@ export default async function build(
               '**/*.d.ts',
               '**/*.map',
               '**/next/dist/pages/**/*',
+              '**/next/dist/compiled/jest-worker/**/*',
               '**/next/dist/compiled/webpack/(bundle4|bundle5).js',
               '**/node_modules/webpack5/**/*',
               '**/next/dist/server/lib/squoosh/**/*.wasm',
@@ -2082,7 +2011,14 @@ export default async function build(
               ...additionalIgnores,
             ]
             const ignoreFn = (pathname: string) => {
-              return isMatch(pathname, ignores, { contains: true, dot: true })
+              if (path.isAbsolute(pathname) && !pathname.startsWith(root)) {
+                return true
+              }
+
+              return isMatch(pathname, ignores, {
+                contains: true,
+                dot: true,
+              })
             }
             const traceContext = path.join(nextServerEntry, '..', '..')
             const tracedFiles = new Set<string>()

packages/next/src/build/output/index.ts

@@ -95,7 +95,14 @@ export function formatAmpMessages(amp: AmpPageStatus) {
   return output
 }
 
-const buildStore = createStore<BuildStatusStore>()
+const buildStore = createStore<BuildStatusStore>({
+  // @ts-expect-error initial value
+  client: {},
+  // @ts-expect-error initial value
+  server: {},
+  // @ts-expect-error initial value
+  edgeServer: {},
+})
 let buildWasDone = false
 let clientWasLoading = true
 let serverWasLoading = true

packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts

@@ -16,7 +16,30 @@ const originModules = [
 
 const RUNTIME_NAMES = ['webpack-runtime', 'webpack-api-runtime']
 
-function deleteCache(filePath: string) {
+export function deleteAppClientCache() {
+  if ((global as any)._nextDeleteAppClientCache) {
+    ;(global as any)._nextDeleteAppClientCache()
+  }
+  // ensure we reset the cache for sc_server components
+  // loaded via react-server-dom-webpack
+  const reactServerDomModId = require.resolve(
+    'next/dist/compiled/react-server-dom-webpack/client.edge'
+  )
+  const reactServerDomMod = require.cache[reactServerDomModId]
+
+  if (reactServerDomMod) {
+    for (const child of reactServerDomMod.children) {
+      child.parent = null
+      delete require.cache[child.id]
+    }
+  }
+  delete require.cache[reactServerDomModId]
+}
+
+export function deleteCache(filePath: string) {
+  if ((global as any)._nextDeleteCache) {
+    ;(global as any)._nextDeleteCache(filePath)
+  }
   try {
     filePath = realpathSync(filePath)
   } catch (e) {
@@ -82,20 +105,6 @@ export class NextJsRequireCacheHotReloader implements WebpackPluginInstance {
       })
 
       if (hasAppPath) {
-        // ensure we reset the cache for sc_server components
-        // loaded via react-server-dom-webpack
-        const reactServerDomModId = require.resolve(
-          'next/dist/compiled/react-server-dom-webpack/client.edge'
-        )
-        const reactServerDomMod = require.cache[reactServerDomModId]
-
-        if (reactServerDomMod) {
-          for (const child of reactServerDomMod.children) {
-            child.parent = null
-            delete require.cache[child.id]
-          }
-        }
-        delete require.cache[reactServerDomModId]
       }
 
       entries.forEach((page) => {