From bae886c2e44fb4e7efdd6a9ad4f81f37177ab9a8 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Mon, 12 Feb 2024 10:28:23 +0100 Subject: [PATCH 1/9] Initial Turbopack build --- packages/next/src/build/index.ts | 50 +++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 28b5cf8edc4ec..050ba4e74066f 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -127,9 +127,11 @@ import { isEdgeRuntime } from '../lib/is-edge-runtime' import { recursiveCopy } from '../lib/recursive-copy' import { recursiveReadDir } from '../lib/recursive-readdir' import { + loadBindings, lockfilePatchPromise, teardownTraceSubscriber, teardownHeapProfiler, + createDefineEnv, } from './swc' import { getNamedRouteRegex } from '../shared/lib/router/utils/route-regex' import { getFilesInDir } from '../lib/get-files-in-dir' @@ -165,6 +167,7 @@ import { interopDefault } from '../lib/interop-default' import { formatDynamicImportPath } from '../lib/format-dynamic-import-path' import { isInterceptionRouteAppPath } from '../server/future/helpers/interception-routes' import { buildCustomRoute } from '../lib/build-custom-route' +import { getTurbopackJsConfig } from '../server/dev/hot-reloader-turbopack' interface ExperimentalBypassForInfo { experimentalBypassFor?: RouteHas[] @@ -720,6 +723,13 @@ export default async function build( .traceAsyncFn(() => loadCustomRoutes(config)) const { headers, rewrites, redirects } = customRoutes + const combinedRewrites: Rewrite[] = [ + ...rewrites.beforeFiles, + ...rewrites.afterFiles, + ...rewrites.fallback, + ] + const hasRewrites = combinedRewrites.length > 0 + NextBuildContext.originalRewrites = config._originalRewrites NextBuildContext.originalRedirects = config._originalRedirects @@ -1132,12 +1142,6 @@ export default async function build( } } - const combinedRewrites: Rewrite[] = [ - ...rewrites.beforeFiles, - ...rewrites.afterFiles, - ...rewrites.fallback, - ] - if (config.experimental.clientRouterFilter) { const nonInternalRedirects = (config._originalRedirects || []).filter( (r: any) => !r.internal @@ -1305,8 +1309,38 @@ export default async function build( return serverFilesManifest }) - async function turbopackBuild(): Promise { - throw new Error("next build doesn't support turbopack yet") + async function turbopackBuild(): Promise { + if (!process.env.TURBOPACK || !process.env.TURBOPACK_BUILD) { + throw new Error("next build doesn't support turbopack yet") + } + const bindings = await loadBindings(config?.experimental?.useWasmBinary) + const project = await bindings.turbo.createProject({ + projectPath: dir, + rootPath: config.experimental.outputFileTracingRoot || dir, + nextConfig: config.nextConfig, + jsConfig: await getTurbopackJsConfig(dir, config), + watch: true, + env: process.env as Record, + defineEnv: createDefineEnv({ + isTurbopack: true, + allowedRevalidateHeaderKeys: undefined, + clientRouterFilters: undefined, + config, + dev: true, + distDir, + fetchCacheKeyPrefix: undefined, + hasRewrites, + middlewareMatchers: undefined, + previewModeId: undefined, + }), + // TODO: fix + // serverAddr: `127.0.0.1:${opts.port}`, + serverAddr: `127.0.0.1:3000`, + }) + + const iter = project.entrypointsSubscribe() + // for await (const entrypoints of iter) { + // } } let buildTraceContext: undefined | BuildTraceContext let buildTracesPromise: Promise | undefined = undefined From 06cf2a51e5c2295f66171a7c2b3d60a16a916757 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Mon, 12 Feb 2024 10:28:40 +0100 Subject: [PATCH 2/9] Refactor hot-reloader-turbopack further --- .../src/server/dev/hot-reloader-turbopack.ts | 675 +++++++++--------- 1 file changed, 348 insertions(+), 327 deletions(-) diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index d42eff342a3af..4406ecbc08bde 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -93,6 +93,7 @@ import { type ServerFields, type SetupOpts, } from '../lib/router-utils/setup-dev-bundler' +import type { NextConfigComplete } from '../config-shared' const MILLISECONDS_IN_NANOSECOND = 1_000_000 const wsServer = new ws.Server({ noServer: true }) @@ -104,6 +105,126 @@ const isTestMode = !!( class ModuleBuildError extends Error {} +export async function getTurbopackJsConfig( + dir: string, + nextConfig: NextConfigComplete +) { + const { jsConfig } = await loadJsConfig(dir, nextConfig) + return jsConfig ?? { compilerOptions: {} } +} + +function issueKey(issue: Issue): string { + return [ + issue.severity, + issue.filePath, + JSON.stringify(issue.title), + JSON.stringify(issue.description), + ].join('-') +} + +function formatIssue(issue: Issue) { + const { filePath, title, description, source } = issue + let { documentationLink } = issue + let formattedTitle = renderStyledStringToErrorAnsi(title).replace( + /\n/g, + '\n ' + ) + + // TODO: Use error codes to identify these + // TODO: Generalize adapting Turbopack errors to Next.js errors + if (formattedTitle.includes('Module not found')) { + // For compatiblity with webpack + // TODO: include columns in webpack errors. + documentationLink = 'https://nextjs.org/docs/messages/module-not-found' + } + + let formattedFilePath = filePath + .replace('[project]/', './') + .replaceAll('/./', '/') + .replace('\\\\?\\', '') + + let message + + if (source && source.range) { + const { start } = source.range + message = `${formattedFilePath}:${start.line + 1}:${ + start.column + 1 + }\n${formattedTitle}` + } else if (formattedFilePath) { + message = `${formattedFilePath}\n${formattedTitle}` + } else { + message = formattedTitle + } + message += '\n' + + if (source?.range && source.source.content) { + const { start, end } = source.range + const { codeFrameColumns } = require('next/dist/compiled/babel/code-frame') + + message += + codeFrameColumns( + source.source.content, + { + start: { + line: start.line + 1, + column: start.column + 1, + }, + end: { + line: end.line + 1, + column: end.column + 1, + }, + }, + { forceColor: true } + ).trim() + '\n\n' + } + + if (description) { + message += renderStyledStringToErrorAnsi(description) + '\n\n' + } + + // TODO: make it possible to enable this for debugging, but not in tests. + // if (detail) { + // message += renderStyledStringToErrorAnsi(detail) + '\n\n' + // } + + // TODO: Include a trace from the issue. + + if (documentationLink) { + message += documentationLink + '\n\n' + } + + return message +} + +type Issues = Map> + +function processIssues( + issues: Issues, + name: string, + result: TurbopackResult, + throwIssue = false +) { + const newIssues = new Map() + issues.set(name, newIssues) + + const relevantIssues = new Set() + + for (const issue of result.issues) { + if (issue.severity !== 'error' && issue.severity !== 'fatal') continue + const key = issueKey(issue) + const formatted = formatIssue(issue) + newIssues.set(key, issue) + + // We show errors in node_modules to the console, but don't throw for them + if (/(^|\/)node_modules(\/|$)/.test(issue.filePath)) continue + relevantIssues.add(formatted) + } + + if (relevantIssues.size && throwIssue) { + throw new ModuleBuildError([...relevantIssues].join('\n\n')) + } +} + export async function createHotReloaderTurbopack( opts: SetupOpts, serverFields: ServerFields, @@ -116,8 +237,6 @@ export async function createHotReloaderTurbopack( let bindings = await loadBindings() - const { jsConfig } = await loadJsConfig(dir, opts.nextConfig) - // For the debugging purpose, check if createNext or equivalent next instance setup in test cases // works correctly. Normally `run-test` hides output so only will be visible when `--debug` flag is used. if (process.env.TURBOPACK && isTestMode) { @@ -143,7 +262,7 @@ export async function createHotReloaderTurbopack( projectPath: dir, rootPath: opts.nextConfig.experimental.outputFileTracingRoot || dir, nextConfig: opts.nextConfig, - jsConfig: jsConfig ?? { compilerOptions: {} }, + jsConfig: await getTurbopackJsConfig(dir, nextConfig), watch: true, env: process.env as Record, defineEnv: createDefineEnv({ @@ -161,7 +280,7 @@ export async function createHotReloaderTurbopack( serverAddr: `127.0.0.1:${opts.port}`, }) const iter = project.entrypointsSubscribe() - const curEntries: Map = new Map() + const currentEntries: Map = new Map() const changeSubscriptions: Map< string, Promise> @@ -183,119 +302,7 @@ export async function createHotReloaderTurbopack( const hmrPayloads = new Map() const turbopackUpdates: TurbopackUpdate[] = [] - const issues = new Map>() - - function issueKey(issue: Issue): string { - return [ - issue.severity, - issue.filePath, - JSON.stringify(issue.title), - JSON.stringify(issue.description), - ].join('-') - } - - function formatIssue(issue: Issue) { - const { filePath, title, description, source } = issue - let { documentationLink } = issue - let formattedTitle = renderStyledStringToErrorAnsi(title).replace( - /\n/g, - '\n ' - ) - - // TODO: Use error codes to identify these - // TODO: Generalize adapting Turbopack errors to Next.js errors - if (formattedTitle.includes('Module not found')) { - // For compatiblity with webpack - // TODO: include columns in webpack errors. - documentationLink = 'https://nextjs.org/docs/messages/module-not-found' - } - - let formattedFilePath = filePath - .replace('[project]/', './') - .replaceAll('/./', '/') - .replace('\\\\?\\', '') - - let message - - if (source && source.range) { - const { start } = source.range - message = `${formattedFilePath}:${start.line + 1}:${ - start.column + 1 - }\n${formattedTitle}` - } else if (formattedFilePath) { - message = `${formattedFilePath}\n${formattedTitle}` - } else { - message = formattedTitle - } - message += '\n' - - if (source?.range && source.source.content) { - const { start, end } = source.range - const { - codeFrameColumns, - } = require('next/dist/compiled/babel/code-frame') - - message += - codeFrameColumns( - source.source.content, - { - start: { - line: start.line + 1, - column: start.column + 1, - }, - end: { - line: end.line + 1, - column: end.column + 1, - }, - }, - { forceColor: true } - ).trim() + '\n\n' - } - - if (description) { - message += renderStyledStringToErrorAnsi(description) + '\n\n' - } - - // TODO: make it possible to enable this for debugging, but not in tests. - // if (detail) { - // message += renderStyledStringToErrorAnsi(detail) + '\n\n' - // } - - // TODO: Include a trace from the issue. - - if (documentationLink) { - message += documentationLink + '\n\n' - } - - return message - } - - function processIssues( - name: string, - result: TurbopackResult, - throwIssue = false - ) { - const newIssues = new Map() - issues.set(name, newIssues) - - const relevantIssues = new Set() - - for (const issue of result.issues) { - if (issue.severity !== 'error' && issue.severity !== 'fatal') continue - const key = issueKey(issue) - const formatted = formatIssue(issue) - newIssues.set(key, issue) - - // We show errors in node_modules to the console, but don't throw for them - if (/(^|\/)node_modules(\/|$)/.test(issue.filePath)) continue - relevantIssues.add(formatted) - } - - if (relevantIssues.size && throwIssue) { - throw new ModuleBuildError([...relevantIssues].join('\n\n')) - } - } - + const issues: Issues = new Map() const serverPathState = new Map() async function processResult( @@ -426,7 +433,15 @@ export async function createHotReloaderTurbopack( } async function loadPartialManifest( - name: string, + name: + | typeof MIDDLEWARE_MANIFEST + | typeof BUILD_MANIFEST + | typeof APP_BUILD_MANIFEST + | typeof PAGES_MANIFEST + | typeof APP_PATHS_MANIFEST + | `${typeof SERVER_REFERENCE_MANIFEST}.json` + | `${typeof NEXT_FONT_MANIFEST}.json` + | typeof REACT_LOADABLE_MANIFEST, pageName: string, type: | 'pages' @@ -558,7 +573,7 @@ export async function createHotReloaderTurbopack( const changed = await changedPromise for await (const change of changed) { - processIssues(page, change) + processIssues(issues, page, change) const payload = await makePayload(page, change) if (payload) { sendHmr('endpoint-change', key, payload) @@ -622,12 +637,12 @@ export async function createHotReloaderTurbopack( ? (normalizeRewritesForBuildManifest(rewrites) as any) : { afterFiles: [], beforeFiles: [], fallback: [] }, ...Object.fromEntries( - [...curEntries.keys()].map((pathname) => [ + [...currentEntries.keys()].map((pathname) => [ pathname, `static/chunks/pages${pathname === '/' ? '/index' : pathname}.js`, ]) ), - sortedPages: [...curEntries.keys()], + sortedPages: [...currentEntries.keys()], } const buildManifestJs = `self.__BUILD_MANIFEST = ${JSON.stringify( content @@ -804,7 +819,7 @@ export async function createHotReloaderTurbopack( await subscription.next() for await (const data of subscription) { - processIssues(id, data) + processIssues(issues, id, data) if (data.type !== 'issues') { sendTurbopackMessage(data) } @@ -842,7 +857,7 @@ export async function createHotReloaderTurbopack( globalEntries.document = entrypoints.pagesDocumentEndpoint globalEntries.error = entrypoints.pagesErrorEndpoint - curEntries.clear() + currentEntries.clear() for (const [pathname, route] of entrypoints.routes) { switch (route.type) { @@ -850,7 +865,7 @@ export async function createHotReloaderTurbopack( case 'page-api': case 'app-page': case 'app-route': { - curEntries.set(pathname, route) + currentEntries.set(pathname, route) break } default: @@ -865,7 +880,7 @@ export async function createHotReloaderTurbopack( continue } - if (!curEntries.has(pathname)) { + if (!currentEntries.has(pathname)) { const subscription = await subscriptionPromise subscription.return?.() changeSubscriptions.delete(pathname) @@ -901,7 +916,7 @@ export async function createHotReloaderTurbopack( displayName, await instrumentation[prop].writeToDisk() ) - processIssues(name, writtenEndpoint) + processIssues(issues, name, writtenEndpoint) } await processInstrumentation( 'instrumentation (node.js)', @@ -936,7 +951,7 @@ export async function createHotReloaderTurbopack( 'middleware', await middleware.endpoint.writeToDisk() ) - processIssues('middleware', writtenEndpoint) + processIssues(issues, 'middleware', writtenEndpoint) await loadMiddlewareManifest('middleware', 'middleware') serverFields.middleware = { match: null as any, @@ -1022,6 +1037,200 @@ export async function createHotReloaderTurbopack( let versionInfo: VersionInfo = await getVersionInfo( true || isTestMode || opts.telemetry.isEnabled ) + + async function handleRouteType( + page: string, + route: Route, + requestUrl: string | undefined + ) { + let finishBuilding: (() => void) | undefined = undefined + + try { + switch (route.type) { + case 'page': { + finishBuilding = startBuilding(page, requestUrl) + try { + if (globalEntries.app) { + const writtenEndpoint = await processResult( + '_app', + await globalEntries.app.writeToDisk() + ) + processIssues(issues, '_app', writtenEndpoint) + } + await loadBuildManifest('_app') + await loadPagesManifest('_app') + + if (globalEntries.document) { + const writtenEndpoint = await processResult( + '_document', + await globalEntries.document.writeToDisk() + ) + processIssues(issues, '_document', writtenEndpoint) + } + await loadPagesManifest('_document') + + const writtenEndpoint = await processResult( + page, + await route.htmlEndpoint.writeToDisk() + ) + + const type = writtenEndpoint?.type + + await loadBuildManifest(page) + await loadPagesManifest(page) + if (type === 'edge') { + await loadMiddlewareManifest(page, 'pages') + } else { + middlewareManifests.delete(page) + } + await loadFontManifest(page, 'pages') + await loadLoadableManifest(page, 'pages') + + await writeManifests() + + processIssues(issues, page, writtenEndpoint) + } finally { + changeSubscription( + page, + 'server', + false, + route.dataEndpoint, + (pageName) => { + // Report the next compilation again + readyIds.delete(page) + return { + event: HMR_ACTIONS_SENT_TO_BROWSER.SERVER_ONLY_CHANGES, + pages: [pageName], + } + } + ) + changeSubscription( + page, + 'client', + false, + route.htmlEndpoint, + () => { + return { + event: HMR_ACTIONS_SENT_TO_BROWSER.CLIENT_CHANGES, + } + } + ) + if (globalEntries.document) { + changeSubscription( + '_document', + 'server', + false, + globalEntries.document, + () => { + return { action: HMR_ACTIONS_SENT_TO_BROWSER.RELOAD_PAGE } + } + ) + } + } + + break + } + case 'page-api': { + finishBuilding = startBuilding(page, requestUrl) + const writtenEndpoint = await processResult( + page, + await route.endpoint.writeToDisk() + ) + + const type = writtenEndpoint?.type + + await loadPagesManifest(page) + if (type === 'edge') { + await loadMiddlewareManifest(page, 'pages') + } else { + middlewareManifests.delete(page) + } + await loadLoadableManifest(page, 'pages') + + await writeManifests() + + processIssues(issues, page, writtenEndpoint) + + break + } + case 'app-page': { + finishBuilding = startBuilding(page, requestUrl) + const writtenEndpoint = await processResult( + page, + await route.htmlEndpoint.writeToDisk() + ) + + changeSubscription( + page, + 'server', + true, + route.rscEndpoint, + (_page, change) => { + if (change.issues.some((issue) => issue.severity === 'error')) { + // Ignore any updates that has errors + // There will be another update without errors eventually + return + } + // Report the next compilation again + readyIds.delete(page) + return { + action: HMR_ACTIONS_SENT_TO_BROWSER.SERVER_COMPONENT_CHANGES, + } + } + ) + + const type = writtenEndpoint?.type + + if (type === 'edge') { + await loadMiddlewareManifest(page, 'app') + } else { + middlewareManifests.delete(page) + } + + await loadAppBuildManifest(page) + await loadBuildManifest(page, 'app') + await loadAppPathManifest(page, 'app') + await loadActionManifest(page) + await loadFontManifest(page, 'app') + await writeManifests() + + processIssues(issues, page, writtenEndpoint, true) + + break + } + case 'app-route': { + finishBuilding = startBuilding(page, requestUrl) + const writtenEndpoint = await processResult( + page, + await route.endpoint.writeToDisk() + ) + + const type = writtenEndpoint?.type + + await loadAppPathManifest(page, 'app-route') + if (type === 'edge') { + await loadMiddlewareManifest(page, 'app-route') + } else { + middlewareManifests.delete(page) + } + + await writeManifests() + + processIssues(issues, page, writtenEndpoint, true) + + break + } + default: { + throw new Error( + `unknown route type ${(route as any).type} for ${page}` + ) + } + } + } finally { + if (finishBuilding) finishBuilding() + } + } + const hotReloader: NextJsHotReloaderInterface = { turbopackProject: project, activeWebpackConfigs: undefined, @@ -1228,7 +1437,7 @@ export async function createHotReloaderTurbopack( '_app', await globalEntries.app.writeToDisk() ) - processIssues('_app', writtenEndpoint) + processIssues(issues, '_app', writtenEndpoint) } await loadBuildManifest('_app') await loadPagesManifest('_app') @@ -1248,7 +1457,7 @@ export async function createHotReloaderTurbopack( return { action: HMR_ACTIONS_SENT_TO_BROWSER.RELOAD_PAGE } } ) - processIssues('_document', writtenEndpoint) + processIssues(issues, '_document', writtenEndpoint) } await loadPagesManifest('_document') @@ -1257,7 +1466,7 @@ export async function createHotReloaderTurbopack( '_error', await globalEntries.error.writeToDisk() ) - processIssues(page, writtenEndpoint) + processIssues(issues, page, writtenEndpoint) } await loadBuildManifest('_error') await loadPagesManifest('_error') @@ -1271,8 +1480,8 @@ export async function createHotReloaderTurbopack( } await currentEntriesHandling const route = - curEntries.get(page) ?? - curEntries.get( + currentEntries.get(page) ?? + currentEntries.get( normalizeAppPath( normalizeMetadataRoute(definition?.page ?? inputPage) ) @@ -1290,202 +1499,14 @@ export async function createHotReloaderTurbopack( throw new PageNotFoundError(`route not found ${page}`) } - let finishBuilding: (() => void) | undefined = undefined - - try { - switch (route.type) { - case 'page': { - if (isApp) { - throw new Error( - `mis-matched route type: isApp && page for ${page}` - ) - } - - finishBuilding = startBuilding(page, requestUrl) - try { - if (globalEntries.app) { - const writtenEndpoint = await processResult( - '_app', - await globalEntries.app.writeToDisk() - ) - processIssues('_app', writtenEndpoint) - } - await loadBuildManifest('_app') - await loadPagesManifest('_app') - - if (globalEntries.document) { - const writtenEndpoint = await processResult( - '_document', - await globalEntries.document.writeToDisk() - ) - processIssues('_document', writtenEndpoint) - } - await loadPagesManifest('_document') - - const writtenEndpoint = await processResult( - page, - await route.htmlEndpoint.writeToDisk() - ) - - const type = writtenEndpoint?.type - - await loadBuildManifest(page) - await loadPagesManifest(page) - if (type === 'edge') { - await loadMiddlewareManifest(page, 'pages') - } else { - middlewareManifests.delete(page) - } - await loadFontManifest(page, 'pages') - await loadLoadableManifest(page, 'pages') - - await writeManifests() - - processIssues(page, writtenEndpoint) - } finally { - changeSubscription( - page, - 'server', - false, - route.dataEndpoint, - (pageName) => { - // Report the next compilation again - readyIds.delete(page) - return { - event: HMR_ACTIONS_SENT_TO_BROWSER.SERVER_ONLY_CHANGES, - pages: [pageName], - } - } - ) - changeSubscription( - page, - 'client', - false, - route.htmlEndpoint, - () => { - return { - event: HMR_ACTIONS_SENT_TO_BROWSER.CLIENT_CHANGES, - } - } - ) - if (globalEntries.document) { - changeSubscription( - '_document', - 'server', - false, - globalEntries.document, - () => { - return { action: HMR_ACTIONS_SENT_TO_BROWSER.RELOAD_PAGE } - } - ) - } - } - - break - } - case 'page-api': { - // We don't throw on ensureOpts.isApp === true here - // since this can happen when app pages make - // api requests to page API routes. - - finishBuilding = startBuilding(page, requestUrl) - const writtenEndpoint = await processResult( - page, - await route.endpoint.writeToDisk() - ) - - const type = writtenEndpoint?.type - - await loadPagesManifest(page) - if (type === 'edge') { - await loadMiddlewareManifest(page, 'pages') - } else { - middlewareManifests.delete(page) - } - await loadLoadableManifest(page, 'pages') - - await writeManifests() - - processIssues(page, writtenEndpoint) - - break - } - case 'app-page': { - finishBuilding = startBuilding(page, requestUrl) - const writtenEndpoint = await processResult( - page, - await route.htmlEndpoint.writeToDisk() - ) - - changeSubscription( - page, - 'server', - true, - route.rscEndpoint, - (_page, change) => { - if (change.issues.some((issue) => issue.severity === 'error')) { - // Ignore any updates that has errors - // There will be another update without errors eventually - return - } - // Report the next compilation again - readyIds.delete(page) - return { - action: HMR_ACTIONS_SENT_TO_BROWSER.SERVER_COMPONENT_CHANGES, - } - } - ) - - const type = writtenEndpoint?.type - - if (type === 'edge') { - await loadMiddlewareManifest(page, 'app') - } else { - middlewareManifests.delete(page) - } - - await loadAppBuildManifest(page) - await loadBuildManifest(page, 'app') - await loadAppPathManifest(page, 'app') - await loadActionManifest(page) - await loadFontManifest(page, 'app') - await writeManifests() - - processIssues(page, writtenEndpoint, true) - - break - } - case 'app-route': { - finishBuilding = startBuilding(page, requestUrl) - const writtenEndpoint = await processResult( - page, - await route.endpoint.writeToDisk() - ) - - const type = writtenEndpoint?.type - - await loadAppPathManifest(page, 'app-route') - if (type === 'edge') { - await loadMiddlewareManifest(page, 'app-route') - } else { - middlewareManifests.delete(page) - } - - await writeManifests() - - processIssues(page, writtenEndpoint, true) - - break - } - default: { - throw new Error( - `unknown route type ${(route as any).type} for ${page}` - ) - } - } - } finally { - if (finishBuilding) finishBuilding() + // We don't throw on ensureOpts.isApp === true for page-api + // since this can happen when app pages make + // api requests to page API routes. + if (isApp && route.type === 'page') { + throw new Error(`mis-matched route type: isApp && page for ${page}`) } + + await handleRouteType(page, route, requestUrl) }, } From fbeeafaf02beb6e48e88dc06134784b3dc13db47 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Mon, 12 Feb 2024 10:30:26 +0100 Subject: [PATCH 3/9] Avoid double check for version --- .../src/server/dev/hot-reloader-turbopack.ts | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index 4406ecbc08bde..29b805a58d027 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -280,7 +280,7 @@ export async function createHotReloaderTurbopack( serverAddr: `127.0.0.1:${opts.port}`, }) const iter = project.entrypointsSubscribe() - const currentEntries: Map = new Map() + const curEntries: Map = new Map() const changeSubscriptions: Map< string, Promise> @@ -637,12 +637,12 @@ export async function createHotReloaderTurbopack( ? (normalizeRewritesForBuildManifest(rewrites) as any) : { afterFiles: [], beforeFiles: [], fallback: [] }, ...Object.fromEntries( - [...currentEntries.keys()].map((pathname) => [ + [...curEntries.keys()].map((pathname) => [ pathname, `static/chunks/pages${pathname === '/' ? '/index' : pathname}.js`, ]) ), - sortedPages: [...currentEntries.keys()], + sortedPages: [...curEntries.keys()], } const buildManifestJs = `self.__BUILD_MANIFEST = ${JSON.stringify( content @@ -857,7 +857,7 @@ export async function createHotReloaderTurbopack( globalEntries.document = entrypoints.pagesDocumentEndpoint globalEntries.error = entrypoints.pagesErrorEndpoint - currentEntries.clear() + curEntries.clear() for (const [pathname, route] of entrypoints.routes) { switch (route.type) { @@ -865,7 +865,7 @@ export async function createHotReloaderTurbopack( case 'page-api': case 'app-page': case 'app-route': { - currentEntries.set(pathname, route) + curEntries.set(pathname, route) break } default: @@ -880,7 +880,7 @@ export async function createHotReloaderTurbopack( continue } - if (!currentEntries.has(pathname)) { + if (!curEntries.has(pathname)) { const subscription = await subscriptionPromise subscription.return?.() changeSubscriptions.delete(pathname) @@ -1034,8 +1034,8 @@ export async function createHotReloaderTurbopack( await writeManifests() const overlayMiddleware = getOverlayMiddleware(project) - let versionInfo: VersionInfo = await getVersionInfo( - true || isTestMode || opts.telemetry.isEnabled + const versionInfo: VersionInfo = await getVersionInfo( + isTestMode || opts.telemetry.isEnabled ) async function handleRouteType( @@ -1376,13 +1376,7 @@ export async function createHotReloaderTurbopack( clearHmrServerError() { // Not implemented yet. }, - async start() { - const enabled = isTestMode || opts.telemetry.isEnabled - const nextVersionInfo = await getVersionInfo(enabled) - if (nextVersionInfo) { - versionInfo = nextVersionInfo - } - }, + async start() {}, async stop() { // Not implemented yet. }, @@ -1480,8 +1474,8 @@ export async function createHotReloaderTurbopack( } await currentEntriesHandling const route = - currentEntries.get(page) ?? - currentEntries.get( + curEntries.get(page) ?? + curEntries.get( normalizeAppPath( normalizeMetadataRoute(definition?.page ?? inputPage) ) From 898070ccb0b6d110e36fa47b7bc1ac6883a4cc03 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Mon, 12 Feb 2024 10:33:48 +0100 Subject: [PATCH 4/9] Update hot-reloader-turbopack.ts --- packages/next/src/server/dev/hot-reloader-turbopack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index 29b805a58d027..979beb4032997 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -1421,7 +1421,7 @@ export async function createHotReloaderTurbopack( isApp, url: requestUrl, }) { - let page = definition?.pathname ?? inputPage + const page = definition?.pathname ?? inputPage if (page === '/_error') { let finishBuilding = startBuilding(page, requestUrl) From c9454c01c20cadc845095b4f63a12ce544c61ea8 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Mon, 12 Feb 2024 10:36:23 +0100 Subject: [PATCH 5/9] Update hot-reloader-turbopack.ts --- packages/next/src/server/dev/hot-reloader-turbopack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index 979beb4032997..c50dd81156082 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -314,7 +314,7 @@ export async function createHotReloaderTurbopack( for (const { path: p, contentHash } of result.serverPaths) { // We ignore source maps if (p.endsWith('.map')) continue - let key = `${id}:${p}` + const key = `${id}:${p}` const localHash = serverPathState.get(key) const globalHash = serverPathState.get(p) if ( From 65f944aae6113822883b7865e1ead43b13d6a276 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Mon, 12 Feb 2024 10:41:22 +0100 Subject: [PATCH 6/9] Update hot-reloader-turbopack.ts --- .../src/server/dev/hot-reloader-turbopack.ts | 92 +++++++++---------- 1 file changed, 41 insertions(+), 51 deletions(-) diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index c50dd81156082..71e26280dce0e 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -22,7 +22,7 @@ import type { import ws from 'next/dist/compiled/ws' import { createDefineEnv } from '../../build/swc' -import path from 'path' +import { join, posix } from 'path' import * as Log from '../../build/output/log' import { getVersionInfo, @@ -305,31 +305,31 @@ export async function createHotReloaderTurbopack( const issues: Issues = new Map() const serverPathState = new Map() - async function processResult( + async function handleRequireCacheClearing( id: string, result: TurbopackResult ): Promise> { // Figure out if the server files have changed let hasChange = false - for (const { path: p, contentHash } of result.serverPaths) { + for (const { path, contentHash } of result.serverPaths) { // We ignore source maps - if (p.endsWith('.map')) continue - const key = `${id}:${p}` + if (path.endsWith('.map')) continue + const key = `${id}:${path}` const localHash = serverPathState.get(key) - const globalHash = serverPathState.get(p) + const globalHash = serverPathState.get(path) if ( (localHash && localHash !== contentHash) || (globalHash && globalHash !== contentHash) ) { hasChange = true serverPathState.set(key, contentHash) - serverPathState.set(p, contentHash) + serverPathState.set(path, contentHash) } else { if (!localHash) { serverPathState.set(key, contentHash) } if (!globalHash) { - serverPathState.set(p, contentHash) + serverPathState.set(path, contentHash) } } } @@ -347,7 +347,7 @@ export async function createHotReloaderTurbopack( } const serverPaths = result.serverPaths.map(({ path: p }) => - path.join(distDir, p) + join(distDir, p) ) for (const file of serverPaths) { @@ -450,7 +450,7 @@ export async function createHotReloaderTurbopack( | 'middleware' | 'instrumentation' = 'pages' ): Promise { - const manifestPath = path.posix.join( + const manifestPath = posix.join( distDir, `server`, type === 'app-route' ? 'app' : type, @@ -464,9 +464,7 @@ export async function createHotReloaderTurbopack( type === 'app' ? 'page' : type === 'app-route' ? 'route' : '', name ) - return JSON.parse( - await readFile(path.posix.join(manifestPath), 'utf-8') - ) as T + return JSON.parse(await readFile(posix.join(manifestPath), 'utf-8')) as T } const buildManifests = new Map() @@ -598,13 +596,13 @@ export async function createHotReloaderTurbopack( rewrites: SetupOpts['fsChecker']['rewrites'] ): Promise { const buildManifest = mergeBuildManifests(buildManifests.values()) - const buildManifestPath = path.join(distDir, BUILD_MANIFEST) - const middlewareBuildManifestPath = path.join( + const buildManifestPath = join(distDir, BUILD_MANIFEST) + const middlewareBuildManifestPath = join( distDir, 'server', `${MIDDLEWARE_BUILD_MANIFEST}.js` ) - const interceptionRewriteManifestPath = path.join( + const interceptionRewriteManifestPath = join( distDir, 'server', `${INTERCEPTION_ROUTE_REWRITE_MANIFEST}.js` @@ -648,11 +646,11 @@ export async function createHotReloaderTurbopack( content )};self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB()` await writeFileAtomic( - path.join(distDir, 'static', 'development', '_buildManifest.js'), + join(distDir, 'static', 'development', '_buildManifest.js'), buildManifestJs ) await writeFileAtomic( - path.join(distDir, 'static', 'development', '_ssgManifest.js'), + join(distDir, 'static', 'development', '_ssgManifest.js'), srcEmptySsgManifest ) } @@ -663,7 +661,7 @@ export async function createHotReloaderTurbopack( Boolean ) as BuildManifest[] ) - const fallbackBuildManifestPath = path.join( + const fallbackBuildManifestPath = join( distDir, `fallback-${BUILD_MANIFEST}` ) @@ -676,7 +674,7 @@ export async function createHotReloaderTurbopack( async function writeAppBuildManifest(): Promise { const appBuildManifest = mergeAppBuildManifests(appBuildManifests.values()) - const appBuildManifestPath = path.join(distDir, APP_BUILD_MANIFEST) + const appBuildManifestPath = join(distDir, APP_BUILD_MANIFEST) deleteCache(appBuildManifestPath) await writeFileAtomic( appBuildManifestPath, @@ -686,7 +684,7 @@ export async function createHotReloaderTurbopack( async function writePagesManifest(): Promise { const pagesManifest = mergePagesManifests(pagesManifests.values()) - const pagesManifestPath = path.join(distDir, 'server', PAGES_MANIFEST) + const pagesManifestPath = join(distDir, 'server', PAGES_MANIFEST) deleteCache(pagesManifestPath) await writeFileAtomic( pagesManifestPath, @@ -696,11 +694,7 @@ export async function createHotReloaderTurbopack( async function writeAppPathsManifest(): Promise { const appPathsManifest = mergePagesManifests(appPathsManifests.values()) - const appPathsManifestPath = path.join( - distDir, - 'server', - APP_PATHS_MANIFEST - ) + const appPathsManifestPath = join(distDir, 'server', APP_PATHS_MANIFEST) deleteCache(appPathsManifestPath) await writeFileAtomic( appPathsManifestPath, @@ -712,11 +706,7 @@ export async function createHotReloaderTurbopack( const middlewareManifest = mergeMiddlewareManifests( middlewareManifests.values() ) - const middlewareManifestPath = path.join( - distDir, - 'server', - MIDDLEWARE_MANIFEST - ) + const middlewareManifestPath = join(distDir, 'server', MIDDLEWARE_MANIFEST) deleteCache(middlewareManifestPath) await writeFileAtomic( middlewareManifestPath, @@ -726,12 +716,12 @@ export async function createHotReloaderTurbopack( async function writeActionManifest(): Promise { const actionManifest = await mergeActionManifests(actionManifests.values()) - const actionManifestJsonPath = path.join( + const actionManifestJsonPath = join( distDir, 'server', `${SERVER_REFERENCE_MANIFEST}.json` ) - const actionManifestJsPath = path.join( + const actionManifestJsPath = join( distDir, 'server', `${SERVER_REFERENCE_MANIFEST}.js` @@ -751,12 +741,12 @@ export async function createHotReloaderTurbopack( const fontManifest = mergeFontManifests(fontManifests.values()) const json = JSON.stringify(fontManifest, null, 2) - const fontManifestJsonPath = path.join( + const fontManifestJsonPath = join( distDir, 'server', `${NEXT_FONT_MANIFEST}.json` ) - const fontManifestJsPath = path.join( + const fontManifestJsPath = join( distDir, 'server', `${NEXT_FONT_MANIFEST}.js` @@ -772,8 +762,8 @@ export async function createHotReloaderTurbopack( async function writeLoadableManifest(): Promise { const loadableManifest = mergeLoadableManifests(loadableManifests.values()) - const loadableManifestPath = path.join(distDir, REACT_LOADABLE_MANIFEST) - const middlewareloadableManifestPath = path.join( + const loadableManifestPath = join(distDir, REACT_LOADABLE_MANIFEST) + const middlewareloadableManifestPath = join( distDir, 'server', `${MIDDLEWARE_REACT_LOADABLE_MANIFEST}.js` @@ -912,7 +902,7 @@ export async function createHotReloaderTurbopack( name: string, prop: 'nodeJs' | 'edge' ) => { - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( displayName, await instrumentation[prop].writeToDisk() ) @@ -947,7 +937,7 @@ export async function createHotReloaderTurbopack( } if (middleware) { const processMiddleware = async () => { - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( 'middleware', await middleware.endpoint.writeToDisk() ) @@ -1018,10 +1008,10 @@ export async function createHotReloaderTurbopack( } // Write empty manifests - await mkdir(path.join(distDir, 'server'), { recursive: true }) - await mkdir(path.join(distDir, 'static/development'), { recursive: true }) + await mkdir(join(distDir, 'server'), { recursive: true }) + await mkdir(join(distDir, 'static/development'), { recursive: true }) await writeFile( - path.join(distDir, 'package.json'), + join(distDir, 'package.json'), JSON.stringify( { type: 'commonjs', @@ -1051,7 +1041,7 @@ export async function createHotReloaderTurbopack( finishBuilding = startBuilding(page, requestUrl) try { if (globalEntries.app) { - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( '_app', await globalEntries.app.writeToDisk() ) @@ -1061,7 +1051,7 @@ export async function createHotReloaderTurbopack( await loadPagesManifest('_app') if (globalEntries.document) { - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( '_document', await globalEntries.document.writeToDisk() ) @@ -1069,7 +1059,7 @@ export async function createHotReloaderTurbopack( } await loadPagesManifest('_document') - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( page, await route.htmlEndpoint.writeToDisk() ) @@ -1132,7 +1122,7 @@ export async function createHotReloaderTurbopack( } case 'page-api': { finishBuilding = startBuilding(page, requestUrl) - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( page, await route.endpoint.writeToDisk() ) @@ -1155,7 +1145,7 @@ export async function createHotReloaderTurbopack( } case 'app-page': { finishBuilding = startBuilding(page, requestUrl) - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( page, await route.htmlEndpoint.writeToDisk() ) @@ -1200,7 +1190,7 @@ export async function createHotReloaderTurbopack( } case 'app-route': { finishBuilding = startBuilding(page, requestUrl) - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( page, await route.endpoint.writeToDisk() ) @@ -1427,7 +1417,7 @@ export async function createHotReloaderTurbopack( let finishBuilding = startBuilding(page, requestUrl) try { if (globalEntries.app) { - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( '_app', await globalEntries.app.writeToDisk() ) @@ -1438,7 +1428,7 @@ export async function createHotReloaderTurbopack( await loadFontManifest('_app') if (globalEntries.document) { - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( '_document', await globalEntries.document.writeToDisk() ) @@ -1456,7 +1446,7 @@ export async function createHotReloaderTurbopack( await loadPagesManifest('_document') if (globalEntries.error) { - const writtenEndpoint = await processResult( + const writtenEndpoint = await handleRequireCacheClearing( '_error', await globalEntries.error.writeToDisk() ) From d82a6a122b7afbdc717bd61e0995de4ccde252c1 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Mon, 12 Feb 2024 10:43:59 +0100 Subject: [PATCH 7/9] Update index.ts --- packages/next/src/build/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 050ba4e74066f..52b88ae7272d0 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1338,6 +1338,7 @@ export default async function build( serverAddr: `127.0.0.1:3000`, }) + // eslint-disable-next-line @typescript-eslint/no-unused-vars const iter = project.entrypointsSubscribe() // for await (const entrypoints of iter) { // } From 2b6679c76f0d37eac6cafe23e45f83e79c611cd1 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Mon, 12 Feb 2024 10:50:46 +0100 Subject: [PATCH 8/9] Fix compile errors --- packages/next/src/build/index.ts | 5 +++-- .../next/src/server/dev/hot-reloader-turbopack.ts | 11 +---------- packages/next/src/server/dev/turbopack-utils.ts | 10 ++++++++++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 52b88ae7272d0..c23899866586d 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -166,8 +166,8 @@ import { hasCustomExportOutput } from '../export/utils' import { interopDefault } from '../lib/interop-default' import { formatDynamicImportPath } from '../lib/format-dynamic-import-path' import { isInterceptionRouteAppPath } from '../server/future/helpers/interception-routes' +import { getTurbopackJsConfig } from '../server/dev/turbopack-utils' import { buildCustomRoute } from '../lib/build-custom-route' -import { getTurbopackJsConfig } from '../server/dev/hot-reloader-turbopack' interface ExperimentalBypassForInfo { experimentalBypassFor?: RouteHas[] @@ -1309,7 +1309,7 @@ export default async function build( return serverFilesManifest }) - async function turbopackBuild(): Promise { + async function turbopackBuild(): Promise { if (!process.env.TURBOPACK || !process.env.TURBOPACK_BUILD) { throw new Error("next build doesn't support turbopack yet") } @@ -1342,6 +1342,7 @@ export default async function build( const iter = project.entrypointsSubscribe() // for await (const entrypoints of iter) { // } + throw new Error("next build doesn't support turbopack yet") } let buildTraceContext: undefined | BuildTraceContext let buildTracesPromise: Promise | undefined = undefined diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index 71e26280dce0e..2842b56a9126c 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -28,7 +28,6 @@ import { getVersionInfo, matchNextPageBundleRequest, } from './hot-reloader-webpack' -import loadJsConfig from '../../build/load-jsconfig' import { normalizeAppPath } from '../../shared/lib/router/utils/app-paths' import { isInterceptionRouteRewrite } from '../../lib/generate-interception-routes-rewrites' import { store as consoleStore } from '../../build/output/store' @@ -79,6 +78,7 @@ import { decodeMagicIdentifier, } from '../../shared/lib/magic-identifier' import { + getTurbopackJsConfig, mergeActionManifests, mergeAppBuildManifests, mergeBuildManifests, @@ -93,7 +93,6 @@ import { type ServerFields, type SetupOpts, } from '../lib/router-utils/setup-dev-bundler' -import type { NextConfigComplete } from '../config-shared' const MILLISECONDS_IN_NANOSECOND = 1_000_000 const wsServer = new ws.Server({ noServer: true }) @@ -105,14 +104,6 @@ const isTestMode = !!( class ModuleBuildError extends Error {} -export async function getTurbopackJsConfig( - dir: string, - nextConfig: NextConfigComplete -) { - const { jsConfig } = await loadJsConfig(dir, nextConfig) - return jsConfig ?? { compilerOptions: {} } -} - function issueKey(issue: Issue): string { return [ issue.severity, diff --git a/packages/next/src/server/dev/turbopack-utils.ts b/packages/next/src/server/dev/turbopack-utils.ts index 983a347664dbc..8a5744abb9447 100644 --- a/packages/next/src/server/dev/turbopack-utils.ts +++ b/packages/next/src/server/dev/turbopack-utils.ts @@ -10,6 +10,8 @@ import type { import type { PagesManifest } from '../../build/webpack/plugins/pages-manifest-plugin' import type { AppBuildManifest } from '../../build/webpack/plugins/app-build-manifest-plugin' import type { BuildManifest } from '../get-page-files' +import type { NextConfigComplete } from '../config-shared' +import loadJsConfig from '../../build/load-jsconfig' export interface InstrumentationDefinition { files: string[] @@ -169,3 +171,11 @@ export function mergeLoadableManifests(manifests: Iterable) { } return manifest } + +export async function getTurbopackJsConfig( + dir: string, + nextConfig: NextConfigComplete +) { + const { jsConfig } = await loadJsConfig(dir, nextConfig) + return jsConfig ?? { compilerOptions: {} } +} From 3d1d9f1c5db168fba8227c6add2419f2d749820e Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Mon, 12 Feb 2024 12:13:19 +0100 Subject: [PATCH 9/9] Fix options --- packages/next/src/build/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index c23899866586d..fd515d0866afe 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1319,14 +1319,14 @@ export default async function build( rootPath: config.experimental.outputFileTracingRoot || dir, nextConfig: config.nextConfig, jsConfig: await getTurbopackJsConfig(dir, config), - watch: true, + watch: false, env: process.env as Record, defineEnv: createDefineEnv({ isTurbopack: true, allowedRevalidateHeaderKeys: undefined, clientRouterFilters: undefined, config, - dev: true, + dev: false, distDir, fetchCacheKeyPrefix: undefined, hasRewrites,