From dca95750d709e99060f80933910ce10fbe78156c Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 17 May 2021 14:04:06 +0200 Subject: [PATCH] performance improvement of static generation (#25035) ### move all access to built pages into worker pool to allow parallelizing and avoid loading the bundles in the main thread This improves performance of the static check step a bit and helps reducing memory load in main thread ### enable splitChunks for server build in webpack 5 This improves performance for static generation by loading less code due to reduced duplication --- packages/next/build/entries.ts | 2 +- packages/next/build/index.ts | 374 +++++++++--------- packages/next/build/webpack-config.ts | 12 +- .../webpack/plugins/pages-manifest-plugin.ts | 4 +- packages/next/client/next-dev.js | 2 +- 5 files changed, 205 insertions(+), 189 deletions(-) diff --git a/packages/next/build/entries.ts b/packages/next/build/entries.ts index 89418ef6283fd..07fa0180433cf 100644 --- a/packages/next/build/entries.ts +++ b/packages/next/build/entries.ts @@ -58,7 +58,7 @@ export type WebpackEntrypoints = { | string[] | { import: string | string[] - dependOn: string | string[] + dependOn?: string | string[] } } diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts index 182680febcfc0..d9c6a555025d6 100644 --- a/packages/next/build/index.ts +++ b/packages/next/build/index.ts @@ -81,9 +81,6 @@ import { detectConflictingPaths, computeFromManifest, getJsPageSizeInKb, - getNamedExports, - hasCustomGetInitialProps, - isPageStatic, PageInfo, printCustomRoutes, printTreeView, @@ -266,7 +263,7 @@ export default async function build( ) const pageKeys = Object.keys(mappedPages) const conflictingPublicFiles: string[] = [] - const hasCustomErrorPage = mappedPages['/_error'].startsWith( + const hasCustomErrorPage: boolean = mappedPages['/_error'].startsWith( 'private-next-pages' ) const hasPages404 = Boolean( @@ -656,219 +653,228 @@ export default async function build( await promises.readFile(buildManifestPath, 'utf8') ) as BuildManifest - let customAppGetInitialProps: boolean | undefined - let namedExports: Array | undefined - let isNextImageImported: boolean | undefined const analysisBegin = process.hrtime() - let hasSsrAmpPages = false const staticCheckSpan = nextBuildSpan.traceChild('static-check') - const { hasNonStaticErrorPage } = await staticCheckSpan.traceAsyncFn( - async () => { - process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD + const { + customAppGetInitialProps, + namedExports, + isNextImageImported, + hasSsrAmpPages, + hasNonStaticErrorPage, + } = await staticCheckSpan.traceAsyncFn(async () => { + process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD + + const staticCheckWorkers = new Worker(staticCheckWorker, { + numWorkers: config.experimental.cpus, + enableWorkerThreads: config.experimental.workerThreads, + }) as Worker & typeof import('./utils') + + staticCheckWorkers.getStdout().pipe(process.stdout) + staticCheckWorkers.getStderr().pipe(process.stderr) + + const runtimeEnvConfig = { + publicRuntimeConfig: config.publicRuntimeConfig, + serverRuntimeConfig: config.serverRuntimeConfig, + } - const staticCheckWorkers = new Worker(staticCheckWorker, { - numWorkers: config.experimental.cpus, - enableWorkerThreads: config.experimental.workerThreads, - }) as Worker & { isPageStatic: typeof isPageStatic } + const nonStaticErrorPageSpan = staticCheckSpan.traceChild( + 'check-static-error-page' + ) + const nonStaticErrorPagePromise = nonStaticErrorPageSpan.traceAsyncFn( + async () => + hasCustomErrorPage && + (await staticCheckWorkers.hasCustomGetInitialProps( + '/_error', + distDir, + isLikeServerless, + runtimeEnvConfig, + false + )) + ) + // we don't output _app in serverless mode so use _app export + // from _error instead + const appPageToCheck = isLikeServerless ? '/_error' : '/_app' - staticCheckWorkers.getStdout().pipe(process.stdout) - staticCheckWorkers.getStderr().pipe(process.stderr) + const customAppGetInitialPropsPromise = staticCheckWorkers.hasCustomGetInitialProps( + appPageToCheck, + distDir, + isLikeServerless, + runtimeEnvConfig, + true + ) - const runtimeEnvConfig = { - publicRuntimeConfig: config.publicRuntimeConfig, - serverRuntimeConfig: config.serverRuntimeConfig, - } + const namedExportsPromise = staticCheckWorkers.getNamedExports( + appPageToCheck, + distDir, + isLikeServerless, + runtimeEnvConfig + ) - const nonStaticErrorPageSpan = staticCheckSpan.traceChild( - 'check-static-error-page' - ) - const nonStaticErrorPage = await nonStaticErrorPageSpan.traceAsyncFn( - async () => - hasCustomErrorPage && - (await hasCustomGetInitialProps( - '/_error', + // eslint-disable-next-line no-shadow + let isNextImageImported: boolean | undefined + // eslint-disable-next-line no-shadow + let hasSsrAmpPages = false + + const computedManifestData = await computeFromManifest( + buildManifest, + distDir, + config.experimental.gzipSize + ) + await Promise.all( + pageKeys.map(async (page) => { + const checkPageSpan = staticCheckSpan.traceChild('check-page', { + page, + }) + return checkPageSpan.traceAsyncFn(async () => { + const actualPage = normalizePagePath(page) + const [selfSize, allSize] = await getJsPageSizeInKb( + actualPage, distDir, - isLikeServerless, - runtimeEnvConfig, - false - )) - ) - // we don't output _app in serverless mode so use _app export - // from _error instead - const appPageToCheck = isLikeServerless ? '/_error' : '/_app' + buildManifest, + config.experimental.gzipSize, + computedManifestData + ) - customAppGetInitialProps = await hasCustomGetInitialProps( - appPageToCheck, - distDir, - isLikeServerless, - runtimeEnvConfig, - true - ) + let isSsg = false + let isStatic = false + let isHybridAmp = false + let ssgPageRoutes: string[] | null = null - namedExports = await getNamedExports( - appPageToCheck, - distDir, - isLikeServerless, - runtimeEnvConfig - ) + const nonReservedPage = !page.match( + /^\/(_app|_error|_document|api(\/|$))/ + ) - if (customAppGetInitialProps) { - console.warn( - chalk.bold.yellow(`Warning: `) + - chalk.yellow( - `You have opted-out of Automatic Static Optimization due to \`getInitialProps\` in \`pages/_app\`. This does not opt-out pages with \`getStaticProps\`` - ) - ) - console.warn( - 'Read more: https://nextjs.org/docs/messages/opt-out-auto-static-optimization\n' - ) - } + if (nonReservedPage) { + try { + let isPageStaticSpan = checkPageSpan.traceChild( + 'is-page-static' + ) + let workerResult = await isPageStaticSpan.traceAsyncFn(() => { + return staticCheckWorkers.isPageStatic( + page, + distDir, + isLikeServerless, + runtimeEnvConfig, + config.i18n?.locales, + config.i18n?.defaultLocale, + isPageStaticSpan.id + ) + }) - const computedManifestData = await computeFromManifest( - buildManifest, - distDir, - config.experimental.gzipSize - ) - await Promise.all( - pageKeys.map(async (page) => { - const checkPageSpan = staticCheckSpan.traceChild('check-page', { - page, - }) - return checkPageSpan.traceAsyncFn(async () => { - const actualPage = normalizePagePath(page) - const [selfSize, allSize] = await getJsPageSizeInKb( - actualPage, - distDir, - buildManifest, - config.experimental.gzipSize, - computedManifestData - ) + if ( + workerResult.isStatic === false && + (workerResult.isHybridAmp || workerResult.isAmpOnly) + ) { + hasSsrAmpPages = true + } - let isSsg = false - let isStatic = false - let isHybridAmp = false - let ssgPageRoutes: string[] | null = null + if (workerResult.isHybridAmp) { + isHybridAmp = true + hybridAmpPages.add(page) + } - const nonReservedPage = !page.match( - /^\/(_app|_error|_document|api(\/|$))/ - ) + if (workerResult.isNextImageImported) { + isNextImageImported = true + } - if (nonReservedPage) { - try { - let isPageStaticSpan = checkPageSpan.traceChild( - 'is-page-static' - ) - let workerResult = await isPageStaticSpan.traceAsyncFn(() => { - return staticCheckWorkers.isPageStatic( - page, - distDir, - isLikeServerless, - runtimeEnvConfig, - config.i18n?.locales, - config.i18n?.defaultLocale, - isPageStaticSpan.id - ) - }) + if (workerResult.hasStaticProps) { + ssgPages.add(page) + isSsg = true if ( - workerResult.isStatic === false && - (workerResult.isHybridAmp || workerResult.isAmpOnly) + workerResult.prerenderRoutes && + workerResult.encodedPrerenderRoutes ) { - hasSsrAmpPages = true - } - - if (workerResult.isHybridAmp) { - isHybridAmp = true - hybridAmpPages.add(page) - } - - if (workerResult.isNextImageImported) { - isNextImageImported = true - } - - if (workerResult.hasStaticProps) { - ssgPages.add(page) - isSsg = true - - if ( - workerResult.prerenderRoutes && + additionalSsgPaths.set(page, workerResult.prerenderRoutes) + additionalSsgPathsEncoded.set( + page, workerResult.encodedPrerenderRoutes - ) { - additionalSsgPaths.set(page, workerResult.prerenderRoutes) - additionalSsgPathsEncoded.set( - page, - workerResult.encodedPrerenderRoutes - ) - ssgPageRoutes = workerResult.prerenderRoutes - } - - if (workerResult.prerenderFallback === 'blocking') { - ssgBlockingFallbackPages.add(page) - } else if (workerResult.prerenderFallback === true) { - ssgStaticFallbackPages.add(page) - } - } else if (workerResult.hasServerProps) { - serverPropsPages.add(page) - } else if ( - workerResult.isStatic && - customAppGetInitialProps === false - ) { - staticPages.add(page) - isStatic = true + ) + ssgPageRoutes = workerResult.prerenderRoutes } - if (hasPages404 && page === '/404') { - if ( - !workerResult.isStatic && - !workerResult.hasStaticProps - ) { - throw new Error( - `\`pages/404\` ${STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR}` - ) - } - // we need to ensure the 404 lambda is present since we use - // it when _app has getInitialProps - if ( - customAppGetInitialProps && - !workerResult.hasStaticProps - ) { - staticPages.delete(page) - } + if (workerResult.prerenderFallback === 'blocking') { + ssgBlockingFallbackPages.add(page) + } else if (workerResult.prerenderFallback === true) { + ssgStaticFallbackPages.add(page) } + } else if (workerResult.hasServerProps) { + serverPropsPages.add(page) + } else if ( + workerResult.isStatic && + (await customAppGetInitialPropsPromise) === false + ) { + staticPages.add(page) + isStatic = true + } + if (hasPages404 && page === '/404') { + if (!workerResult.isStatic && !workerResult.hasStaticProps) { + throw new Error( + `\`pages/404\` ${STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR}` + ) + } + // we need to ensure the 404 lambda is present since we use + // it when _app has getInitialProps if ( - STATIC_STATUS_PAGES.includes(page) && - !workerResult.isStatic && + (await customAppGetInitialPropsPromise) && !workerResult.hasStaticProps ) { - throw new Error( - `\`pages${page}\` ${STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR}` - ) + staticPages.delete(page) } - } catch (err) { - if (err.message !== 'INVALID_DEFAULT_EXPORT') throw err - invalidPages.add(page) } + + if ( + STATIC_STATUS_PAGES.includes(page) && + !workerResult.isStatic && + !workerResult.hasStaticProps + ) { + throw new Error( + `\`pages${page}\` ${STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR}` + ) + } + } catch (err) { + if (err.message !== 'INVALID_DEFAULT_EXPORT') throw err + invalidPages.add(page) } + } - pageInfos.set(page, { - size: selfSize, - totalSize: allSize, - static: isStatic, - isSsg, - isHybridAmp, - ssgPageRoutes, - initialRevalidateSeconds: false, - }) + pageInfos.set(page, { + size: selfSize, + totalSize: allSize, + static: isStatic, + isSsg, + isHybridAmp, + ssgPageRoutes, + initialRevalidateSeconds: false, }) }) - ) - staticCheckWorkers.end() - - return { hasNonStaticErrorPage: nonStaticErrorPage } + }) + ) + const returnValue = { + customAppGetInitialProps: await customAppGetInitialPropsPromise, + namedExports: await namedExportsPromise, + isNextImageImported, + hasSsrAmpPages, + hasNonStaticErrorPage: await nonStaticErrorPagePromise, } - ) + + staticCheckWorkers.end() + return returnValue + }) + + if (customAppGetInitialProps) { + console.warn( + chalk.bold.yellow(`Warning: `) + + chalk.yellow( + `You have opted-out of Automatic Static Optimization due to \`getInitialProps\` in \`pages/_app\`. This does not opt-out pages with \`getStaticProps\`` + ) + ) + console.warn( + 'Read more: https://nextjs.org/docs/messages/opt-out-auto-static-optimization\n' + ) + } if (!hasSsrAmpPages) { requiredServerFiles.ignore.push( diff --git a/packages/next/build/webpack-config.ts b/packages/next/build/webpack-config.ts index 438796298254a..ed25e4604c852 100644 --- a/packages/next/build/webpack-config.ts +++ b/packages/next/build/webpack-config.ts @@ -822,7 +822,17 @@ export default async function getBaseWebpackConfig( ...(isWebpack5 ? { emitOnErrors: !dev } : { noEmitOnErrors: dev }), checkWasmTypes: false, nodeEnv: false, - splitChunks: isServer ? false : splitChunksConfig, + splitChunks: isServer + ? isWebpack5 + ? { + // allow to split entrypoints + chunks: 'all', + // size of files is not so relevant for server build + // we want to prefer deduplication to load less code + minSize: 1000, + } + : false + : splitChunksConfig, runtimeChunk: isServer ? isWebpack5 && !isLikeServerless ? { name: 'webpack-runtime' } diff --git a/packages/next/build/webpack/plugins/pages-manifest-plugin.ts b/packages/next/build/webpack/plugins/pages-manifest-plugin.ts index af1ca3ccf7495..8690b95837259 100644 --- a/packages/next/build/webpack/plugins/pages-manifest-plugin.ts +++ b/packages/next/build/webpack/plugins/pages-manifest-plugin.ts @@ -38,7 +38,7 @@ export default class PagesManifestPlugin implements webpack.Plugin { !file.includes('webpack-runtime') && file.endsWith('.js') ) - if (files.length > 1) { + if (!isWebpack5 && files.length > 1) { console.log( `Found more than one file in server entrypoint ${entrypoint.name}`, files @@ -47,7 +47,7 @@ export default class PagesManifestPlugin implements webpack.Plugin { } // Write filename, replace any backslashes in path (on windows) with forwardslashes for cross-platform consistency. - pages[pagePath] = files[0] + pages[pagePath] = files[files.length - 1] if (isWebpack5 && !this.dev) { pages[pagePath] = pages[pagePath].slice(3) diff --git a/packages/next/client/next-dev.js b/packages/next/client/next-dev.js index bd7a59ebde543..1b2fd9a3d1916 100644 --- a/packages/next/client/next-dev.js +++ b/packages/next/client/next-dev.js @@ -48,7 +48,7 @@ initNext({ webpackHMR }) const { pages } = JSON.parse(event.data) const router = window.next.router - if (pages.includes(router.pathname)) { + if (!router.clc && pages.includes(router.pathname)) { console.log('Refreshing page data due to server-side change') buildIndicatorHandler('building')