From 5a0a714a11dd9f44a0220d63159299411365d18c Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Sun, 14 Feb 2021 16:35:20 +0100 Subject: [PATCH 1/3] feat(build): conditional page builds are no longer experimental (they are default) --- .circleci/config.yml | 11 ---- .../overview-of-the-gatsby-build-process.md | 2 +- docs/docs/conditional-page-builds.md | 55 ------------------- .../artifacts/__tests__/index.js | 12 ++-- packages/gatsby-cli/src/create-cli.ts | 10 ++-- packages/gatsby/src/commands/build-html.ts | 10 +--- packages/gatsby/src/commands/build.ts | 10 +--- packages/gatsby/src/services/initialize.ts | 10 ++-- 8 files changed, 21 insertions(+), 99 deletions(-) delete mode 100644 docs/docs/conditional-page-builds.md diff --git a/.circleci/config.yml b/.circleci/config.yml index b6037621bc440..6cc1a86c8e5f6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -258,15 +258,6 @@ jobs: - e2e-test: test_path: integration-tests/artifacts - # temporary - integration_tests_artifacts_conditional_page_builds: - executor: node - steps: - - e2e-test: - test_path: integration-tests/artifacts - environment: - GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES: 1 - integration_tests_ssr: executor: node steps: @@ -606,8 +597,6 @@ workflows: <<: *e2e-test-workflow - integration_tests_artifacts: <<: *e2e-test-workflow - - integration_tests_artifacts_conditional_page_builds: - <<: *e2e-test-workflow - integration_tests_ssr: <<: *e2e-test-workflow - integration_tests_images: diff --git a/docs/docs/conceptual/overview-of-the-gatsby-build-process.md b/docs/docs/conceptual/overview-of-the-gatsby-build-process.md index 6d648abc51cbf..9c89148d881c6 100644 --- a/docs/docs/conceptual/overview-of-the-gatsby-build-process.md +++ b/docs/docs/conceptual/overview-of-the-gatsby-build-process.md @@ -304,7 +304,7 @@ Page queries that were queued up earlier from query extraction are run so the da With everything ready for the HTML pages in place, HTML is compiled and written out to files so it can be served up statically. Since HTML is being produced in a Node.js server context, [references to browser APIs like `window` can break the build](/docs/debugging-html-builds/) and must be conditionally applied. -By default, Gatsby rebuilds static HTML for all pages on each build. There is an experimental feature flag `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES` which enables [conditional page builds](/docs/conditional-page-builds/). +Gatsby will smartly rebuild only needed HTML files. This might mean 0 html files being generated if nothing that was used for html files changed, some part of pages in case if data that is used changed or all files in case of code change. ## What do you get from a successful build? diff --git a/docs/docs/conditional-page-builds.md b/docs/docs/conditional-page-builds.md deleted file mode 100644 index ccc19a03f405a..0000000000000 --- a/docs/docs/conditional-page-builds.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -title: Conditional Page Builds ---- - -If you have a large site, you may be able to improve build times for data updates by enabling an experimental feature called "conditional page builds". While this is not as fast as true [Incremental Builds](https://support.gatsbyjs.com/hc/en-us/articles/360053099253-Distributed-Builds-and-Incremental-Builds) available in Gatsby Cloud, it can save time on the HTML-generation step by not re-rendering HTML for pages with unchanged data. This feature is experimental, but _may_ improve build times for sites with a large number of complex pages. Test it thoroughly with your site before deploying to production. - -For more info on the standard build process, please see the [overview of the Gatsby build process](/docs/conceptual/overview-of-the-gatsby-build-process/). - -## How to use - -To enable conditional page builds, use the environment variable `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES=true` in your `gatsby build` command, for example: - -`GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES=true gatsby build --log-pages` - -This will run the Gatsby build process, but only build pages that have data changes since your last build. If there are any changes to code (JS, CSS) the bundling process returns a new webpack compilation hash which causes all pages to be rebuilt. - -### Reporting what has been built - -You may want to retrieve a list of the pages that were built. For example, if you want to perform a sync action in your CI/CD pipeline. - -To list the paths in the build assets (`public`) folder, you can use one (or both) of the following arguments in your `build` command. - -- `--log-pages` parameter will output all the file paths that were updated or deleted at the end of the build stage. - -```shell -success Building production JavaScript and CSS bundles - 82.198s -success run queries - 82.762s - 4/4 0.05/s -success Building static HTML for pages - 19.386s - 2/2 0.10/s -+ success Delete previous page data - 1.512s -info Done building in 152.084 sec -+ info Built pages: -+ Updated page: /about -+ Updated page: /accounts/example -+ info Deleted pages: -+ Deleted page: /test - -Done in 154.501 sec -``` - -- `--write-to-file` creates two files in the `.cache` folder, with lists of the changed paths in the build assets (`public`) folder. - - - `newPages.txt` will contain a list of new or changed paths - - `deletedPages.txt` will contain a list of deleted paths - -## More information - -- This feature works by comparing the page data from the previous build to the new page data. This creates a list of page directories that are passed to the static build process. - -- To enable this build option you will need to set an environment variable, which requires access to do so in your build environment. - -- You should not try to use this flag alongside Incremental Builds in Gatsby Cloud, as it uses a different process and may conflict with it. - -- You will need to persist the `.cache` and `public` directories between builds. This allows for comparisons and reuse of previously built files. If `.cache` directory was not persisted then a full build will be triggered. If `public` directory was not persisted then you might experience failing builds or builds that are missing certain assets. - -- Any code changes (templates, components, source handling, new plugins etc) will prompt the creation of a new webpack compilation hash and trigger a full build. diff --git a/integration-tests/artifacts/__tests__/index.js b/integration-tests/artifacts/__tests__/index.js index 2f05bd5619e76..94fdad3d220e6 100644 --- a/integration-tests/artifacts/__tests__/index.js +++ b/integration-tests/artifacts/__tests__/index.js @@ -463,13 +463,11 @@ describe(`Second run (different pages created, data changed)`, () => { }) }) - if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) { - it(`should recreate only some html files`, () => { - expect(manifest[runNumber].generated.sort()).toEqual( - expectedPagesToBeGenerated.sort() - ) - }) - } + it(`should recreate only some html files`, () => { + expect(manifest[runNumber].generated.sort()).toEqual( + expectedPagesToBeGenerated.sort() + ) + }) }) describe(`page-data files`, () => { diff --git a/packages/gatsby-cli/src/create-cli.ts b/packages/gatsby-cli/src/create-cli.ts index bccffe11ee8ea..2518fad2500bc 100644 --- a/packages/gatsby-cli/src/create-cli.ts +++ b/packages/gatsby-cli/src/create-cli.ts @@ -243,18 +243,20 @@ function buildLocalCommands(cli: yargs.Argv, isLocalSite: boolean): void { type: `string`, describe: `Tracer configuration file (OpenTracing compatible). See https://gatsby.dev/tracing`, }) - // log-pages and write-to-file are specific to experimental GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES feature - // because of that they are hidden from `--help` but still defined so `yargs` know about them + // log-pages and write-to-file were added specifically to experimental GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES feature + // in gatsby@2. They are useful, but not very applicable (specifically `--write-to-file`) as generic approach, as it only + // list pages without other artifacts, so it's useful in very narrow scope. Because we don't have alternative right now + // those toggles are kept for users that rely on them, but we won't promote them and will keep them "hidden". .option(`log-pages`, { type: `boolean`, default: false, - describe: `Log the pages that changes since last build (only available when using GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES).`, + describe: `Log the pages that changes since last build.`, hidden: true, }) .option(`write-to-file`, { type: `boolean`, default: false, - describe: `Save the log of changed pages for future comparison (only available when using GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES).`, + describe: `Save the log of changed pages for future comparison.`, hidden: true, }), handler: handlerP( diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts index 399a736d3dba1..7812eedfdf617 100644 --- a/packages/gatsby/src/commands/build-html.ts +++ b/packages/gatsby/src/commands/build-html.ts @@ -271,13 +271,9 @@ export async function buildHTMLPagesAndDeleteStaleArtifacts({ }> { buildUtils.markHtmlDirtyIfResultOfUsedStaticQueryChanged() - const { toRegenerate, toDelete } = process.env - .GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES - ? buildUtils.calcDirtyHtmlFiles(store.getState()) - : { - toRegenerate: [...store.getState().pages.keys()], - toDelete: [], - } + const { toRegenerate, toDelete } = buildUtils.calcDirtyHtmlFiles( + store.getState() + ) if (toRegenerate.length > 0) { const buildHTMLActivityProgress = reporter.createProgress( diff --git a/packages/gatsby/src/commands/build.ts b/packages/gatsby/src/commands/build.ts index 79c99a74827ce..b2d066dc9f0b0 100644 --- a/packages/gatsby/src/commands/build.ts +++ b/packages/gatsby/src/commands/build.ts @@ -233,10 +233,7 @@ module.exports = async function build(program: IBuildArgs): Promise { workerPool.end() buildActivity.end() - if ( - process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES && - process.argv.includes(`--log-pages`) - ) { + if (process.argv.includes(`--log-pages`)) { if (toRegenerate.length) { report.info( `Built pages:\n${toRegenerate @@ -254,10 +251,7 @@ module.exports = async function build(program: IBuildArgs): Promise { } } - if ( - process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES && - process.argv.includes(`--write-to-file`) - ) { + if (process.argv.includes(`--write-to-file`)) { const createdFilesPath = path.resolve( `${program.directory}/.cache`, `newPages.txt` diff --git a/packages/gatsby/src/services/initialize.ts b/packages/gatsby/src/services/initialize.ts index 0ab178fdbff30..03c5f1f07b3f2 100644 --- a/packages/gatsby/src/services/initialize.ts +++ b/packages/gatsby/src/services/initialize.ts @@ -297,14 +297,12 @@ export async function initialize({ const cacheJsonDirExists = fs.existsSync(`${cacheDirectory}/json`) const publicDirExists = fs.existsSync(publicDirectory) - // During builds, delete html and css files from the public directory as we don't want - // deleted pages and styles from previous builds to stick around. - // For Conditional Page Builds, we do want to remove those when there is `public` dir - // but cache doesn't exist + // For builds in case public dir exists, but cache doesn't, we need to clean up potentially stale + // artifacts from previous builds (due to cache not being available, we can't rely on tracking of artifacts) if ( process.env.NODE_ENV === `production` && - (!process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES || - (publicDirExists && !cacheJsonDirExists)) + publicDirExists && + !cacheJsonDirExists ) { activity = reporter.activityTimer( `delete html and css files from previous builds`, From 1048eda5d3c0ae16c55c3296280359bff559488f Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 17 Feb 2021 16:51:19 +0100 Subject: [PATCH 2/3] Update docs/docs/conceptual/overview-of-the-gatsby-build-process.md Co-authored-by: Ward Peeters --- docs/docs/conceptual/overview-of-the-gatsby-build-process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/conceptual/overview-of-the-gatsby-build-process.md b/docs/docs/conceptual/overview-of-the-gatsby-build-process.md index 9c89148d881c6..f235d9141ab74 100644 --- a/docs/docs/conceptual/overview-of-the-gatsby-build-process.md +++ b/docs/docs/conceptual/overview-of-the-gatsby-build-process.md @@ -304,7 +304,7 @@ Page queries that were queued up earlier from query extraction are run so the da With everything ready for the HTML pages in place, HTML is compiled and written out to files so it can be served up statically. Since HTML is being produced in a Node.js server context, [references to browser APIs like `window` can break the build](/docs/debugging-html-builds/) and must be conditionally applied. -Gatsby will smartly rebuild only needed HTML files. This might mean 0 html files being generated if nothing that was used for html files changed, some part of pages in case if data that is used changed or all files in case of code change. +Gatsby will smartly rebuild only needed HTML files. This can result in no HTML files being generated if nothing used for HTML files changed. The opposite can also be true and lead to a full site rebuild when data is used on all pages or when a code change happens. ## What do you get from a successful build? From 39e908016f14817821d57985b82ac44079b8a156 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 17 Feb 2021 18:30:14 +0100 Subject: [PATCH 3/3] fix cli integration tests --- integration-tests/gatsby-cli/__tests__/build.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration-tests/gatsby-cli/__tests__/build.js b/integration-tests/gatsby-cli/__tests__/build.js index f12bea4ad92a6..2e27036e58672 100644 --- a/integration-tests/gatsby-cli/__tests__/build.js +++ b/integration-tests/gatsby-cli/__tests__/build.js @@ -15,9 +15,6 @@ describe(`gatsby build`, () => { logs.should.contain(`success open and validate gatsby-configs`) logs.should.contain(`success load plugins`) logs.should.contain(`success onPreInit`) - logs.should.contain( - `success delete html and css files from previous builds` - ) logs.should.contain(`success initialize cache`) logs.should.contain(`success copy gatsby files`) logs.should.contain(`success onPreBootstrap`)