refactor(conditional-page-build): explicitly persist status of genera…

…ted html files

docs/docs/conditional-page-builds.md

@@ -50,12 +50,8 @@ Done in 154.501 sec
 
 - To enable this build option you will need to set an environment variable, which requires access to do so in your build environment.
 
-- This feature is not available with `gatsby develop`.
-
 - You should not try to use this flag alongside Incremental Builds in Gatsby Cloud, as it uses a different process and may conflict with it.
 
 - You will need to persist the `.cache` and `public` directories between builds. This allows for comparisons and reuse of previously built files. If `.cache` directory was not persisted then a full build will be triggered. If `public` directory was not persisted then you might experience failing builds or builds that are missing certain assets.
 
 - Any code changes (templates, components, source handling, new plugins etc) will prompt the creation of a new webpack compilation hash and trigger a full build.
-
-Note: When using the `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES` flag it is important to do so consistently when building your project. Otherwise, the cache will be cleared and the necessary data for comparison will no longer be available, removing the ability to check for incremental data changes.

packages/gatsby/src/commands/build-html.ts

@@ -6,7 +6,7 @@ import telemetry from "gatsby-telemetry"
 import { chunk } from "lodash"
 import webpack from "webpack"
 
-import { emitter } from "../redux"
+import { emitter, store } from "../redux"
 import webpackConfig from "../utils/webpack.config"
 import { structureWebpackErrors } from "../utils/webpack-error-utils"
 
@@ -153,6 +153,11 @@ const renderHTMLQueue = async (
       paths: pageSegment,
     })
 
+    store.dispatch({
+      type: `HTML_GENERATED`,
+      payload: pageSegment,
+    })
+
     if (activity && activity.tick) {
       activity.tick(pageSegment.length)
     }

packages/gatsby/src/commands/build-utils.ts

@@ -6,49 +6,11 @@ import {
   getPageHtmlFilePath,
 } from "../utils/page-html"
 import { removePageData, fixedPagePath } from "../utils/page-data"
-import { IGatsbyState } from "../redux/types"
+import { store } from "../redux"
 
 const checkFolderIsEmpty = (path: string): boolean =>
   fs.existsSync(path) && !fs.readdirSync(path).length
 
-export const getChangedPageDataKeys = (
-  state: IGatsbyState,
-  cachedPageData: Map<string, string>
-): Array<string> => {
-  if (cachedPageData && state.pageData) {
-    const pageKeys: Array<string> = []
-    state.pageData.forEach((newPageDataHash: string, key: string) => {
-      if (!cachedPageData.has(key)) {
-        pageKeys.push(key)
-      } else {
-        const previousPageDataHash = cachedPageData.get(key)
-        if (newPageDataHash !== previousPageDataHash) {
-          pageKeys.push(key)
-        }
-      }
-    })
-    return pageKeys
-  }
-
-  return [...state.pages.keys()]
-}
-
-export const collectRemovedPageData = (
-  state: IGatsbyState,
-  cachedPageData: Map<string, string>
-): Array<string> => {
-  if (cachedPageData && state.pageData) {
-    const deletedPageKeys: Array<string> = []
-    cachedPageData.forEach((_value: string, key: string) => {
-      if (!state.pageData.has(key)) {
-        deletedPageKeys.push(key)
-      }
-    })
-    return deletedPageKeys
-  }
-  return []
-}
-
 const checkAndRemoveEmptyDir = (publicDir: string, pagePath: string): void => {
   const pageHtmlDirectory = path.dirname(
     getPageHtmlFilePath(publicDir, pagePath)
@@ -78,9 +40,16 @@ export const removePageFiles = async (
   publicDir: string,
   pageKeys: Array<string>
 ): Promise<void> => {
-  const removePages = pageKeys.map(pagePath =>
-    removePageHtmlFile({ publicDir }, pagePath)
-  )
+  const removePages = pageKeys.map(pagePath => {
+    const removePromise = removePageHtmlFile({ publicDir }, pagePath)
+    removePromise.then(() => {
+      store.dispatch({
+        type: `HTML_REMOVED`,
+        payload: pagePath,
+      })
+    })
+    return removePromise
+  })
 
   const removePageDataList = pageKeys.map(pagePath =>
     removePageData(publicDir, pagePath)

packages/gatsby/src/commands/build.ts

@@ -5,6 +5,7 @@ import fs from "fs-extra"
 import telemetry from "gatsby-telemetry"
 
 import { doBuildPages, buildRenderer, deleteRenderer } from "./build-html"
+import { calcDirtyHtmlFiles } from "../utils/page-html"
 import { buildProductionBundle } from "./build-javascript"
 import { bootstrap } from "../bootstrap"
 import apiRunnerNode from "../utils/api-runner-node"
@@ -40,15 +41,6 @@ import {
 } from "../utils/webpack-status"
 import { updateSiteMetadata } from "gatsby-core-utils"
 
-let cachedPageData
-let cachedWebpackCompilationHash
-if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
-  const { pageData, webpackCompilationHash } = readState()
-  // extract only data that we need to reuse and let v8 garbage collect rest of state
-  cachedPageData = pageData
-  cachedWebpackCompilationHash = webpackCompilationHash
-}
-
 interface IBuildArgs extends IProgram {
   directory: string
   sitePackageJson: PackageJson
@@ -173,19 +165,6 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
   await flushPendingPageDataWrites()
   markWebpackStatusAsDone()
 
-  if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
-    const { pages } = store.getState()
-    if (cachedPageData) {
-      cachedPageData.forEach((_value, key) => {
-        if (!pages.has(key)) {
-          boundActionCreators.removePageData({
-            id: key,
-          })
-        }
-      })
-    }
-  }
-
   if (telemetry.isTrackingEnabled()) {
     // transform asset size to kB (from bytes) to fit 64 bit to numbers
     const bundleSizes = stats
@@ -210,27 +189,13 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
   // we need to save it again to make sure our latest state has been saved
   await db.saveState()
 
-  let pagePaths = [...store.getState().pages.keys()]
-
-  // Rebuild subset of pages if user opt into GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES
-  // if there were no source files (for example components, static queries, etc) changes since last build, otherwise rebuild all pages
-  if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
-    if (
-      cachedWebpackCompilationHash === store.getState().webpackCompilationHash
-    ) {
-      pagePaths = buildUtils.getChangedPageDataKeys(
-        store.getState(),
-        cachedPageData
-      )
-    } else if (cachedWebpackCompilationHash) {
-      report.info(
-        report.stripIndent(`
-          One or more of your source files have changed since the last time you ran Gatsby. All
-          pages will be rebuilt.
-        `)
-      )
-    }
-  }
+  const { toRegenerate, toDelete } = process.env
+    .GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES
+    ? calcDirtyHtmlFiles(store.getState())
+    : {
+        toRegenerate: [...store.getState().pages.keys()],
+        toDelete: [],
+      }
 
   const buildSSRBundleActivityProgress = report.activityTimer(
     `Building HTML renderer`,
@@ -248,7 +213,7 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
 
   const buildHTMLActivityProgress = report.createProgress(
     `Building static HTML for pages`,
-    pagePaths.length,
+    toRegenerate.length,
     0,
     {
       parentSpan: buildSpan,
@@ -258,7 +223,7 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
   try {
     await doBuildPages(
       pageRenderer,
-      pagePaths,
+      toRegenerate,
       buildHTMLActivityProgress,
       workerPool
     )
@@ -293,17 +258,12 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
     }
   }
 
-  let deletedPageKeys: Array<string> = []
-  if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
+  if (toDelete.length > 0) {
     const deletePageDataActivityTimer = report.activityTimer(
       `Delete previous page data`
     )
     deletePageDataActivityTimer.start()
-    deletedPageKeys = buildUtils.collectRemovedPageData(
-      store.getState(),
-      cachedPageData
-    )
-    await buildUtils.removePageFiles(publicDir, deletedPageKeys)
+    await buildUtils.removePageFiles(publicDir, toDelete)
 
     deletePageDataActivityTimer.end()
   }
@@ -336,17 +296,17 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
     process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES &&
     process.argv.includes(`--log-pages`)
   ) {
-    if (pagePaths.length) {
+    if (toRegenerate.length) {
       report.info(
-        `Built pages:\n${pagePaths
+        `Built pages:\n${toRegenerate
           .map(path => `Updated page: ${path}`)
           .join(`\n`)}`
       )
     }
 
-    if (deletedPageKeys.length) {
+    if (toDelete.length) {
       report.info(
-        `Deleted pages:\n${deletedPageKeys
+        `Deleted pages:\n${toDelete
           .map(path => `Deleted page: ${path}`)
           .join(`\n`)}`
       )
@@ -361,16 +321,16 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
       `${program.directory}/.cache`,
       `newPages.txt`
     )
-    const createdFilesContent = pagePaths.length
-      ? `${pagePaths.join(`\n`)}\n`
+    const createdFilesContent = toRegenerate.length
+      ? `${toRegenerate.join(`\n`)}\n`
       : ``
 
     const deletedFilesPath = path.resolve(
       `${program.directory}/.cache`,
       `deletedPages.txt`
     )
-    const deletedFilesContent = deletedPageKeys.length
-      ? `${deletedPageKeys.join(`\n`)}\n`
+    const deletedFilesContent = toDelete.length
+      ? `${toDelete.join(`\n`)}\n`
       : ``
 
     await fs.writeFile(createdFilesPath, createdFilesContent, `utf8`)

packages/gatsby/src/commands/develop-process.ts

@@ -101,15 +101,6 @@ module.exports = async (program: IDevelopArgs): Promise<void> => {
     }
   )
 
-  if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
-    reporter.panic(
-      `The flag ${chalk.yellow(
-        `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES`
-      )} is not available with ${chalk.cyan(
-        `gatsby develop`
-      )}, please retry using ${chalk.cyan(`gatsby build`)}`
-    )
-  }
   initTracer(program.openTracingConfigFile)
   markWebpackStatusAsPending()
   reporter.pendingActivity({ id: `webpack-develop` })

packages/gatsby/src/query/query-runner.ts

@@ -203,17 +203,8 @@ export async function queryRunner(
     path: queryJob.id,
     componentPath: queryJob.componentPath,
     isPage: queryJob.isPage,
+    resultHash,
   })
 
-  // Sets pageData to the store, here for easier access to the resultHash
-  if (
-    process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES &&
-    queryJob.isPage
-  ) {
-    boundActionCreators.setPageData({
-      id: queryJob.id,
-      resultHash,
-    })
-  }
   return result
 }

packages/gatsby/src/redux/actions/internal.ts

@@ -225,15 +225,25 @@ export const setProgramStatus = (
  * @private
  */
 export const pageQueryRun = (
-  { path, componentPath, isPage },
+  {
+    path,
+    componentPath,
+    isPage,
+    resultHash,
+  }: {
+    path: string
+    componentPath: string
+    isPage: boolean
+    resultHash: string
+  },
   plugin: IGatsbyPlugin,
   traceId?: string
 ): IPageQueryRunAction => {
   return {
     type: `PAGE_QUERY_RUN`,
     plugin,
     traceId,
-    payload: { path, componentPath, isPage },
+    payload: { path, componentPath, isPage, resultHash },
   }
 }
 

packages/gatsby/src/redux/actions/public.js

@@ -1378,19 +1378,6 @@ actions.setPageData = (pageData: PageData) => {
   }
 }
 
-/**
- * Remove page data from the store.
- *
- * @param {Object} $0
- * @param {string} $0.id the path to the page.
- */
-actions.removePageData = (id: PageDataRemove) => {
-  return {
-    type: `REMOVE_PAGE_DATA`,
-    payload: id,
-  }
-}
-
 /**
  * Record that a page was visited on the server..
  *

packages/gatsby/src/redux/index.ts

@@ -103,10 +103,10 @@ export const saveState = (): void => {
     staticQueryComponents: state.staticQueryComponents,
     webpackCompilationHash: state.webpackCompilationHash,
     pageDataStats: state.pageDataStats,
-    pageData: state.pageData,
     pendingPageDataWrites: state.pendingPageDataWrites,
     staticQueriesByTemplate: state.staticQueriesByTemplate,
     queries: state.queries,
+    html: state.html,
   })
 }