Reduce excess file system writes by implementing query queue

docs/docs/gatsby-starters.md

@@ -288,11 +288,11 @@ Community:
   * Colorful homepage, and also includes a Landing Page and Generic Page components.
   * Many elements are available, including buttons, forms, tables, and pagination.
   * Styling with SCSS
-  
+
 * [gatsby-firebase-authentication](https://github.com/rwieruch/gatsby-firebase-authentication) [(demo)](https://react-firebase-authentication.wieruch.com/)
-  
+
   Features:
-  
+
   * Sign In, Sign Up, Sign Out
   * Password Forget
   * Password Change

packages/gatsby/package.json

@@ -24,6 +24,7 @@
     "babel-runtime": "^6.26.0",
     "babel-traverse": "^6.24.1",
     "babylon": "^6.17.3",
+    "better-queue": "^3.8.6",
     "bluebird": "^3.5.0",
     "chalk": "^1.1.3",
     "chokidar": "^1.7.0",
@@ -66,6 +67,7 @@
     "lodash": "^4.17.4",
     "lodash-id": "^0.14.0",
     "lowdb": "^0.16.2",
+    "md5": "^2.2.1",
     "md5-file": "^3.1.1",
     "mime": "^1.3.6",
     "mitt": "^1.1.2",

packages/gatsby/src/bootstrap/page-hot-reloader.js

@@ -61,6 +61,8 @@ const runCreatePages = async () => {
       deleteComponentsDependencies([page.path])
       deletePage(page)
     })
+
+  emitter.emit(`CREATE_PAGE_END`)
 }
 
 const debouncedCreatePages = _.debounce(runCreatePages, 100)

packages/gatsby/src/commands/data-explorer.js

@@ -24,4 +24,3 @@ module.exports = async (program: any) => {
   console.log(`Gatsby data explorer running at`, `http://${host}:${port}`)
   app.listen(port, host)
 }
-

packages/gatsby/src/internal-plugins/query-runner/page-query-runner.js

@@ -6,10 +6,9 @@
  */
 
 const _ = require(`lodash`)
-const async = require(`async`)
 
+const queue = require(`./query-queue`)
 const { store, emitter } = require(`../../redux`)
-const queryRunner = require(`./query-runner`)
 
 let queuedDirtyActions = []
 let active = false
@@ -95,34 +94,27 @@ const findIdsWithoutDataDependencies = () => {
 }
 
 const runQueriesForIds = ids => {
-  ids = _.uniq(ids)
-  if (ids.length < 1) {
+  const state = store.getState()
+  const pagesAndLayouts = [...state.pages, ...state.layouts]
+  let didNotQueueItems = true
+  ids.forEach(id => {
+    const plObj = pagesAndLayouts.find(
+      pl => pl.path === id || `LAYOUT___${pl.id}` === id
+    )
+    if (plObj) {
+      didNotQueueItems = false
+      queue.push({ ...plObj, _id: plObj.id, id: plObj.jsonName })
+    }
+  })
+
+  if (didNotQueueItems || !ids || ids.length === 0) {
     return Promise.resolve()
   }
-  const state = store.getState()
 
-  return new Promise((resolve, reject) => {
-    async.mapLimit(
-      ids,
-      4,
-      (id, callback) => {
-        const pagesAndLayouts = [...state.pages, ...state.layouts]
-        const plObj = pagesAndLayouts.find(
-          pl => pl.path === id || `LAYOUT___${pl.id}` === id
-        )
-        if (plObj) {
-          return queryRunner(plObj, state.components[plObj.component]).then(
-            result => callback(null, result),
-            error => callback(error)
-          )
-        } else {
-          return callback(null, null)
-        }
-      },
-      (error, result) => {
-        error ? reject(error) : resolve(result)
-      }
-    )
+  return new Promise(resolve => {
+    queue.on(`drain`, () => {
+      resolve()
+    })
   })
 }
 

packages/gatsby/src/internal-plugins/query-runner/pages-writer.js

@@ -54,11 +54,6 @@ const writePages = async () => {
   pageLayouts = _.uniq(pageLayouts)
   components = _.uniqBy(components, c => c.componentChunkName)
 
-  await fs.writeFile(
-    joinPath(program.directory, `.cache/pages.json`),
-    JSON.stringify(pagesData, null, 4)
-  )
-
   // Create file with sync requires of layouts/components/json files.
   let syncRequires = `// prefer default export if available
 const preferDefault = m => m && m.default || m
@@ -93,10 +88,6 @@ const preferDefault = m => m && m.default || m
     .join(`,\n`)}
 }`
 
-  await fs.writeFile(
-    `${program.directory}/.cache/sync-requires.js`,
-    syncRequires
-  )
   // Create file with async requires of layouts/components/json files.
   let asyncRequires = `// prefer default export if available
 const preferDefault = m => m && m.default || m
@@ -131,10 +122,17 @@ const preferDefault = m => m && m.default || m
     .join(`,\n`)}
 }`
 
-  await fs.writeFile(
-    joinPath(program.directory, `.cache/async-requires.js`),
-    asyncRequires
-  )
+  await Promise.all([
+    fs.writeFile(
+      joinPath(program.directory, `.cache/pages.json`),
+      JSON.stringify(pagesData, null, 4)
+    ),
+    fs.writeFile(`${program.directory}/.cache/sync-requires.js`, syncRequires),
+    fs.writeFile(
+      joinPath(program.directory, `.cache/async-requires.js`),
+      asyncRequires
+    ),
+  ])
 
   return
 }
@@ -143,15 +141,19 @@ exports.writePages = writePages
 
 let bootstrapFinished = false
 let oldPages
-const debouncedWritePages = _.debounce(() => {
-  // Don't write pages again until bootstrap has finished.
-  if (bootstrapFinished && !_.isEqual(oldPages, store.getState().pages)) {
-    writePages()
-    oldPages = store.getState().pages
-  }
-}, 250)
+const debouncedWritePages = _.debounce(
+  () => {
+    // Don't write pages again until bootstrap has finished.
+    if (bootstrapFinished && !_.isEqual(oldPages, store.getState().pages)) {
+      writePages()
+      oldPages = store.getState().pages
+    }
+  },
+  500,
+  { leading: true }
+)
 
-emitter.on(`CREATE_PAGE`, () => {
+emitter.on(`CREATE_PAGE_END`, () => {
   debouncedWritePages()
 })
 emitter.on(`DELETE_PAGE`, () => {

packages/gatsby/src/internal-plugins/query-runner/query-queue.js

@@ -0,0 +1,17 @@
+const Queue = require(`better-queue`)
+
+const queryRunner = require(`./query-runner`)
+const { store } = require(`../../redux`)
+
+const queue = new Queue(
+  (plObj, callback) => {
+    const state = store.getState()
+    return queryRunner(plObj, state.components[plObj.component]).then(
+      result => callback(null, result),
+      error => callback(error)
+    )
+  },
+  { concurrent: 4 }
+)
+
+module.exports = queue

packages/gatsby/src/internal-plugins/query-runner/query-runner.js

@@ -1,12 +1,16 @@
 import { graphql as graphqlFunction } from "graphql"
 const fs = require(`fs-extra`)
 const report = require(`gatsby-cli/lib/reporter`)
+const md5 = require(`md5`)
 
 const { joinPath } = require(`../../utils/path`)
 const { store } = require(`../../redux`)
 
+const resultHashes = {}
+
 // Run query for a page
 module.exports = async (pageOrLayout, component) => {
+  pageOrLayout.id = pageOrLayout._id
   const { schema, program } = store.getState()
 
   const graphql = (query, context) =>
@@ -51,10 +55,17 @@ module.exports = async (pageOrLayout, component) => {
     contextKey = `layoutContext`
   }
   result[contextKey] = pageOrLayout.context
-  const resultJSON = JSON.stringify(result, null, 4)
-
-  await fs.writeFile(
-    joinPath(program.directory, `.cache`, `json`, pageOrLayout.jsonName),
-    resultJSON
+  const resultJSON = JSON.stringify(result)
+  const resultHash = md5(resultJSON)
+  const resultPath = joinPath(
+    program.directory,
+    `.cache`,
+    `json`,
+    pageOrLayout.jsonName
   )
+
+  if (resultHashes[resultPath] !== resultHash) {
+    resultHashes[resultPath] = resultHash
+    await fs.writeFile(resultPath, resultJSON)
+  }
 }

packages/gatsby/src/internal-plugins/query-runner/query-watcher.js

@@ -10,12 +10,11 @@
 
 const _ = require(`lodash`)
 const chokidar = require(`chokidar`)
-const async = require(`async`)
 
 const { store } = require(`../../redux/`)
 const { boundActionCreators } = require(`../../redux/actions`)
 const queryCompiler = require(`./query-compiler`).default
-const queryRunner = require(`./query-runner`)
+const queue = require(`./query-queue`)
 const invariant = require(`invariant`)
 const normalize = require(`normalize-path`)
 
@@ -57,21 +56,12 @@ const runQueriesForComponent = componentPath => {
   boundActionCreators.deleteComponentsDependencies(
     pages.map(p => p.path || p.id)
   )
-  const component = store.getState().components[componentPath]
-  return new Promise((resolve, reject) => {
-    async.mapLimit(
-      pages,
-      4,
-      (page, callback) => {
-        queryRunner(page, component).then(
-          result => callback(null, result),
-          error => callback(error)
-        )
-      },
-      (error, result) => {
-        error ? reject(error) : resolve(result)
-      }
-    )
+  pages.forEach(page =>
+    queue.push({ ...page, _id: page.id, id: page.jsonName })
+  )
+
+  return new Promise(resolve => {
+    queue.on(`drain`, () => resolve())
   })
 }
 

scripts/publish-site.sh

@@ -1,5 +1,5 @@
-
-yarn global add gatsby-dev-cli
+yarn bootstrap
+npm install -g gatsby-dev-cli
 gatsby-dev --set-path-to-repo .
 
 echo "=== Installing the website dependencies"

yarn.lock

@@ -1528,6 +1528,18 @@ better-assert@~1.0.0:
   dependencies:
     callsite "1.0.0"
 
+better-queue-memory@^1.0.1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/better-queue-memory/-/better-queue-memory-1.0.2.tgz#aa6d169aa1d0cc77409185cb9cb5c7dc251bcd41"
+
+better-queue@^3.8.6:
+  version "3.8.6"
+  resolved "https://registry.yarnpkg.com/better-queue/-/better-queue-3.8.6.tgz#73220bdfab403924cffa7497220dd387abb73a63"
+  dependencies:
+    better-queue-memory "^1.0.1"
+    node-eta "^0.9.0"
+    uuid "^3.0.0"
+
 big.js@^3.1.3:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e"
@@ -7705,7 +7717,7 @@ md5.js@^1.3.4:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
-md5@^2.0.0:
+md5@^2.0.0, md5@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/md5/-/md5-2.2.1.tgz#53ab38d5fe3c8891ba465329ea23fac0540126f9"
   dependencies:
@@ -8161,6 +8173,10 @@ node-emoji@^1.0.4:
   dependencies:
     lodash.toarray "^4.4.0"
 
+node-eta@^0.9.0:
+  version "0.9.0"
+  resolved "https://registry.yarnpkg.com/node-eta/-/node-eta-0.9.0.tgz#9fb0b099bcd2a021940e603c64254dc003d9a7a8"
+
 node-fetch@^1.0.1:
   version "1.7.3"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef"