[WIP] Reload webpack if needed (#2076)

* Reload webpack via hot-reloader when needed.
We need to do this specially we removed a previosly
built page from the filesystem.

* Make sure reloading is happen only once

* Reload only if there's a missing page error.

* Remove debug logs.

* 2.4.2

* Refactor the codebase a bit.

* Move some commonly used regexp to a utils module.

* Handle the reloading well when there's a custom error page.

* Add a HMR test case.

* Close the browser in the test case.

server/build/plugins/pages-plugin.js

@@ -1,17 +1,19 @@
+import {
+  IS_BUNDLED_PAGE,
+  MATCH_ROUTE_NAME
+} from '../../utils'
+
 export default class PagesPlugin {
   apply (compiler) {
-    const isBundledPage = /^bundles[/\\]pages.*\.js$/
-    const matchRouteName = /^bundles[/\\]pages[/\\](.*)\.js$/
-
     compiler.plugin('after-compile', (compilation, callback) => {
       const pages = Object
         .keys(compilation.namedChunks)
         .map(key => compilation.namedChunks[key])
-        .filter(chunk => isBundledPage.test(chunk.name))
+        .filter(chunk => IS_BUNDLED_PAGE.test(chunk.name))
 
       pages.forEach((chunk) => {
         const page = compilation.assets[chunk.name]
-        const pageName = matchRouteName.exec(chunk.name)[1]
+        const pageName = MATCH_ROUTE_NAME.exec(chunk.name)[1]
         let routeName = `/${pageName.replace(/[/\\]?index$/, '')}`
 
         // We need to convert \ into / when we are in windows

server/hot-reloader.js

@@ -1,13 +1,14 @@
 import { join, relative, sep } from 'path'
-import webpackDevMiddleware from 'webpack-dev-middleware'
-import webpackHotMiddleware from 'webpack-hot-middleware'
+import WebpackDevMiddleware from 'webpack-dev-middleware'
+import WebpackHotMiddleware from 'webpack-hot-middleware'
 import onDemandEntryHandler from './on-demand-entry-handler'
 import isWindowsBash from 'is-windows-bash'
 import webpack from './build/webpack'
 import clean from './build/clean'
 import getConfig from './config'
-
-const isBundledPage = /^bundles[/\\]pages.*\.js$/
+import {
+  IS_BUNDLED_PAGE
+} from './utils'
 
 export default class HotReloader {
   constructor (dir, { quiet, conf } = {}) {
@@ -44,22 +45,53 @@ export default class HotReloader {
       clean(this.dir)
     ])
 
-    this.prepareMiddlewares(compiler)
+    const buildTools = await this.prepareBuildTools(compiler)
+    this.assignBuildTools(buildTools)
+
     this.stats = await this.waitUntilValid()
   }
 
-  async stop () {
-    if (this.webpackDevMiddleware) {
+  async stop (webpackDevMiddleware) {
+    const middleware = webpackDevMiddleware || this.webpackDevMiddleware
+    if (middleware) {
       return new Promise((resolve, reject) => {
-        this.webpackDevMiddleware.close((err) => {
+        middleware.close((err) => {
           if (err) return reject(err)
           resolve()
         })
       })
     }
   }
 
-  async prepareMiddlewares (compiler) {
+  async reload () {
+    this.stats = null
+
+    const [compiler] = await Promise.all([
+      webpack(this.dir, { dev: true, quiet: this.quiet }),
+      clean(this.dir)
+    ])
+
+    const buildTools = await this.prepareBuildTools(compiler)
+    this.stats = await this.waitUntilValid(buildTools.webpackDevMiddleware)
+
+    const oldWebpackDevMiddleware = this.webpackDevMiddleware
+
+    this.assignBuildTools(buildTools)
+    await this.stop(oldWebpackDevMiddleware)
+  }
+
+  assignBuildTools ({ webpackDevMiddleware, webpackHotMiddleware, onDemandEntries }) {
+    this.webpackDevMiddleware = webpackDevMiddleware
+    this.webpackHotMiddleware = webpackHotMiddleware
+    this.onDemandEntries = onDemandEntries
+    this.middlewares = [
+      webpackDevMiddleware,
+      webpackHotMiddleware,
+      onDemandEntries.middleware()
+    ]
+  }
+
+  async prepareBuildTools (compiler) {
     compiler.plugin('after-emit', (compilation, callback) => {
       const { assets } = compilation
 
@@ -83,7 +115,7 @@ export default class HotReloader {
       const chunkNames = new Set(
         compilation.chunks
           .map((c) => c.name)
-          .filter(name => isBundledPage.test(name))
+          .filter(name => IS_BUNDLED_PAGE.test(name))
       )
 
       const failedChunkNames = new Set(compilation.errors
@@ -95,7 +127,7 @@ export default class HotReloader {
 
       const chunkHashes = new Map(
         compilation.chunks
-          .filter(c => isBundledPage.test(c.name))
+          .filter(c => IS_BUNDLED_PAGE.test(c.name))
           .map((c) => [c.name, c.hash])
       )
 
@@ -163,33 +195,38 @@ export default class HotReloader {
       webpackDevMiddlewareConfig = this.config.webpackDevMiddleware(webpackDevMiddlewareConfig)
     }
 
-    this.webpackDevMiddleware = webpackDevMiddleware(compiler, webpackDevMiddlewareConfig)
+    const webpackDevMiddleware = WebpackDevMiddleware(compiler, webpackDevMiddlewareConfig)
 
-    this.webpackHotMiddleware = webpackHotMiddleware(compiler, {
+    const webpackHotMiddleware = WebpackHotMiddleware(compiler, {
       path: '/_next/webpack-hmr',
       log: false,
       heartbeat: 2500
     })
-    this.onDemandEntries = onDemandEntryHandler(this.webpackDevMiddleware, compiler, {
+    const onDemandEntries = onDemandEntryHandler(webpackDevMiddleware, compiler, {
       dir: this.dir,
       dev: true,
+      reload: this.reload.bind(this),
       ...this.config.onDemandEntries
     })
 
-    this.middlewares = [
-      this.webpackDevMiddleware,
-      this.webpackHotMiddleware,
-      this.onDemandEntries.middleware()
-    ]
+    return {
+      webpackDevMiddleware,
+      webpackHotMiddleware,
+      onDemandEntries
+    }
   }
 
-  waitUntilValid () {
+  waitUntilValid (webpackDevMiddleware) {
+    const middleware = webpackDevMiddleware || this.webpackDevMiddleware
     return new Promise((resolve) => {
-      this.webpackDevMiddleware.waitUntilValid(resolve)
+      middleware.waitUntilValid(resolve)
     })
   }
 
-  getCompilationErrors () {
+  async getCompilationErrors () {
+    // When we are reloading, we need to wait until it's reloaded properly.
+    await this.onDemandEntries.waitUntilReloaded()
+
     if (!this.compilationErrors) {
       this.compilationErrors = new Map()
 

server/index.js

@@ -161,7 +161,7 @@ export default class Server {
             return await renderScriptError(req, res, page, error, {}, this.renderOpts)
           }
 
-          const compilationErr = this.getCompilationError(page)
+          const compilationErr = await this.getCompilationError(page, req, res)
           if (compilationErr) {
             const customFields = { statusCode: 500 }
             return await renderScriptError(req, res, page, compilationErr, customFields, this.renderOpts)
@@ -240,7 +240,7 @@ export default class Server {
 
   async renderToHTML (req, res, pathname, query) {
     if (this.dev) {
-      const compilationErr = this.getCompilationError(pathname)
+      const compilationErr = await this.getCompilationError(pathname)
       if (compilationErr) {
         res.statusCode = 500
         return this.renderErrorToHTML(compilationErr, req, res, pathname, query)
@@ -268,7 +268,7 @@ export default class Server {
 
   async renderErrorToHTML (err, req, res, pathname, query) {
     if (this.dev) {
-      const compilationErr = this.getCompilationError('/_error')
+      const compilationErr = await this.getCompilationError('/_error')
       if (compilationErr) {
         res.statusCode = 500
         return renderErrorToHTML(compilationErr, req, res, pathname, query, this.renderOpts)
@@ -349,10 +349,10 @@ export default class Server {
     return true
   }
 
-  getCompilationError (page) {
+  async getCompilationError (page, req, res) {
     if (!this.hotReloader) return
 
-    const errors = this.hotReloader.getCompilationErrors()
+    const errors = await this.hotReloader.getCompilationErrors()
     if (!errors.size) return
 
     const id = join(this.dir, this.dist, 'bundles', 'pages', page)

server/on-demand-entry-handler.js

@@ -4,6 +4,7 @@ import { join } from 'path'
 import { parse } from 'url'
 import resolvePath from './resolve'
 import touch from 'touch'
+import { MATCH_ROUTE_NAME, IS_BUNDLED_PAGE } from './utils'
 
 const ADDED = Symbol('added')
 const BUILDING = Symbol('building')
@@ -12,13 +13,17 @@ const BUILT = Symbol('built')
 export default function onDemandEntryHandler (devMiddleware, compiler, {
   dir,
   dev,
+  reload,
   maxInactiveAge = 1000 * 25
 }) {
-  const entries = {}
-  const lastAccessPages = ['']
-  const doneCallbacks = new EventEmitter()
+  let entries = {}
+  let lastAccessPages = ['']
+  let doneCallbacks = new EventEmitter()
   const invalidator = new Invalidator(devMiddleware)
   let touchedAPage = false
+  let reloading = false
+  let stopped = false
+  let reloadCallbacks = new EventEmitter()
 
   compiler.plugin('make', function (compilation, done) {
     invalidator.startBuilding()
@@ -35,6 +40,27 @@ export default function onDemandEntryHandler (devMiddleware, compiler, {
   })
 
   compiler.plugin('done', function (stats) {
+    const { compilation } = stats
+    const hardFailedPages = compilation.errors
+      .filter(e => {
+        // Make sure to only pick errors which marked with missing modules
+        const hasNoModuleFoundError = /ENOENT/.test(e.message) || /Module not found/.test(e.message)
+        if (!hasNoModuleFoundError) return false
+
+        // The page itself is missing. So this is a failed page.
+        if (IS_BUNDLED_PAGE.test(e.module.name)) return true
+
+        // No dependencies means this is a top level page.
+        // So this is a failed page.
+        return e.module.dependencies.length === 0
+      })
+      .map(e => e.module.chunks)
+      .reduce((a, b) => [...a, ...b], [])
+      .map(c => {
+        const pageName = MATCH_ROUTE_NAME.exec(c.name)[1]
+        return normalizePage(`/${pageName}`)
+      })
+
     // Call all the doneCallbacks
     Object.keys(entries).forEach((page) => {
       const entryInfo = entries[page]
@@ -57,14 +83,48 @@ export default function onDemandEntryHandler (devMiddleware, compiler, {
     })
 
     invalidator.doneBuilding()
+
+    if (hardFailedPages.length > 0 && !reloading) {
+      console.log(`> Reloading webpack due to inconsistant state of pages(s): ${hardFailedPages.join(', ')}`)
+      reloading = true
+      reload()
+        .then(() => {
+          console.log('> Webpack reloaded.')
+          reloadCallbacks.emit('done')
+          stop()
+        })
+        .catch(err => {
+          console.error(`> Webpack reloading failed: ${err.message}`)
+          console.error(err.stack)
+          process.exit(1)
+        })
+    }
   })
 
-  setInterval(function () {
+  const disposeHandler = setInterval(function () {
+    if (stopped) return
     disposeInactiveEntries(devMiddleware, entries, lastAccessPages, maxInactiveAge)
   }, 5000)
 
+  function stop () {
+    clearInterval(disposeHandler)
+    stopped = true
+    doneCallbacks = null
+    reloadCallbacks = null
+  }
+
   return {
+    waitUntilReloaded () {
+      if (!reloading) return Promise.resolve(true)
+      return new Promise((resolve) => {
+        reloadCallbacks.once('done', function () {
+          resolve()
+        })
+      })
+    },
+
     async ensurePage (page) {
+      await this.waitUntilReloaded()
       page = normalizePage(page)
 
       const pagePath = join(dir, 'pages', page)
@@ -103,31 +163,49 @@ export default function onDemandEntryHandler (devMiddleware, compiler, {
     },
 
     middleware () {
-      return function (req, res, next) {
-        if (!/^\/_next\/on-demand-entries-ping/.test(req.url)) return next()
-
-        const { query } = parse(req.url, true)
-        const page = normalizePage(query.page)
-        const entryInfo = entries[page]
-
-        // If there's no entry.
-        // Then it seems like an weird issue.
-        if (!entryInfo) {
-          const message = `Client pings, but there's no entry for page: ${page}`
-          console.error(message)
-          sendJson(res, { invalid: true })
-          return
-        }
+      return (req, res, next) => {
+        if (stopped) {
+          // If this handler is stopped, we need to reload the user's browser.
+          // So the user could connect to the actually running handler.
+          res.statusCode = 302
+          res.setHeader('Location', req.url)
+          res.end('302')
+        } else if (reloading) {
+          // Webpack config is reloading. So, we need to wait until it's done and
+          // reload user's browser.
+          // So the user could connect to the new handler and webpack setup.
+          this.waitUntilReloaded()
+            .then(() => {
+              res.statusCode = 302
+              res.setHeader('Location', req.url)
+              res.end('302')
+            })
+        } else {
+          if (!/^\/_next\/on-demand-entries-ping/.test(req.url)) return next()
+
+          const { query } = parse(req.url, true)
+          const page = normalizePage(query.page)
+          const entryInfo = entries[page]
+
+          // If there's no entry.
+          // Then it seems like an weird issue.
+          if (!entryInfo) {
+            const message = `Client pings, but there's no entry for page: ${page}`
+            console.error(message)
+            sendJson(res, { invalid: true })
+            return
+          }
 
-        sendJson(res, { success: true })
+          sendJson(res, { success: true })
 
-        // We don't need to maintain active state of anything other than BUILT entries
-        if (entryInfo.status !== BUILT) return
+          // We don't need to maintain active state of anything other than BUILT entries
+          if (entryInfo.status !== BUILT) return
 
-        // If there's an entryInfo
-        lastAccessPages.pop()
-        lastAccessPages.unshift(page)
-        entryInfo.lastActiveTime = Date.now()
+          // If there's an entryInfo
+          lastAccessPages.pop()
+          lastAccessPages.unshift(page)
+          entryInfo.lastActiveTime = Date.now()
+        }
       }
     }
   }

server/utils.js

@@ -0,0 +1,2 @@
+export const IS_BUNDLED_PAGE = /^bundles[/\\]pages.*\.js$/
+export const MATCH_ROUTE_NAME = /^bundles[/\\]pages[/\\](.*)\.js$/