Populate sourcemap ignoreList when Webpack is used (#71821)

This enables DevTools (e.g. Chrome debugger) to collapse stackframes from 3rd party dependencies.

Webpack only. Turbopack added support in #71770. Replays from RSC will follow.



Had to fork `EvalSourceMapDevToolPlugin` (with blessing from @sokra) to be able to inject `ignoreList`. 

For `source-map`, we can use https://github.com/mondaychen/devtools-ignore-webpack-plugin/ instead since we can operate on the assets on disk. I inlined it to iterate on it faster. Though it'd be faster for bundling to also fork `SourceMapDevToolPlugin` since `DevToolsIgnorePlugin` adds another parse/serialize roundtrip.

## test plan

We'll start leveraging `ignoreList` in the terminal as well which will allow us to write automated tests. I haven't found a way to automatically test this ignore-listing in browsers.

Note that this is on Chrome Beta. Chrome Stable does not ignore-list logged stacks yet. Only stacks of the actual `console` call or in the debugger.
(the frame from our console instrumentation is a bug that may be fixed once we populate our own sourcemaps)
`pnpm debug dev test/e2e/app-dir/server-source-maps/fixtures/default/`
`/ssr-error-log` shows
browser:
![CleanShot 2024-10-24 at 20 37 43](https://github.com/user-attachments/assets/fec14ec7-4c2c-4ce3-8454-1e43a0c8d0a7)
![CleanShot 2024-10-24 at 20 37 52](https://github.com/user-attachments/assets/53a233be-448e-4110-93c5-b5c4f6f98b6b)

Node.js debugger doesn't seem to work. Will look at that in a follow-up

packages/next/src/build/webpack/config/blocks/base.ts

@@ -2,6 +2,19 @@ import curry from 'next/dist/compiled/lodash.curry'
 import type { webpack } from 'next/dist/compiled/webpack/webpack'
 import { COMPILER_NAMES } from '../../../../shared/lib/constants'
 import type { ConfigurationContext } from '../utils'
+import DevToolsIgnorePlugin from '../../plugins/devtools-ignore-list-plugin'
+import EvalSourceMapDevToolPlugin from '../../plugins/eval-source-map-dev-tool-plugin'
+
+function shouldIgnorePath(modulePath: string): boolean {
+  // TODO: How to ignore list 'webpack:///../../../src/shared/lib/is-thenable.ts'
+  return (
+    modulePath.includes('node_modules') ||
+    // would filter 'webpack://_N_E/./app/page.tsx'
+    // modulePath.startsWith('webpack://_N_E/') ||
+    // e.g. 'webpack:///external commonjs "next/dist/compiled/next-server/app-page.runtime.dev.js"'
+    modulePath.includes('next/dist')
+  )
+}
 
 export const base = curry(function base(
   ctx: ConfigurationContext,
@@ -29,7 +42,14 @@ export const base = curry(function base(
       // original source, including columns and original variable names.
       // This is desirable so the in-browser debugger can correctly pause
       // and show scoped variables with their original names.
-      config.devtool = 'eval-source-map'
+      // We're using a fork of `eval-source-map`
+      config.devtool = false
+      config.plugins ??= []
+      config.plugins.push(
+        new EvalSourceMapDevToolPlugin({
+          shouldIgnorePath,
+        })
+      )
     }
   } else {
     if (
@@ -39,6 +59,14 @@ export const base = curry(function base(
       (ctx.productionBrowserSourceMaps && ctx.isClient)
     ) {
       config.devtool = 'source-map'
+      config.plugins ??= []
+      config.plugins.push(
+        new DevToolsIgnorePlugin({
+          // TODO: eval-source-map has different module paths than source-map.
+          // We're currently not actually ignore listing anything.
+          shouldIgnorePath,
+        })
+      )
     } else {
       config.devtool = false
     }

packages/next/src/build/webpack/plugins/devtools-ignore-list-plugin.ts

@@ -0,0 +1,93 @@
+// Source: https://github.com/mondaychen/devtools-ignore-webpack-plugin/blob/e35ce41d9606a92a455ef247f509a1c2ccab5778/src/index.ts
+
+import { webpack } from 'next/dist/compiled/webpack/webpack'
+
+// Following the naming conventions from
+// https://tc39.es/source-map/#source-map-format
+const IGNORE_LIST = 'ignoreList'
+
+const PLUGIN_NAME = 'devtools-ignore-plugin'
+
+interface SourceMap {
+  sources: string[]
+  [IGNORE_LIST]: number[]
+}
+
+interface PluginOptions {
+  shouldIgnorePath?: (path: string) => boolean
+  isSourceMapAsset?: (name: string) => boolean
+}
+
+interface ValidatedOptions extends PluginOptions {
+  shouldIgnorePath: Required<PluginOptions>['shouldIgnorePath']
+  isSourceMapAsset: Required<PluginOptions>['isSourceMapAsset']
+}
+
+function defaultShouldIgnorePath(path: string): boolean {
+  return path.includes('/node_modules/') || path.includes('/webpack/')
+}
+
+function defaultIsSourceMapAsset(name: string): boolean {
+  return name.endsWith('.map')
+}
+
+/**
+ * This plugin adds a field to source maps that identifies which sources are
+ * vendored or runtime-injected (aka third-party) sources. These are consumed by
+ * Chrome DevTools to automatically ignore-list sources.
+ */
+export default class DevToolsIgnorePlugin {
+  options: ValidatedOptions
+
+  constructor(options: PluginOptions = {}) {
+    this.options = {
+      shouldIgnorePath: options.shouldIgnorePath ?? defaultShouldIgnorePath,
+      isSourceMapAsset: options.isSourceMapAsset ?? defaultIsSourceMapAsset,
+    }
+  }
+
+  apply(compiler: webpack.Compiler) {
+    const { RawSource } = compiler.webpack.sources
+
+    compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => {
+      compilation.hooks.processAssets.tap(
+        {
+          name: PLUGIN_NAME,
+          stage: webpack.Compilation.PROCESS_ASSETS_STAGE_DEV_TOOLING,
+          additionalAssets: true,
+        },
+        (assets) => {
+          for (const [name, asset] of Object.entries(assets)) {
+            // Instead of using `asset.map()` to fetch the source maps from
+            // SourceMapSource assets, process them directly as a RawSource.
+            // This is because `.map()` is slow and can take several seconds.
+            if (!this.options.isSourceMapAsset(name)) {
+              // Ignore non source map files.
+              continue
+            }
+
+            const mapContent = asset.source().toString()
+            if (!mapContent) {
+              continue
+            }
+
+            const sourcemap = JSON.parse(mapContent) as SourceMap
+
+            const ignoreList = []
+            for (const [index, path] of sourcemap.sources.entries()) {
+              if (this.options.shouldIgnorePath(path)) {
+                ignoreList.push(index)
+              }
+            }
+
+            sourcemap[IGNORE_LIST] = ignoreList
+            compilation.updateAsset(
+              name,
+              new RawSource(JSON.stringify(sourcemap))
+            )
+          }
+        }
+      )
+    })
+  }
+}

packages/next/src/build/webpack/plugins/eval-source-map-dev-tool-plugin.ts

@@ -0,0 +1,249 @@
+/*
+	MIT License http://www.opensource.org/licenses/mit-license.php
+	Author Tobias Koppers @sokra
+*/
+import {
+  type webpack,
+  type SourceMapDevToolPluginOptions,
+  ConcatenatedModule,
+  makePathsAbsolute,
+  ModuleFilenameHelpers,
+  NormalModule,
+  RuntimeGlobals,
+  SourceMapDevToolModuleOptionsPlugin,
+} from 'next/dist/compiled/webpack/webpack'
+import type { RawSourceMap } from 'next/dist/compiled/source-map'
+
+const cache = new WeakMap<webpack.sources.Source, webpack.sources.Source>()
+
+const devtoolWarningMessage = `/*
+ * ATTENTION: An "eval-source-map" devtool has been used.
+ * This devtool is neither made for production nor for readable output files.
+ * It uses "eval()" calls to create a separate source file with attached SourceMaps in the browser devtools.
+ * If you are trying to read the output file, select a different devtool (https://webpack.js.org/configuration/devtool/)
+ * or disable the default devtool with "devtool: false".
+ * If you are looking for production-ready output files, see mode: "production" (https://webpack.js.org/configuration/mode/).
+ */
+`
+
+// @ts-expect-error -- can't compare `string` with `number` in `version`Ï
+interface SourceMap extends RawSourceMap {
+  ignoreList?: number[]
+  version: number
+}
+
+export interface EvalSourceMapDevToolPluginOptions
+  extends SourceMapDevToolPluginOptions {
+  // Fork
+  shouldIgnorePath?: (modulePath: string) => boolean
+}
+
+// Fork of webpack's EvalSourceMapDevToolPlugin with support for adding `ignoreList`.
+// https://github.com/webpack/webpack/blob/e237b580e2bda705c5ab39973f786f7c5a7026bc/lib/EvalSourceMapDevToolPlugin.js#L37
+export default class EvalSourceMapDevToolPlugin {
+  sourceMapComment: string
+  moduleFilenameTemplate: NonNullable<
+    EvalSourceMapDevToolPluginOptions['moduleFilenameTemplate']
+  >
+  namespace: NonNullable<EvalSourceMapDevToolPluginOptions['namespace']>
+  options: EvalSourceMapDevToolPluginOptions
+  shouldIgnorePath: (modulePath: string) => boolean
+
+  /**
+   * @param {SourceMapDevToolPluginOptions|string} inputOptions Options object
+   */
+  constructor(inputOptions: EvalSourceMapDevToolPluginOptions) {
+    let options: EvalSourceMapDevToolPluginOptions
+    if (typeof inputOptions === 'string') {
+      options = {
+        append: inputOptions,
+      }
+    } else {
+      options = inputOptions
+    }
+    this.sourceMapComment =
+      options.append && typeof options.append !== 'function'
+        ? options.append
+        : '//# sourceURL=[module]\n//# sourceMappingURL=[url]'
+    this.moduleFilenameTemplate =
+      options.moduleFilenameTemplate ||
+      'webpack://[namespace]/[resource-path]?[hash]'
+    this.namespace = options.namespace || ''
+    this.options = options
+
+    // fork
+    this.shouldIgnorePath = options.shouldIgnorePath ?? (() => false)
+  }
+
+  /**
+   * Apply the plugin
+   * @param compiler the compiler instance
+   */
+  apply(compiler: webpack.Compiler): void {
+    const options = this.options
+    compiler.hooks.compilation.tap(
+      'NextJSEvalSourceMapDevToolPlugin',
+      (compilation) => {
+        const { JavascriptModulesPlugin } = compiler.webpack.javascript
+        const { RawSource, ConcatSource } = compiler.webpack.sources
+
+        const devtoolWarning = new RawSource(devtoolWarningMessage)
+
+        const hooks = JavascriptModulesPlugin.getCompilationHooks(compilation)
+
+        new SourceMapDevToolModuleOptionsPlugin(options).apply(compilation)
+        const matchModule = ModuleFilenameHelpers.matchObject.bind(
+          ModuleFilenameHelpers,
+          options
+        )
+
+        hooks.renderModuleContent.tap(
+          'NextJSEvalSourceMapDevToolPlugin',
+          (source, m, { chunk, runtimeTemplate, chunkGraph }) => {
+            const cachedSource = cache.get(source)
+            if (cachedSource !== undefined) {
+              return cachedSource
+            }
+
+            const result = (
+              r: webpack.sources.Source
+            ): webpack.sources.Source => {
+              cache.set(source, r)
+              return r
+            }
+
+            if (m instanceof NormalModule) {
+              const module = m
+              if (!matchModule(module.resource)) {
+                return result(source)
+              }
+            } else if (m instanceof ConcatenatedModule) {
+              const concatModule = m
+              if (concatModule.rootModule instanceof NormalModule) {
+                const module = concatModule.rootModule
+                if (!matchModule(module.resource)) {
+                  return result(source)
+                }
+              } else {
+                return result(source)
+              }
+            } else {
+              return result(source)
+            }
+
+            const namespace = compilation.getPath(this.namespace, {
+              chunk,
+            })
+            let sourceMap: SourceMap
+            let content
+            if (source.sourceAndMap) {
+              const sourceAndMap = source.sourceAndMap(options)
+              sourceMap = sourceAndMap.map as SourceMap
+              content = sourceAndMap.source
+            } else {
+              sourceMap = source.map(options) as SourceMap
+              content = source.source()
+            }
+            if (!sourceMap) {
+              return result(source)
+            }
+
+            // Clone (flat) the sourcemap to ensure that the mutations below do not persist.
+            sourceMap = { ...sourceMap }
+            const context = compiler.options.context!
+            const root = compiler.root
+            const modules = sourceMap.sources.map((sourceMapSource) => {
+              if (!sourceMapSource.startsWith('webpack://'))
+                return sourceMapSource
+              sourceMapSource = makePathsAbsolute(
+                context,
+                sourceMapSource.slice(10),
+                root
+              )
+              const module = compilation.findModule(sourceMapSource)
+              return module || sourceMapSource
+            })
+            let moduleFilenames = modules.map((module) =>
+              ModuleFilenameHelpers.createFilename(
+                module,
+                {
+                  moduleFilenameTemplate: this.moduleFilenameTemplate,
+                  namespace,
+                },
+                {
+                  requestShortener: runtimeTemplate.requestShortener,
+                  chunkGraph,
+                  // @ts-expect-error -- Original code
+                  hashFunction: compilation.outputOptions.hashFunction,
+                }
+              )
+            )
+            moduleFilenames = ModuleFilenameHelpers.replaceDuplicates(
+              moduleFilenames,
+              (filename, _i, n) => {
+                for (let j = 0; j < n; j++) filename += '*'
+                return filename
+              }
+            )
+            sourceMap.sources = moduleFilenames
+            sourceMap.ignoreList = []
+            for (let index = 0; index < moduleFilenames.length; index++) {
+              if (this.shouldIgnorePath(moduleFilenames[index])) {
+                sourceMap.ignoreList.push(index)
+              }
+            }
+            if (options.noSources) {
+              sourceMap.sourcesContent = undefined
+            }
+            sourceMap.sourceRoot = options.sourceRoot || ''
+            const moduleId =
+              /** @type {ModuleId} */
+              chunkGraph.getModuleId(m)
+            sourceMap.file =
+              typeof moduleId === 'number' ? `${moduleId}.js` : moduleId
+
+            const footer = `${this.sourceMapComment.replace(
+              /\[url\]/g,
+              `data:application/json;charset=utf-8;base64,${Buffer.from(
+                JSON.stringify(sourceMap),
+                'utf8'
+              ).toString('base64')}`
+            )}\n//# sourceURL=webpack-internal:///${moduleId}\n` // workaround for chrome bug
+
+            return result(
+              new RawSource(
+                `eval(${
+                  compilation.outputOptions.trustedTypes
+                    ? `${RuntimeGlobals.createScript}(${JSON.stringify(
+                        content + footer
+                      )})`
+                    : JSON.stringify(content + footer)
+                });`
+              )
+            )
+          }
+        )
+        hooks.inlineInRuntimeBailout.tap(
+          'EvalDevToolModulePlugin',
+          () => 'the eval-source-map devtool is used.'
+        )
+        hooks.render.tap(
+          'EvalSourceMapDevToolPlugin',
+          (source) => new ConcatSource(devtoolWarning, source)
+        )
+        hooks.chunkHash.tap('EvalSourceMapDevToolPlugin', (_chunk, hash) => {
+          hash.update('EvalSourceMapDevToolPlugin')
+          hash.update('2')
+        })
+        if (compilation.outputOptions.trustedTypes) {
+          compilation.hooks.additionalModuleRuntimeRequirements.tap(
+            'EvalSourceMapDevToolPlugin',
+            (_module, set, _context) => {
+              set.add(RuntimeGlobals.createScript)
+            }
+          )
+        }
+      }
+    )
+  }
+}