add support for dynamic requests in require() and import() (#7153)

### Description

Refactors `PatternMatching` to support a map of multiple matches by key.
Based on the new `RequestKey` support for resolving.

Code Generation for `require("./dir/"+ expr)` will emit a map of
`__turbopack_lookup__({ "./a": () => ..., "./b": () => ... }, "./dir/" +
expr)`.

The output format isn't really optimized yet, that's work for future
PRs.

### Testing Instructions

added a test case, enabled one previously skipped test case


Closes PACK-986

crates/turbo-tasks/src/debug/mod.rs

@@ -69,6 +69,12 @@ pub trait ValueDebugFormat {
     fn value_debug_format(&self, depth: usize) -> ValueDebugFormatString;
 }
 
+impl ValueDebugFormat for String {
+    fn value_debug_format(&self, _depth: usize) -> ValueDebugFormatString {
+        ValueDebugFormatString::Sync(format!("{:#?}", self))
+    }
+}
+
 // Use autoref specialization [1] to implement `ValueDebugFormat` for `T:
 // Debug` as a fallback if `T` does not implement it directly, hence the `for
 // &T` clause.

crates/turbopack-ecmascript-runtime/js/src/build/runtime.ts

@@ -30,7 +30,7 @@ type ExternalImport = (id: ModuleId) => Promise<Exports | EsmNamespaceObject>;
 type ResolveAbsolutePath = (modulePath?: string) => string;
 
 interface TurbopackNodeBuildContext extends TurbopackBaseContext {
-  p: ResolveAbsolutePath;
+  P: ResolveAbsolutePath;
   R: ResolvePathFromModule;
   x: ExternalRequire;
   y: ExternalImport;
@@ -179,6 +179,7 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
       i: esmImport.bind(null, module),
       s: esmExport.bind(null, module, module.exports),
       j: dynamicExport.bind(null, module, module.exports),
+      p: moduleLookup,
       v: exportValue.bind(null, module),
       n: exportNamespace.bind(null, module),
       m: module,
@@ -188,7 +189,7 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
       w: loadWebAssembly,
       u: loadWebAssemblyModule,
       g: globalThis,
-      p: resolveAbsolutePath,
+      P: resolveAbsolutePath,
       U: relativeURL,
       R: createResolvePathFromModule(r),
       __dirname: module.id.replace(/(^|\/)[\/]+$/, ""),

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/base/runtime-base.ts

@@ -330,6 +330,7 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
           i: esmImport.bind(null, module),
           s: esmExport.bind(null, module, module.exports),
           j: dynamicExport.bind(null, module, module.exports),
+          p: moduleLookup,
           v: exportValue.bind(null, module),
           n: exportNamespace.bind(null, module),
           m: module,

crates/turbopack-ecmascript-runtime/js/src/shared/runtime-types.d.ts

@@ -28,6 +28,11 @@ type EsmExport = (exportGetters: Record<string, () => any>) => void;
 type ExportValue = (value: any) => void;
 type ExportNamespace = (namespace: any) => void;
 type DynamicExport = (object: Record<string, any>) => void;
+type ModuleLookup = (
+  object: Record<string, any>,
+  name: string,
+  returnPromise?: boolean
+) => any;
 
 type LoadChunk = (chunkPath: ChunkPath) => Promise<any> | undefined;
 type LoadWebAssembly = (
@@ -61,6 +66,7 @@ interface TurbopackBaseContext {
   i: EsmImport;
   s: EsmExport;
   j: DynamicExport;
+  p: ModuleLookup;
   v: ExportValue;
   n: ExportNamespace;
   m: Module;

crates/turbopack-ecmascript-runtime/js/src/shared/runtime-utils.ts

@@ -142,6 +142,28 @@ function dynamicExport(
   }
 }
 
+/**
+ * Access one entry from a mapping from name to functor.
+ */
+function moduleLookup(
+  map: Record<string, () => any>,
+  name: string,
+  returnPromise: boolean = false
+) {
+  if (hasOwnProperty.call(map, name)) {
+    return map[name]();
+  }
+  const e = new Error(`Cannot find module '${name}'`);
+  (e as any).code = "MODULE_NOT_FOUND";
+  if (returnPromise) {
+    return Promise.resolve().then(() => {
+      throw e;
+    });
+  } else {
+    throw e;
+  }
+}
+
 function exportValue(module: Module, value: any) {
   module.exports = value;
 }

crates/turbopack-ecmascript/src/chunk/item.rs

@@ -91,7 +91,8 @@ impl EcmascriptChunkItemContent {
             "M: __turbopack_modules__",
             "l: __turbopack_load__",
             "j: __turbopack_dynamic__",
-            "p: __turbopack_resolve_absolute_path__",
+            "p: __turbopack_lookup__",
+            "P: __turbopack_resolve_absolute_path__",
             "U: __turbopack_relative_url__",
             "R: __turbopack_resolve_module_id_path__",
             "g: global",

crates/turbopack-ecmascript/src/references/amd.rs

@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
 use swc_core::{
     common::DUMMY_SP,
     ecma::{
-        ast::{CallExpr, Callee, Expr, ExprOrSpread},
+        ast::{CallExpr, Callee, Expr, ExprOrSpread, Lit},
         utils::private_ident,
     },
     quote, quote_expr,
@@ -83,11 +83,12 @@ impl ValueToString for AmdDefineAssetReference {
 #[turbo_tasks::value_impl]
 impl ChunkableModuleReference for AmdDefineAssetReference {}
 
-#[derive(
-    ValueDebugFormat, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, Copy, Clone,
-)]
+#[derive(ValueDebugFormat, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, Clone)]
 pub enum AmdDefineDependencyElement {
-    Request(Vc<Request>),
+    Request {
+        request: Vc<Request>,
+        request_str: String,
+    },
     Exports,
     Module,
     Require,
@@ -147,24 +148,25 @@ impl CodeGenerateable for AmdDefineWithDependenciesCodeGen {
             .iter()
             .map(|element| async move {
                 Ok(match element {
-                    AmdDefineDependencyElement::Request(request) => {
-                        ResolvedElement::PatternMapping(
-                            PatternMapping::resolve_request(
-                                *request,
+                    AmdDefineDependencyElement::Request {
+                        request,
+                        request_str,
+                    } => ResolvedElement::PatternMapping {
+                        pattern_mapping: PatternMapping::resolve_request(
+                            *request,
+                            self.origin,
+                            Vc::upcast(chunking_context),
+                            cjs_resolve(
                                 self.origin,
-                                Vc::upcast(chunking_context),
-                                cjs_resolve(
-                                    self.origin,
-                                    *request,
-                                    Some(self.issue_source),
-                                    try_to_severity(self.in_try),
-                                ),
-                                Value::new(ChunkItem),
-                            )
-                            .await?,
-                            request.await?.request(),
+                                *request,
+                                Some(self.issue_source),
+                                try_to_severity(self.in_try),
+                            ),
+                            Value::new(ChunkItem),
                         )
-                    }
+                        .await?,
+                        request_str: request_str.to_string(),
+                    },
                     AmdDefineDependencyElement::Exports => {
                         ResolvedElement::Expr(quote!("exports" as Expr))
                     }
@@ -193,7 +195,10 @@ impl CodeGenerateable for AmdDefineWithDependenciesCodeGen {
 }
 
 enum ResolvedElement {
-    PatternMapping(ReadRef<PatternMapping>, Option<String>),
+    PatternMapping {
+        pattern_mapping: ReadRef<PatternMapping>,
+        request_str: String,
+    },
     Expr(Expr),
 }
 
@@ -219,23 +224,14 @@ fn transform_amd_factory(
     let deps = resolved_elements
         .iter()
         .map(|element| match element {
-            ResolvedElement::PatternMapping(pm, req) => match &**pm {
-                PatternMapping::Invalid => quote_expr!("undefined"),
-                pm => {
-                    let arg = if let Some(req) = req {
-                        pm.apply(req.as_str().into())
-                    } else {
-                        pm.create()
-                    };
-
-                    if pm.is_internal_import() {
-                        quote_expr!("__turbopack_require__($arg)", arg: Expr = arg)
-                    } else {
-                        quote_expr!("__turbopack_external_require__($arg)", arg: Expr = arg)
-                    }
-                }
-            },
-            ResolvedElement::Expr(expr) => Box::new(expr.clone()),
+            ResolvedElement::PatternMapping {
+                pattern_mapping: pm,
+                request_str: request,
+            } => {
+                let key_expr = Expr::Lit(Lit::Str(request.as_str().into()));
+                pm.create_require(key_expr)
+            }
+            ResolvedElement::Expr(expr) => expr.clone(),
         })
         .map(ExprOrSpread::from)
         .collect();

crates/turbopack-ecmascript/src/references/cjs.rs

@@ -1,7 +1,8 @@
 use anyhow::Result;
 use swc_core::{
-    common::DUMMY_SP,
-    ecma::ast::{Callee, Expr, ExprOrSpread, Ident, ObjectLit},
+    common::{util::take::Take, DUMMY_SP},
+    ecma::ast::{CallExpr, Expr, ExprOrSpread, Ident, Lit},
+    quote,
 };
 use turbo_tasks::{Value, ValueToString, Vc};
 use turbopack_core::{
@@ -16,7 +17,7 @@ use crate::{
     chunk::EcmascriptChunkingContext,
     code_gen::{CodeGenerateable, CodeGeneration},
     create_visitor,
-    references::{util::throw_module_not_found_expr, AstPath},
+    references::AstPath,
     resolve::{cjs_resolve, try_to_severity},
 };
 
@@ -154,45 +155,29 @@ impl CodeGenerateable for CjsRequireAssetReference {
         let mut visitors = Vec::new();
 
         let path = &self.path.await?;
-        match &*pm {
-            PatternMapping::Invalid => {
-                let request_string = self.request.to_string().await?;
-                visitors.push(create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
-                    // In Node.js, a require call that cannot be resolved will throw an error.
-                    *expr = throw_module_not_found_expr(&request_string);
-                }));
-            }
-            PatternMapping::Ignored => {
-                visitors.push(create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
-                    // Ignored modules behave as if they have no code nor exports.
-                    *expr = Expr::Object(ObjectLit {
-                        span: DUMMY_SP,
-                        props: vec![],
-                    });
-                }));
-            }
-            _ => {
-                visitors.push(
-                    create_visitor!(exact path, visit_mut_call_expr(call_expr: &mut CallExpr) {
-                        call_expr.callee = Callee::Expr(
-                            Box::new(Expr::Ident(Ident::new(
-                                if pm.is_internal_import() {
-                                    "__turbopack_require__"
-                                } else {
-                                    "__turbopack_external_require__"
-                                }.into(), DUMMY_SP
-                            )))
-                        );
-                        let old_args = std::mem::take(&mut call_expr.args);
-                        let expr = match old_args.into_iter().next() {
-                            Some(ExprOrSpread { expr, spread: None }) => pm.apply(*expr),
-                            _ => pm.create(),
-                        };
-                        call_expr.args.push(ExprOrSpread { spread: None, expr: Box::new(expr) });
-                    }),
-                );
-            }
-        }
+        visitors.push(create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
+            let old_expr = expr.take();
+            let message = if let Expr::Call(CallExpr { args, ..}) = old_expr {
+                match args.into_iter().next() {
+                    Some(ExprOrSpread { spread: None, expr: key_expr }) => {
+                        *expr = pm.create_require(*key_expr);
+                        return;
+                    }
+                    Some(ExprOrSpread { spread: Some(_), expr: _ }) => {
+                        "spread operator is not analyse-able in require() expressions."
+                    }
+                    _ => {
+                        "require() expressions require at least 1 argument"
+                    }
+                }
+            } else {
+                "visitor must be executed on a CallExpr"
+            };
+            *expr = quote!(
+                "(() => { throw new Error($message); })()" as Expr,
+                message: Expr = Expr::Lit(Lit::Str(message.into()))
+            );
+        }));
 
         Ok(CodeGeneration { visitors }.into())
     }
@@ -278,27 +263,33 @@ impl CodeGenerateable for CjsRequireResolveAssetReference {
         let mut visitors = Vec::new();
 
         let path = &self.path.await?;
-        if let PatternMapping::Invalid = &*pm {
-            let request_string = self.request.to_string().await?;
-            visitors.push(create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
-                // In Node.js, a require.resolve call that cannot be resolved will throw an error.
-                *expr = throw_module_not_found_expr(&request_string);
-            }));
-        } else {
-            // Inline the result of the `require.resolve` call as a string literal.
-            visitors.push(create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
-                if let Expr::Call(call_expr) = expr {
-                    let args = std::mem::take(&mut call_expr.args);
-                    *expr = match args.into_iter().next() {
-                        Some(ExprOrSpread { expr, spread: None }) => pm.apply(*expr),
-                        _ => pm.create(),
-                    };
-                }
-                // CjsRequireResolveAssetReference will only be used for Expr::Call.
-                // Due to eventual consistency the path might match something else,
-                // but we can ignore that as it will be recomputed anyway.
-            }));
-        }
+        // Inline the result of the `require.resolve` call as a literal.
+        visitors.push(create_visitor!(path, visit_mut_expr(expr: &mut Expr) {
+            if let Expr::Call(call_expr) = expr {
+                let args = std::mem::take(&mut call_expr.args);
+                *expr = match args.into_iter().next() {
+                    Some(ExprOrSpread { expr, spread: None }) => pm.create_require(*expr),
+                    other => {
+                        let message = match other {
+                            // These are SWC bugs: https://github.com/swc-project/swc/issues/5394
+                            Some(ExprOrSpread { spread: Some(_), expr: _ }) => {
+                                "spread operator is not analyse-able in require() expressions."
+                            }
+                            _ => {
+                                "require() expressions require at least 1 argument"
+                            }
+                        };
+                        quote!(
+                            "(() => { throw new Error($message); })()" as Expr,
+                            message: Expr = Expr::Lit(Lit::Str(message.into()))
+                        )
+                    },
+                };
+            }
+            // CjsRequireResolveAssetReference will only be used for Expr::Call.
+            // Due to eventual consistency the path might match something else,
+            // but we can ignore that as it will be recomputed anyway.
+        }));
 
         Ok(CodeGeneration { visitors }.into())
     }