refactor to have different types of externals (#7363)

### Description

<!--
  ✍️ Write a short summary of your work.
  If necessary, include relevant screenshots.
-->

### Testing Instructions

<!--
  Give a quick description of steps to test your changes.
-->


Closes PACK-2481

crates/turbopack-core/src/environment.rs

@@ -109,6 +109,19 @@ impl Environment {
         })
     }
 
+    #[turbo_tasks::function]
+    pub async fn supports_esm_externals(self: Vc<Self>) -> Result<Vc<bool>> {
+        let this = self.await?;
+        Ok(match this.execution {
+            ExecutionEnvironment::NodeJsBuildTime(..) | ExecutionEnvironment::NodeJsLambda(_) => {
+                Vc::cell(true)
+            }
+            ExecutionEnvironment::Browser(_) => Vc::cell(false),
+            ExecutionEnvironment::EdgeWorker(_) => Vc::cell(false),
+            ExecutionEnvironment::Custom(_) => todo!(),
+        })
+    }
+
     #[turbo_tasks::function]
     pub async fn supports_commonjs_externals(self: Vc<Self>) -> Result<Vc<bool>> {
         let this = self.await?;

crates/turbopack-core/src/resolve/mod.rs

@@ -65,8 +65,7 @@ use crate::{error::PrettyPrintError, issue::IssueSeverity};
 pub enum ModuleResolveResultItem {
     Module(Vc<Box<dyn Module>>),
     OutputAsset(Vc<Box<dyn OutputAsset>>),
-    OriginalReferenceExternal,
-    OriginalReferenceTypeExternal(String),
+    External(String, ExternalType),
     Ignore,
     Empty,
     Custom(u8),
@@ -363,11 +362,30 @@ impl ModuleResolveResultOption {
     }
 }
 
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize, TraceRawVcs)]
+pub enum ExternalType {
+    OriginalReference,
+    Url,
+    CommonJs,
+    EcmaScriptModule,
+}
+
+impl Display for ExternalType {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            ExternalType::OriginalReference => write!(f, "original reference"),
+            ExternalType::CommonJs => write!(f, "commonjs"),
+            ExternalType::EcmaScriptModule => write!(f, "esm"),
+            ExternalType::Url => write!(f, "url"),
+        }
+    }
+}
+
 #[turbo_tasks::value(shared)]
 #[derive(Clone, Debug)]
 pub enum ResolveResultItem {
     Source(Vc<Box<dyn Source>>),
-    OriginalReferenceTypeExternal(String),
+    External(String, ExternalType),
     Ignore,
     Empty,
     Custom(u8),
@@ -442,9 +460,10 @@ impl ValueToString for ResolveResult {
                 ResolveResultItem::Source(a) => {
                     result.push_str(&a.ident().to_string().await?);
                 }
-                ResolveResultItem::OriginalReferenceTypeExternal(s) => {
+                ResolveResultItem::External(s, ty) => {
                     result.push_str("external ");
                     result.push_str(s);
+                    write!(result, " ({})", ty)?;
                 }
                 ResolveResultItem::Ignore => {
                     result.push_str("ignore");
@@ -633,8 +652,8 @@ impl ResolveResult {
                             request,
                             match item {
                                 ResolveResultItem::Source(source) => asset_fn(source).await?,
-                                ResolveResultItem::OriginalReferenceTypeExternal(s) => {
-                                    ModuleResolveResultItem::OriginalReferenceTypeExternal(s)
+                                ResolveResultItem::External(s, ty) => {
+                                    ModuleResolveResultItem::External(s, ty)
                                 }
                                 ResolveResultItem::Ignore => ModuleResolveResultItem::Ignore,
                                 ResolveResultItem::Empty => ModuleResolveResultItem::Empty,
@@ -1641,7 +1660,7 @@ async fn resolve_internal_inline(
                 let uri = format!("{}{}", protocol, remainder);
                 ResolveResult::primary_with_key(
                     RequestKey::new(uri.clone()),
-                    ResolveResultItem::OriginalReferenceTypeExternal(uri),
+                    ResolveResultItem::External(uri, ExternalType::Url),
                 )
                 .into()
             }

crates/turbopack-core/src/resolve/options.rs

@@ -9,7 +9,7 @@ use turbo_tasks_fs::{glob::Glob, FileSystemPath};
 
 use super::{
     alias_map::{AliasMap, AliasTemplate},
-    AliasPattern, ResolveResult, ResolveResultItem,
+    AliasPattern, ExternalType, ResolveResult, ResolveResultItem,
 };
 use crate::resolve::{parse::Request, plugin::ResolvePlugin};
 
@@ -92,7 +92,7 @@ pub enum ResolveInPackage {
 #[turbo_tasks::value(shared)]
 #[derive(Clone)]
 pub enum ImportMapping {
-    External(Option<String>),
+    External(Option<String>, ExternalType),
     /// An already resolved result that will be returned directly.
     Direct(Vc<ResolveResult>),
     /// A request alias that will be resolved first, and fall back to resolving
@@ -131,11 +131,11 @@ impl AliasTemplate for Vc<ImportMapping> {
         Box::pin(async move {
             let this = &*self.await?;
             Ok(match this {
-                ImportMapping::External(name) => {
+                ImportMapping::External(name, ty) => {
                     if let Some(name) = name {
-                        ImportMapping::External(Some(name.clone().replace('*', capture)))
+                        ImportMapping::External(Some(name.clone().replace('*', capture)), *ty)
                     } else {
-                        ImportMapping::External(None)
+                        ImportMapping::External(None, *ty)
                     }
                 }
                 ImportMapping::PrimaryAlternative(name, context) => {
@@ -271,11 +271,11 @@ async fn import_mapping_to_result(
 ) -> Result<ImportMapResult> {
     Ok(match &*mapping.await? {
         ImportMapping::Direct(result) => ImportMapResult::Result(*result),
-        ImportMapping::External(name) => ImportMapResult::Result(
+        ImportMapping::External(name, ty) => ImportMapResult::Result(
             ResolveResult::primary(if let Some(name) = name {
-                ResolveResultItem::OriginalReferenceTypeExternal(name.to_string())
+                ResolveResultItem::External(name.to_string(), *ty)
             } else if let Some(request) = request.await?.request() {
-                ResolveResultItem::OriginalReferenceTypeExternal(request)
+                ResolveResultItem::External(request, *ty)
             } else {
                 bail!("Cannot resolve external reference without request")
             })

crates/turbopack-ecmascript/src/references/async_module.rs

@@ -7,7 +7,10 @@ use swc_core::{
     quote,
 };
 use turbo_tasks::{trace::TraceRawVcs, TryFlatJoinIterExt, TryJoinIterExt, Vc};
-use turbopack_core::chunk::{AsyncModuleInfo, ChunkableModule};
+use turbopack_core::{
+    chunk::{AsyncModuleInfo, ChunkableModule},
+    resolve::ExternalType,
+};
 
 use super::esm::base::ReferencedAsset;
 use crate::{
@@ -93,7 +96,10 @@ impl AsyncModule {
             .map(|r| async {
                 let referenced_asset = r.get_referenced_asset().await?;
                 Ok(match &*referenced_asset {
-                    ReferencedAsset::OriginalReferenceTypeExternal(_) => {
+                    ReferencedAsset::External(
+                        _,
+                        ExternalType::OriginalReference | ExternalType::EcmaScriptModule,
+                    ) => {
                         if self.import_externals {
                             referenced_asset.get_ident().await?
                         } else {
@@ -114,6 +120,7 @@ impl AsyncModule {
                             None
                         }
                     }
+                    ReferencedAsset::External(..) => None,
                     ReferencedAsset::None => None,
                 })
             })
@@ -138,7 +145,10 @@ impl AsyncModule {
                         let referenced_asset = r.get_referenced_asset().await?;
                         Ok(matches!(
                             &*referenced_asset,
-                            ReferencedAsset::OriginalReferenceTypeExternal(_)
+                            ReferencedAsset::External(
+                                _,
+                                ExternalType::OriginalReference | ExternalType::EcmaScriptModule
+                            )
                         ))
                     })
                     .try_join()

crates/turbopack-ecmascript/src/references/esm/base.rs

@@ -18,7 +18,7 @@ use turbopack_core::{
     resolve::{
         origin::{ResolveOrigin, ResolveOriginExt},
         parse::Request,
-        ModulePart, ModuleResolveResult, ModuleResolveResultItem,
+        ExternalType, ModulePart, ModuleResolveResult, ModuleResolveResultItem,
     },
 };
 
@@ -34,17 +34,17 @@ use crate::{
 #[turbo_tasks::value]
 pub enum ReferencedAsset {
     Some(Vc<Box<dyn EcmascriptChunkPlaceable>>),
-    OriginalReferenceTypeExternal(String),
+    External(String, ExternalType),
     None,
 }
 
 impl ReferencedAsset {
     pub async fn get_ident(&self) -> Result<Option<String>> {
         Ok(match self {
             ReferencedAsset::Some(asset) => Some(Self::get_ident_from_placeable(asset).await?),
-            ReferencedAsset::OriginalReferenceTypeExternal(request) => {
-                Some(magic_identifier::mangle(&format!("external {}", request)))
-            }
+            ReferencedAsset::External(request, ty) => Some(magic_identifier::mangle(&format!(
+                "{ty} external {request}"
+            ))),
             ReferencedAsset::None => None,
         })
     }
@@ -67,10 +67,8 @@ impl ReferencedAsset {
         // TODO handle multiple keyed results
         for (_key, result) in resolve_result.await?.primary.iter() {
             match result {
-                ModuleResolveResultItem::OriginalReferenceTypeExternal(request) => {
-                    return Ok(
-                        ReferencedAsset::OriginalReferenceTypeExternal(request.clone()).cell(),
-                    );
+                ModuleResolveResultItem::External(request, ty) => {
+                    return Ok(ReferencedAsset::External(request.clone(), *ty).cell());
                 }
                 &ModuleResolveResultItem::Module(module) => {
                     if let Some(placeable) =
@@ -241,10 +239,13 @@ impl CodeGenerateable for EsmAssetReference {
                             insert_hoisted_stmt(program, stmt);
                         }));
                     }
-                    ReferencedAsset::OriginalReferenceTypeExternal(request) => {
+                    ReferencedAsset::External(
+                        request,
+                        ExternalType::OriginalReference | ExternalType::EcmaScriptModule,
+                    ) => {
                         if !*chunking_context
                             .environment()
-                            .supports_commonjs_externals()
+                            .supports_esm_externals()
                             .await?
                         {
                             bail!(
@@ -255,7 +256,6 @@ impl CodeGenerateable for EsmAssetReference {
                         }
                         let request = request.clone();
                         visitors.push(create_visitor!(visit_mut_program(program: &mut Program) {
-                            // TODO Technically this should insert a ESM external, but we don't support that yet
                             let stmt = if import_externals {
                                 quote!(
                                     "var $name = __turbopack_external_import__($id);" as Stmt,
@@ -272,6 +272,39 @@ impl CodeGenerateable for EsmAssetReference {
                             insert_hoisted_stmt(program, stmt);
                         }));
                     }
+                    ReferencedAsset::External(
+                        request,
+                        ExternalType::CommonJs | ExternalType::Url,
+                    ) => {
+                        if !*chunking_context
+                            .environment()
+                            .supports_commonjs_externals()
+                            .await?
+                        {
+                            bail!(
+                                "the chunking context does not support external modules (request: \
+                                 {})",
+                                request
+                            );
+                        }
+                        let request = request.clone();
+                        visitors.push(create_visitor!(visit_mut_program(program: &mut Program) {
+                            let stmt = quote!(
+                                "var $name = __turbopack_external_require__($id, true);" as Stmt,
+                                name = Ident::new(ident.clone().into(), DUMMY_SP),
+                                id: Expr = Expr::Lit(request.clone().into())
+                            );
+                            insert_hoisted_stmt(program, stmt);
+                        }));
+                    }
+                    #[allow(unreachable_patterns)]
+                    ReferencedAsset::External(request, ty) => {
+                        bail!(
+                            "Unsupported external type {:?} for ESM reference with request: {:?}",
+                            ty,
+                            request
+                        )
+                    }
                     ReferencedAsset::None => {}
                 }
             }

crates/turbopack-ecmascript/src/references/esm/url.rs

@@ -1,4 +1,4 @@
-use anyhow::Result;
+use anyhow::{bail, Result};
 use swc_core::{
     ecma::ast::{Expr, ExprOrSpread, NewExpr},
     quote,
@@ -12,7 +12,9 @@ use turbopack_core::{
     issue::IssueSource,
     reference::ModuleReference,
     reference_type::{ReferenceType, UrlReferenceSubType},
-    resolve::{origin::ResolveOrigin, parse::Request, url_resolve, ModuleResolveResult},
+    resolve::{
+        origin::ResolveOrigin, parse::Request, url_resolve, ExternalType, ModuleResolveResult,
+    },
 };
 
 use super::base::ReferencedAsset;
@@ -181,7 +183,10 @@ impl CodeGenerateable for UrlAssetReference {
                             }
                         }));
                     }
-                    ReferencedAsset::OriginalReferenceTypeExternal(request) => {
+                    ReferencedAsset::External(
+                        request,
+                        ExternalType::OriginalReference | ExternalType::Url,
+                    ) => {
                         let request = request.to_string();
                         visitors.push(create_visitor!(ast_path, visit_mut_expr(new_expr: &mut Expr) {
                             let should_rewrite_to_relative = if let Expr::New(NewExpr { args: Some(args), .. }) = new_expr {
@@ -198,6 +203,13 @@ impl CodeGenerateable for UrlAssetReference {
                             }
                         }));
                     }
+                    ReferencedAsset::External(request, ty) => {
+                        bail!(
+                            "Unsupported external type {:?} for URL reference with request: {:?}",
+                            ty,
+                            request
+                        )
+                    }
                     ReferencedAsset::None => {}
                 }
             }
@@ -263,7 +275,10 @@ impl CodeGenerateable for UrlAssetReference {
                             }
                         }));
                     }
-                    ReferencedAsset::OriginalReferenceTypeExternal(request) => {
+                    ReferencedAsset::External(
+                        request,
+                        ExternalType::OriginalReference | ExternalType::Url,
+                    ) => {
                         let request = request.to_string();
                         visitors.push(create_visitor!(ast_path, visit_mut_expr(new_expr: &mut Expr) {
                             if let Expr::New(NewExpr { args: Some(args), .. }) = new_expr {
@@ -279,6 +294,13 @@ impl CodeGenerateable for UrlAssetReference {
                             }
                         }));
                     }
+                    ReferencedAsset::External(request, ty) => {
+                        bail!(
+                            "Unsupported external type {:?} for URL reference with request: {:?}",
+                            ty,
+                            request
+                        )
+                    }
                     ReferencedAsset::None => {}
                 }
             }