Support @next/mdx with Turbopack (#68397)

Closes PACK-3153
vercel · Aug 14, 2024 · f132862 · f132862
1 parent cc33dc1
commit f132862
Show file tree

Hide file tree

Showing 7 changed files with 127 additions and 100 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/next-core/src/next_import_map.rs b/crates/next-core/src/next_import_map.rs
@@ -806,17 +806,15 @@ async fn insert_next_shared_aliases(
 ) -> Result<()> {
     let package_root = next_js_fs().root();
 
-    if next_config.mdx_rs().await?.is_some() {
-        insert_alias_to_alternatives(
-            import_map,
-            mdx_import_source_file(),
-            vec![
-                request_to_import_mapping(project_path, "./mdx-components"),
-                request_to_import_mapping(project_path, "./src/mdx-components"),
-                request_to_import_mapping(project_path, "@mdx-js/react"),
-            ],
-        );
-    }
+    insert_alias_to_alternatives(
+        import_map,
+        mdx_import_source_file(),
+        vec![
+            request_to_import_mapping(project_path, "./mdx-components"),
+            request_to_import_mapping(project_path, "./src/mdx-components"),
+            request_to_import_mapping(project_path, "@mdx-js/react"),
+        ],
+    );
 
     insert_package_alias(
         import_map,

diff --git a/packages/next-mdx/index.js b/packages/next-mdx/index.js
@@ -23,6 +23,24 @@ module.exports =
         }
 
     return Object.assign({}, nextConfig, {
+      experimental: Object.assign({}, nextConfig?.experimental, {
+        turbo: Object.assign({}, nextConfig?.experimental?.turbo, {
+          rules: Object.assign({}, nextConfig?.experimental?.turbo?.rules, {
+            '*.mdx': {
+              loaders: [loader],
+              as: '*.tsx',
+            },
+          }),
+          resolveAlias: Object.assign(
+            {},
+            nextConfig?.experimental?.turbo?.resolveAlias,
+            {
+              'next-mdx-import-source-file':
+                '@vercel/turbopack-next/mdx-import-source',
+            }
+          ),
+        }),
+      }),
       webpack(config, options) {
         config.resolve.alias['next-mdx-import-source-file'] = [
           'private-next-root-dir/src/mdx-components',

diff --git a/test/e2e/app-dir/mdx/mdx.test.ts b/test/e2e/app-dir/mdx/mdx.test.ts
@@ -1,13 +1,8 @@
-import { createNextDescribe } from 'e2e-utils'
+import { nextTestSetup } from 'e2e-utils'
 
-for (const type of [
-  'with-mdx-rs',
-  // only mdx-rs should work with turbopack
-  ...(process.env.TURBOPACK ? [] : ['without-mdx-rs']),
-]) {
-  createNextDescribe(
-    `mdx ${type}`,
-    {
+for (const type of ['with-mdx-rs', 'without-mdx-rs']) {
+  describe(`mdx ${type}`, () => {
+    const { next } = nextTestSetup({
       files: __dirname,
       dependencies: {
         '@next/mdx': 'canary',
@@ -17,94 +12,89 @@ for (const type of [
       env: {
         WITH_MDX_RS: type === 'with-mdx-rs' ? 'true' : 'false',
       },
-    },
-    ({ next }) => {
-      describe('app directory', () => {
-        it('should work in initial html', async () => {
-          const $ = await next.render$('/')
-          expect($('h1').text()).toBe('Hello World')
-          expect($('p').text()).toBe('This is MDX!')
-        })
+    })
 
-        it('should work using browser', async () => {
-          const browser = await next.browser('/')
-          expect(await browser.elementByCss('h1').text()).toBe('Hello World')
-          expect(await browser.elementByCss('p').text()).toBe('This is MDX!')
-        })
+    describe('app directory', () => {
+      it('should work in initial html', async () => {
+        const $ = await next.render$('/')
+        expect($('h1').text()).toBe('Hello World')
+        expect($('p').text()).toBe('This is MDX!')
+      })
 
-        it('should work in initial html with mdx import', async () => {
-          const $ = await next.render$('/import')
-          expect($('h1').text()).toBe('This is a title')
-          expect($('p').text()).toBe('This is a paragraph')
-        })
+      it('should work using browser', async () => {
+        const browser = await next.browser('/')
+        expect(await browser.elementByCss('h1').text()).toBe('Hello World')
+        expect(await browser.elementByCss('p').text()).toBe('This is MDX!')
+      })
 
-        it('should work using browser with mdx import', async () => {
-          const browser = await next.browser('/import')
-          expect(await browser.elementByCss('h1').text()).toBe(
-            'This is a title'
-          )
-          expect(await browser.elementByCss('p').text()).toBe(
-            'This is a paragraph'
-          )
-        })
+      it('should work in initial html with mdx import', async () => {
+        const $ = await next.render$('/import')
+        expect($('h1').text()).toBe('This is a title')
+        expect($('p').text()).toBe('This is a paragraph')
+      })
 
-        it('should allow overriding components', async () => {
-          const browser = await next.browser('/')
-          expect(await browser.elementByCss('h1').getComputedCss('color')).toBe(
-            'rgb(255, 0, 0)'
-          )
-        })
+      it('should work using browser with mdx import', async () => {
+        const browser = await next.browser('/import')
+        expect(await browser.elementByCss('h1').text()).toBe('This is a title')
+        expect(await browser.elementByCss('p').text()).toBe(
+          'This is a paragraph'
+        )
+      })
 
-        it('should allow importing client components', async () => {
-          const $ = await next.render$('/')
-          expect($('h2').text()).toBe('This is a client component')
-        })
+      it('should allow overriding components', async () => {
+        const browser = await next.browser('/')
+        expect(await browser.elementByCss('h1').getComputedCss('color')).toBe(
+          'rgb(255, 0, 0)'
+        )
+      })
 
-        it('should work with next/image', async () => {
-          const $ = await next.render$('/image')
-          expect($('img').attr('src')).toBe(
-            '/_next/image?url=%2Ftest.jpg&w=384&q=75'
-          )
-        })
+      it('should allow importing client components', async () => {
+        const $ = await next.render$('/')
+        expect($('h2').text()).toBe('This is a client component')
       })
 
-      describe('pages directory', () => {
-        it('should work in initial html', async () => {
-          const $ = await next.render$('/pages')
-          expect($('h1').text()).toBe('Hello World')
-          expect($('p').text()).toBe('This is MDX!')
-        })
+      it('should work with next/image', async () => {
+        const $ = await next.render$('/image')
+        expect($('img').attr('src')).toBe(
+          '/_next/image?url=%2Ftest.jpg&w=384&q=75'
+        )
+      })
+    })
 
-        // Recommended for tests that need a full browser
-        it('should work using browser', async () => {
-          const browser = await next.browser('/pages')
-          expect(await browser.elementByCss('h1').text()).toBe('Hello World')
-          expect(await browser.elementByCss('p').text()).toBe('This is MDX!')
-        })
+    describe('pages directory', () => {
+      it('should work in initial html', async () => {
+        const $ = await next.render$('/pages')
+        expect($('h1').text()).toBe('Hello World')
+        expect($('p').text()).toBe('This is MDX!')
+      })
 
-        it('should work in initial html with mdx import', async () => {
-          const $ = await next.render$('/pages/import')
-          expect($('h1').text()).toBe('This is a title')
-          expect($('p').text()).toBe('This is a paragraph')
-        })
+      // Recommended for tests that need a full browser
+      it('should work using browser', async () => {
+        const browser = await next.browser('/pages')
+        expect(await browser.elementByCss('h1').text()).toBe('Hello World')
+        expect(await browser.elementByCss('p').text()).toBe('This is MDX!')
+      })
 
-        it('should work using browser with mdx import', async () => {
-          const browser = await next.browser('/pages/import')
-          expect(await browser.elementByCss('h1').text()).toBe(
-            'This is a title'
-          )
-          expect(await browser.elementByCss('p').text()).toBe(
-            'This is a paragraph'
-          )
-        })
+      it('should work in initial html with mdx import', async () => {
+        const $ = await next.render$('/pages/import')
+        expect($('h1').text()).toBe('This is a title')
+        expect($('p').text()).toBe('This is a paragraph')
+      })
+
+      it('should work using browser with mdx import', async () => {
+        const browser = await next.browser('/pages/import')
+        expect(await browser.elementByCss('h1').text()).toBe('This is a title')
+        expect(await browser.elementByCss('p').text()).toBe(
+          'This is a paragraph'
+        )
+      })
 
-        it('should allow overriding components', async () => {
-          const browser = await next.browser('/pages')
-          expect(await browser.elementByCss('h1').getComputedCss('color')).toBe(
-            'rgb(255, 0, 0)'
-          )
-        })
+      it('should allow overriding components', async () => {
+        const browser = await next.browser('/pages')
+        expect(await browser.elementByCss('h1').getComputedCss('color')).toBe(
+          'rgb(255, 0, 0)'
+        )
       })
-    }
-  )
+    })
+  })
 }
diff --git a/turbopack/crates/turbopack-node/Cargo.toml b/turbopack/crates/turbopack-node/Cargo.toml
@@ -21,6 +21,7 @@ anyhow = { workspace = true }
 async-stream = "0.3.4"
 async-trait = { workspace = true }
 const_format = "0.2.30"
+either = { workspace = true, features = ["serde"] }
 futures = { workspace = true }
 futures-retry = { workspace = true }
 indexmap = { workspace = true, features = ["serde"] }
@@ -33,6 +34,7 @@ regex = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 #serde_qs = { workspace = true }
+serde_with = { workspace = true, features = ["base64"] }
 tokio = { workspace = true, features = ["full"] }
 tracing = { workspace = true }
 turbo-tasks = { workspace = true }

diff --git a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts
@@ -468,7 +468,7 @@ const transform = (
         if (!result.result) return reject(new Error("No result from loaders"));
         const [source, map] = result.result;
         resolve({
-          source,
+          source: Buffer.isBuffer(source) ? {binary: source.toString('base64')} : source,
           map:
             typeof map === "string"
               ? map

diff --git a/turbopack/crates/turbopack-node/src/transforms/webpack.rs b/turbopack/crates/turbopack-node/src/transforms/webpack.rs
@@ -2,8 +2,10 @@ use std::mem::take;
 
 use anyhow::{bail, Context, Result};
 use async_trait::async_trait;
+use either::Either;
 use serde::{Deserialize, Serialize};
 use serde_json::{json, Value as JsonValue};
+use serde_with::serde_as;
 use turbo_tasks::{
     trace::TraceRawVcs, Completion, RcStr, TaskInput, TryJoinIterExt, Value, ValueToString, Vc,
 };
@@ -52,11 +54,20 @@ use crate::{
     AssetsForSourceMapping,
 };
 
+#[serde_as]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+struct BytesBase64 {
+    #[serde_as(as = "serde_with::base64::Base64")]
+    binary: Vec<u8>,
+}
+
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename_all = "camelCase")]
 #[turbo_tasks::value(serialization = "custom")]
 struct WebpackLoadersProcessingResult {
-    source: RcStr,
+    #[serde(with = "either::serde_untagged")]
+    #[turbo_tasks(debug_ignore, trace_ignore)]
+    source: Either<RcStr, BytesBase64>,
     map: Option<RcStr>,
     #[turbo_tasks(trace_ignore)]
     assets: Option<Vec<EmittedAsset>>,
@@ -250,7 +261,10 @@ impl WebpackLoadersProcessedAsset {
         } else {
             None
         };
-        let file = File::from(processed.source);
+        let file = match processed.source {
+            Either::Left(str) => File::from(str),
+            Either::Right(bytes) => File::from(bytes.binary),
+        };
         let assets = emitted_assets_to_virtual_sources(processed.assets);
         let content = AssetContent::File(FileContent::Content(file).cell()).cell();
         Ok(ProcessWebpackLoadersResult {