feat(rsc-streaming): Integrating RSC builds with Streaming and Client…

… side hydration (#10031)

Co-authored-by: Tobbe Lundberg <tobbe@tlundberg.com>

packages/project-config/src/paths.ts

@@ -130,8 +130,9 @@ const PATH_WEB_DIR_DIST_CLIENT = 'web/dist/client'
 const PATH_WEB_DIR_DIST_RSC = 'web/dist/rsc'
 const PATH_WEB_DIR_DIST_SERVER = 'web/dist/server'
 
-const PATH_WEB_DIR_DIST_SERVER_ENTRY_SERVER = 'web/dist/server/entry.server.js'
-const PATH_WEB_DIR_DIST_DOCUMENT = 'web/dist/server/Document.js'
+// Don't specify extension, handled by resolve file
+const PATH_WEB_DIR_DIST_SERVER_ENTRY_SERVER = 'web/dist/server/entry.server'
+const PATH_WEB_DIR_DIST_DOCUMENT = 'web/dist/server/Document'
 
 const PATH_WEB_DIR_DIST_SERVER_ROUTEHOOKS = 'web/dist/server/routeHooks'
 const PATH_WEB_DIR_DIST_RSC_ENTRIES = 'web/dist/rsc/entries.js'
@@ -246,11 +247,13 @@ export const getPaths = (BASE_DIR: string = getBaseDir()): Paths => {
       distClient: path.join(BASE_DIR, PATH_WEB_DIR_DIST_CLIENT),
       distRsc: path.join(BASE_DIR, PATH_WEB_DIR_DIST_RSC),
       distServer: path.join(BASE_DIR, PATH_WEB_DIR_DIST_SERVER),
-      distEntryServer: path.join(
-        BASE_DIR,
-        PATH_WEB_DIR_DIST_SERVER_ENTRY_SERVER
+      // Allow for the possibility of a .mjs file
+      distEntryServer: mjsOrJs(
+        path.join(BASE_DIR, PATH_WEB_DIR_DIST_SERVER_ENTRY_SERVER)
+      ),
+      distDocumentServer: mjsOrJs(
+        path.join(BASE_DIR, PATH_WEB_DIR_DIST_DOCUMENT)
       ),
-      distDocumentServer: path.join(BASE_DIR, PATH_WEB_DIR_DIST_DOCUMENT),
       distRouteHooks: path.join(BASE_DIR, PATH_WEB_DIR_DIST_SERVER_ROUTEHOOKS),
       distRscEntries: path.join(BASE_DIR, PATH_WEB_DIR_DIST_RSC_ENTRIES),
       routeManifest: path.join(BASE_DIR, PATH_WEB_DIR_ROUTE_MANIFEST),
@@ -430,3 +433,9 @@ export function projectIsEsm() {
 
   return true
 }
+
+/** Default to JS path, but if MJS exists, use it instead */
+const mjsOrJs = (filePath: string) => {
+  const mjsPath = resolveFile(filePath, ['.mjs'])
+  return mjsPath ? mjsPath : filePath + '.js'
+}

packages/vite/src/buildFeServer.ts

@@ -43,9 +43,6 @@ export const buildFeServer = async ({ verbose, webDir }: BuildOptions = {}) => {
     }
 
     await buildRscClientAndServer()
-
-    // Write a route manifest
-    return await buildRouteManifest()
   }
 
   // We generate the RSC client bundle in the rscBuildClient function

packages/vite/src/buildRouteManifest.ts

@@ -5,7 +5,7 @@ import url from 'node:url'
 import type { Manifest as ViteBuildManifest } from 'vite'
 
 import { getProjectRoutes } from '@redwoodjs/internal/dist/routes'
-import { getAppRouteHook, getConfig, getPaths } from '@redwoodjs/project-config'
+import { getAppRouteHook, getPaths } from '@redwoodjs/project-config'
 
 import type { RWRouteManifest } from './types'
 
@@ -14,15 +14,10 @@ import type { RWRouteManifest } from './types'
  * Generate a route manifest file for the web server side.
  */
 export async function buildRouteManifest() {
-  const rscEnabled = getConfig()?.experimental?.rsc?.enabled
-
   const rwPaths = getPaths()
 
   const buildManifestUrl = url.pathToFileURL(
-    path.join(
-      rscEnabled ? rwPaths.web.distClient : rwPaths.web.dist,
-      'client-build-manifest.json'
-    )
+    path.join(getPaths().web.distClient, 'client-build-manifest.json')
   ).href
   const clientBuildManifest: ViteBuildManifest = (
     await import(buildManifestUrl, { with: { type: 'json' } })
@@ -34,7 +29,10 @@ export async function buildRouteManifest() {
     acc[route.pathDefinition] = {
       name: route.name,
       bundle: route.relativeFilePath
-        ? clientBuildManifest[route.relativeFilePath]?.file ?? null
+        ? // @TODO(RSC_DC): this no longer resolves to anything i.e. its always null
+          // Because the clientBuildManifest has no pages, because all pages are Server-components?
+          // This may be a non-issue, because RSC pages don't need a client bundle per page (or atleast not the same bundle)
+          clientBuildManifest[route.relativeFilePath]?.file ?? null
         : null,
       matchRegexString: route.matchRegexString,
       // NOTE this is the path definition, not the actual path

packages/vite/src/client.ts

@@ -22,6 +22,12 @@ const BASE_PATH = '/rw-rsc/'
 export function renderFromRscServer<Props>(rscId: string) {
   console.log('serve rscId', rscId)
 
+  // Temporarily skip rendering this component during SSR
+  // I don't know what we actually should do during SSR yet
+  if (typeof window === 'undefined') {
+    return null
+  }
+
   type SetRerender = (
     rerender: (next: [ReactElement, string]) => void
   ) => () => void

packages/vite/src/index.ts

@@ -8,7 +8,7 @@ import { normalizePath } from 'vite'
 import { getWebSideDefaultBabelConfig } from '@redwoodjs/babel-config'
 import { getConfig, getPaths } from '@redwoodjs/project-config'
 
-import { getDefaultViteConfig } from './lib/getDefaultViteConfig'
+import { getMergedConfig } from './lib/getMergedConfig'
 import handleJsAsJsx from './plugins/vite-plugin-jsx-loader'
 import removeFromBundle from './plugins/vite-plugin-remove-from-bundle'
 import swapApolloProvider from './plugins/vite-plugin-swap-apollo-provider'
@@ -125,7 +125,9 @@ export default function redwoodPluginVite(): PluginOption[] {
       },
       // ---------- End Bundle injection ----------
 
-      config: getDefaultViteConfig(rwConfig, rwPaths),
+      // @MARK: Using the config hook here let's us modify the config
+      // but returning plugins will **not** work
+      config: getMergedConfig(rwConfig, rwPaths),
     },
     // We can remove when streaming is stable
     rwConfig.experimental.streamingSsr.enabled && swapApolloProvider(),
@@ -146,6 +148,7 @@ export default function redwoodPluginVite(): PluginOption[] {
       babel: {
         ...getWebSideDefaultBabelConfig({
           forVite: true,
+          forRscClient: rwConfig.experimental.rsc?.enabled,
         }),
       },
     }),

packages/vite/src/lib/getDefaultViteConfig.ts → packages/vite/src/lib/getMergedConfig.ts

@@ -2,18 +2,28 @@ import path from 'node:path'
 
 import type { InputOption } from 'rollup'
 import type { ConfigEnv, UserConfig } from 'vite'
+import { mergeConfig } from 'vite'
 
 import type { Config, Paths } from '@redwoodjs/project-config'
 import { getConfig, getPaths } from '@redwoodjs/project-config'
 
 import { getEnvVarDefinitions } from './envVarDefinitions'
 
-export function getDefaultViteConfig(rwConfig: Config, rwPaths: Paths) {
-  return (options: UserConfig, env: ConfigEnv): UserConfig => {
+/**
+ * This function will merge in the default Redwood Vite config passed into the
+ * build function (or in Vite.config.xxx)
+ *
+ * Note that returning plugins in this function will have no effect on the
+ * build
+ */
+export function getMergedConfig(rwConfig: Config, rwPaths: Paths) {
+  return (userConfig: UserConfig, env: ConfigEnv): UserConfig => {
     let apiHost = process.env.REDWOOD_API_HOST
     apiHost ??= rwConfig.api.host
     apiHost ??= process.env.NODE_ENV === 'production' ? '0.0.0.0' : '[::]'
 
+    const streamingSsrEnabled = rwConfig.experimental.streamingSsr?.enabled
+    // @MARK: note that most RSC settings sit in their individual build functions
     const rscEnabled = rwConfig.experimental.rsc?.enabled
 
     let apiPort
@@ -101,7 +111,12 @@ export function getDefaultViteConfig(rwConfig: Config, rwPaths: Paths) {
         },
       },
       build: {
-        outDir: options.build?.outDir || rwPaths.web.dist,
+        // NOTE this gets overridden when build gets called anyway!
+        outDir:
+          // @MARK: For RSC and Streaming, we build to dist/client directory
+          streamingSsrEnabled || rscEnabled
+            ? rwPaths.web.distClient
+            : rwPaths.web.dist,
         emptyOutDir: true,
         manifest: !env.ssrBuild ? 'client-build-manifest.json' : undefined,
         // Note that sourcemap can be boolean or 'inline'
@@ -110,9 +125,8 @@ export function getDefaultViteConfig(rwConfig: Config, rwPaths: Paths) {
           input: getRollupInput(!!env.ssrBuild),
         },
       },
-      legacy: {
-        buildSsrCjsExternalHeuristics: rscEnabled ? false : env.ssrBuild,
-      },
+      // @MARK: do not set buildSsrCjsExternalHeuristics here
+      // because rsc builds want false, client and server build wants true
       optimizeDeps: {
         esbuildOptions: {
           // @MARK this is because JS projects in Redwood don't have .jsx
@@ -130,7 +144,7 @@ export function getDefaultViteConfig(rwConfig: Config, rwPaths: Paths) {
       },
     }
 
-    return defaultRwViteConfig
+    return mergeConfig(defaultRwViteConfig, userConfig)
   }
 }
 

packages/vite/src/rsc/rscBuildClient.ts

@@ -1,17 +1,10 @@
-import path from 'node:path'
-
-import react from '@vitejs/plugin-react'
 import { build as viteBuild } from 'vite'
 
-import { getWebSideDefaultBabelConfig } from '@redwoodjs/babel-config'
 import { getPaths } from '@redwoodjs/project-config'
 
-import { getEnvVarDefinitions } from '../lib/envVarDefinitions'
 import { onWarn } from '../lib/onWarn'
 import { ensureProcessDirWeb } from '../utils'
 
-import { rscIndexPlugin } from './rscVitePlugins'
-
 /**
  * RSC build. Step 2.
  * buildFeServer -> buildRscFeServer -> rscBuildClient
@@ -31,31 +24,21 @@ export async function rscBuildClient(clientEntryFiles: Record<string, string>) {
   // unintended consequences on CSS processing
   ensureProcessDirWeb()
 
+  if (!rwPaths.web.entryClient) {
+    throw new Error('Missing web/src/entry.client')
+  }
+
   const clientBuildOutput = await viteBuild({
-    // configFile: viteConfigPath,
-    root: rwPaths.web.src,
-    envPrefix: 'REDWOOD_ENV_',
-    publicDir: path.join(rwPaths.web.base, 'public'),
     envFile: false,
-    define: getEnvVarDefinitions(),
-    plugins: [
-      react({
-        babel: {
-          ...getWebSideDefaultBabelConfig({
-            forVite: true,
-            forRscClient: true,
-          }),
-        },
-      }),
-      rscIndexPlugin(),
-    ],
     build: {
       outDir: rwPaths.web.distClient,
       emptyOutDir: true, // Needed because `outDir` is not inside `root`
       rollupOptions: {
         onwarn: onWarn,
         input: {
-          main: rwPaths.web.html,
+          // @MARK: temporary hack to find the entry client so we can get the
+          // index.css bundle but we don't actually want this on an rsc page!
+          'rwjs-client-entry': rwPaths.web.entryClient,
           // we need this, so that the output contains rsc-specific bundles
           // for the client-only components. They get loaded once the page is
           // rendered

packages/vite/src/rsc/rscBuildClientEntriesFile.ts

@@ -11,6 +11,9 @@ import type { rscBuildForServer } from './rscBuildForServer'
  * `web/dist/rsc/entries.js` file.
  * Only used by the RSC worker.
  */
+// TODO(RSC_DC): This function should eventually be removed.
+// The dev server will need this implemented as a Vite plugin,
+// so worth waiting till implementation to swap out and just include the plugin for the prod build
 export function rscBuildClientEntriesMappings(
   clientBuildOutput: Awaited<ReturnType<typeof rscBuildClient>>,
   serverBuildOutput: Awaited<ReturnType<typeof rscBuildForServer>>,
@@ -25,6 +28,7 @@ export function rscBuildClientEntriesMappings(
   const clientEntries: Record<string, string> = {}
   for (const item of clientBuildOutput) {
     const { name, fileName } = item
+
     const entryFile =
       name &&
       // TODO (RSC) Can't we just compare the names? `item.name === name`

packages/vite/src/rsc/rscBuildForServer.ts

@@ -1,12 +1,9 @@
 import path from 'node:path'
 
-import react from '@vitejs/plugin-react'
 import { build as viteBuild } from 'vite'
 
-import { getWebSideDefaultBabelConfig } from '@redwoodjs/babel-config'
 import { getPaths } from '@redwoodjs/project-config'
 
-import { getEnvVarDefinitions } from '../lib/envVarDefinitions'
 import { onWarn } from '../lib/onWarn'
 
 import { rscTransformPlugin } from './rscVitePlugins'
@@ -40,12 +37,11 @@ export async function rscBuildForServer(
 
   // TODO (RSC): No redwood-vite plugin, add it in here
   const rscServerBuildOutput = await viteBuild({
-    // ...configFileConfig,
-    root: rwPaths.web.src,
-    envPrefix: 'REDWOOD_ENV_',
-    publicDir: path.join(rwPaths.web.base, 'public'),
     envFile: false,
-    define: getEnvVarDefinitions(),
+    legacy: {
+      // @MARK: for the worker, we're building ESM! (not CJS)
+      buildSsrCjsExternalHeuristics: false,
+    },
     ssr: {
       // Externalize everything except packages with files that have
       // 'use client' in them (which are the files in `clientEntryFiles`)
@@ -86,13 +82,6 @@ export async function rscBuildForServer(
       },
     },
     plugins: [
-      react({
-        babel: {
-          ...getWebSideDefaultBabelConfig({
-            forVite: true,
-          }),
-        },
-      }),
       // The rscTransformPlugin maps paths like
       // /Users/tobbe/.../rw-app/node_modules/@tobbe.dev/rsc-test/dist/rsc-test.es.js
       // to

packages/vite/src/rsc/rscVitePlugins.ts

@@ -6,24 +6,6 @@ import type { Plugin } from 'vite'
 import * as RSDWNodeLoader from '../react-server-dom-webpack/node-loader'
 import type { ResolveFunction } from '../react-server-dom-webpack/node-loader'
 
-import { rscWebpackShims } from './rscWebpackShims'
-
-// Used in Step 2 of the build process, for the client bundle
-export function rscIndexPlugin(): Plugin {
-  return {
-    name: 'rsc-index-plugin',
-    async transformIndexHtml() {
-      return [
-        {
-          tag: 'script',
-          children: rscWebpackShims,
-          injectTo: 'body',
-        },
-      ]
-    },
-  }
-}
-
 export function rscTransformPlugin(
   clientEntryFiles: Record<string, string>
 ): Plugin {