From 0b1b4c6218c4d391b9cdd2288bbb058ca9d836ae Mon Sep 17 00:00:00 2001 From: Daniel Choudhury Date: Mon, 4 Mar 2024 23:25:59 +0700 Subject: [PATCH] feat(rsc-streaming): Integrating RSC builds with Streaming and Client side hydration (#10031) Co-authored-by: Tobbe Lundberg --- packages/project-config/src/paths.ts | 21 ++++-- packages/vite/src/buildFeServer.ts | 3 - packages/vite/src/buildRouteManifest.ts | 14 ++-- packages/vite/src/client.ts | 6 ++ packages/vite/src/index.ts | 7 +- ...efaultViteConfig.ts => getMergedConfig.ts} | 28 ++++++-- packages/vite/src/rsc/rscBuildClient.ts | 31 ++------ .../vite/src/rsc/rscBuildClientEntriesFile.ts | 4 ++ packages/vite/src/rsc/rscBuildForServer.ts | 19 ++--- packages/vite/src/rsc/rscVitePlugins.ts | 18 ----- packages/vite/src/runFeServer.ts | 70 +++++-------------- .../streaming/createReactStreamingHandler.ts | 27 +++---- packages/vite/src/streaming/streamHelpers.ts | 3 +- 13 files changed, 97 insertions(+), 154 deletions(-) rename packages/vite/src/lib/{getDefaultViteConfig.ts => getMergedConfig.ts} (83%) diff --git a/packages/project-config/src/paths.ts b/packages/project-config/src/paths.ts index 854d8d538895..407483e729c7 100644 --- a/packages/project-config/src/paths.ts +++ b/packages/project-config/src/paths.ts @@ -130,8 +130,9 @@ const PATH_WEB_DIR_DIST_CLIENT = 'web/dist/client' const PATH_WEB_DIR_DIST_RSC = 'web/dist/rsc' const PATH_WEB_DIR_DIST_SERVER = 'web/dist/server' -const PATH_WEB_DIR_DIST_SERVER_ENTRY_SERVER = 'web/dist/server/entry.server.js' -const PATH_WEB_DIR_DIST_DOCUMENT = 'web/dist/server/Document.js' +// Don't specify extension, handled by resolve file +const PATH_WEB_DIR_DIST_SERVER_ENTRY_SERVER = 'web/dist/server/entry.server' +const PATH_WEB_DIR_DIST_DOCUMENT = 'web/dist/server/Document' const PATH_WEB_DIR_DIST_SERVER_ROUTEHOOKS = 'web/dist/server/routeHooks' const PATH_WEB_DIR_DIST_RSC_ENTRIES = 'web/dist/rsc/entries.js' @@ -246,11 +247,13 @@ export const getPaths = (BASE_DIR: string = getBaseDir()): Paths => { distClient: path.join(BASE_DIR, PATH_WEB_DIR_DIST_CLIENT), distRsc: path.join(BASE_DIR, PATH_WEB_DIR_DIST_RSC), distServer: path.join(BASE_DIR, PATH_WEB_DIR_DIST_SERVER), - distEntryServer: path.join( - BASE_DIR, - PATH_WEB_DIR_DIST_SERVER_ENTRY_SERVER + // Allow for the possibility of a .mjs file + distEntryServer: mjsOrJs( + path.join(BASE_DIR, PATH_WEB_DIR_DIST_SERVER_ENTRY_SERVER) + ), + distDocumentServer: mjsOrJs( + path.join(BASE_DIR, PATH_WEB_DIR_DIST_DOCUMENT) ), - distDocumentServer: path.join(BASE_DIR, PATH_WEB_DIR_DIST_DOCUMENT), distRouteHooks: path.join(BASE_DIR, PATH_WEB_DIR_DIST_SERVER_ROUTEHOOKS), distRscEntries: path.join(BASE_DIR, PATH_WEB_DIR_DIST_RSC_ENTRIES), routeManifest: path.join(BASE_DIR, PATH_WEB_DIR_ROUTE_MANIFEST), @@ -430,3 +433,9 @@ export function projectIsEsm() { return true } + +/** Default to JS path, but if MJS exists, use it instead */ +const mjsOrJs = (filePath: string) => { + const mjsPath = resolveFile(filePath, ['.mjs']) + return mjsPath ? mjsPath : filePath + '.js' +} diff --git a/packages/vite/src/buildFeServer.ts b/packages/vite/src/buildFeServer.ts index 8fcbea89aa4f..fc64bfb0097a 100644 --- a/packages/vite/src/buildFeServer.ts +++ b/packages/vite/src/buildFeServer.ts @@ -43,9 +43,6 @@ export const buildFeServer = async ({ verbose, webDir }: BuildOptions = {}) => { } await buildRscClientAndServer() - - // Write a route manifest - return await buildRouteManifest() } // We generate the RSC client bundle in the rscBuildClient function diff --git a/packages/vite/src/buildRouteManifest.ts b/packages/vite/src/buildRouteManifest.ts index 00ec9cfd907e..7047f5de5029 100644 --- a/packages/vite/src/buildRouteManifest.ts +++ b/packages/vite/src/buildRouteManifest.ts @@ -5,7 +5,7 @@ import url from 'node:url' import type { Manifest as ViteBuildManifest } from 'vite' import { getProjectRoutes } from '@redwoodjs/internal/dist/routes' -import { getAppRouteHook, getConfig, getPaths } from '@redwoodjs/project-config' +import { getAppRouteHook, getPaths } from '@redwoodjs/project-config' import type { RWRouteManifest } from './types' @@ -14,15 +14,10 @@ import type { RWRouteManifest } from './types' * Generate a route manifest file for the web server side. */ export async function buildRouteManifest() { - const rscEnabled = getConfig()?.experimental?.rsc?.enabled - const rwPaths = getPaths() const buildManifestUrl = url.pathToFileURL( - path.join( - rscEnabled ? rwPaths.web.distClient : rwPaths.web.dist, - 'client-build-manifest.json' - ) + path.join(getPaths().web.distClient, 'client-build-manifest.json') ).href const clientBuildManifest: ViteBuildManifest = ( await import(buildManifestUrl, { with: { type: 'json' } }) @@ -34,7 +29,10 @@ export async function buildRouteManifest() { acc[route.pathDefinition] = { name: route.name, bundle: route.relativeFilePath - ? clientBuildManifest[route.relativeFilePath]?.file ?? null + ? // @TODO(RSC_DC): this no longer resolves to anything i.e. its always null + // Because the clientBuildManifest has no pages, because all pages are Server-components? + // This may be a non-issue, because RSC pages don't need a client bundle per page (or atleast not the same bundle) + clientBuildManifest[route.relativeFilePath]?.file ?? null : null, matchRegexString: route.matchRegexString, // NOTE this is the path definition, not the actual path diff --git a/packages/vite/src/client.ts b/packages/vite/src/client.ts index c91b26ac6e9a..8d98276040af 100644 --- a/packages/vite/src/client.ts +++ b/packages/vite/src/client.ts @@ -22,6 +22,12 @@ const BASE_PATH = '/rw-rsc/' export function renderFromRscServer(rscId: string) { console.log('serve rscId', rscId) + // Temporarily skip rendering this component during SSR + // I don't know what we actually should do during SSR yet + if (typeof window === 'undefined') { + return null + } + type SetRerender = ( rerender: (next: [ReactElement, string]) => void ) => () => void diff --git a/packages/vite/src/index.ts b/packages/vite/src/index.ts index 2cda654167ed..ee60be8ee102 100644 --- a/packages/vite/src/index.ts +++ b/packages/vite/src/index.ts @@ -8,7 +8,7 @@ import { normalizePath } from 'vite' import { getWebSideDefaultBabelConfig } from '@redwoodjs/babel-config' import { getConfig, getPaths } from '@redwoodjs/project-config' -import { getDefaultViteConfig } from './lib/getDefaultViteConfig' +import { getMergedConfig } from './lib/getMergedConfig' import handleJsAsJsx from './plugins/vite-plugin-jsx-loader' import removeFromBundle from './plugins/vite-plugin-remove-from-bundle' import swapApolloProvider from './plugins/vite-plugin-swap-apollo-provider' @@ -125,7 +125,9 @@ export default function redwoodPluginVite(): PluginOption[] { }, // ---------- End Bundle injection ---------- - config: getDefaultViteConfig(rwConfig, rwPaths), + // @MARK: Using the config hook here let's us modify the config + // but returning plugins will **not** work + config: getMergedConfig(rwConfig, rwPaths), }, // We can remove when streaming is stable rwConfig.experimental.streamingSsr.enabled && swapApolloProvider(), @@ -146,6 +148,7 @@ export default function redwoodPluginVite(): PluginOption[] { babel: { ...getWebSideDefaultBabelConfig({ forVite: true, + forRscClient: rwConfig.experimental.rsc?.enabled, }), }, }), diff --git a/packages/vite/src/lib/getDefaultViteConfig.ts b/packages/vite/src/lib/getMergedConfig.ts similarity index 83% rename from packages/vite/src/lib/getDefaultViteConfig.ts rename to packages/vite/src/lib/getMergedConfig.ts index 579e6db4db54..2e5d91f7aa5d 100644 --- a/packages/vite/src/lib/getDefaultViteConfig.ts +++ b/packages/vite/src/lib/getMergedConfig.ts @@ -2,18 +2,28 @@ import path from 'node:path' import type { InputOption } from 'rollup' import type { ConfigEnv, UserConfig } from 'vite' +import { mergeConfig } from 'vite' import type { Config, Paths } from '@redwoodjs/project-config' import { getConfig, getPaths } from '@redwoodjs/project-config' import { getEnvVarDefinitions } from './envVarDefinitions' -export function getDefaultViteConfig(rwConfig: Config, rwPaths: Paths) { - return (options: UserConfig, env: ConfigEnv): UserConfig => { +/** + * This function will merge in the default Redwood Vite config passed into the + * build function (or in Vite.config.xxx) + * + * Note that returning plugins in this function will have no effect on the + * build + */ +export function getMergedConfig(rwConfig: Config, rwPaths: Paths) { + return (userConfig: UserConfig, env: ConfigEnv): UserConfig => { let apiHost = process.env.REDWOOD_API_HOST apiHost ??= rwConfig.api.host apiHost ??= process.env.NODE_ENV === 'production' ? '0.0.0.0' : '[::]' + const streamingSsrEnabled = rwConfig.experimental.streamingSsr?.enabled + // @MARK: note that most RSC settings sit in their individual build functions const rscEnabled = rwConfig.experimental.rsc?.enabled let apiPort @@ -101,7 +111,12 @@ export function getDefaultViteConfig(rwConfig: Config, rwPaths: Paths) { }, }, build: { - outDir: options.build?.outDir || rwPaths.web.dist, + // NOTE this gets overridden when build gets called anyway! + outDir: + // @MARK: For RSC and Streaming, we build to dist/client directory + streamingSsrEnabled || rscEnabled + ? rwPaths.web.distClient + : rwPaths.web.dist, emptyOutDir: true, manifest: !env.ssrBuild ? 'client-build-manifest.json' : undefined, // Note that sourcemap can be boolean or 'inline' @@ -110,9 +125,8 @@ export function getDefaultViteConfig(rwConfig: Config, rwPaths: Paths) { input: getRollupInput(!!env.ssrBuild), }, }, - legacy: { - buildSsrCjsExternalHeuristics: rscEnabled ? false : env.ssrBuild, - }, + // @MARK: do not set buildSsrCjsExternalHeuristics here + // because rsc builds want false, client and server build wants true optimizeDeps: { esbuildOptions: { // @MARK this is because JS projects in Redwood don't have .jsx @@ -130,7 +144,7 @@ export function getDefaultViteConfig(rwConfig: Config, rwPaths: Paths) { }, } - return defaultRwViteConfig + return mergeConfig(defaultRwViteConfig, userConfig) } } diff --git a/packages/vite/src/rsc/rscBuildClient.ts b/packages/vite/src/rsc/rscBuildClient.ts index 62daf2b290f0..230a25c64bab 100644 --- a/packages/vite/src/rsc/rscBuildClient.ts +++ b/packages/vite/src/rsc/rscBuildClient.ts @@ -1,17 +1,10 @@ -import path from 'node:path' - -import react from '@vitejs/plugin-react' import { build as viteBuild } from 'vite' -import { getWebSideDefaultBabelConfig } from '@redwoodjs/babel-config' import { getPaths } from '@redwoodjs/project-config' -import { getEnvVarDefinitions } from '../lib/envVarDefinitions' import { onWarn } from '../lib/onWarn' import { ensureProcessDirWeb } from '../utils' -import { rscIndexPlugin } from './rscVitePlugins' - /** * RSC build. Step 2. * buildFeServer -> buildRscFeServer -> rscBuildClient @@ -31,31 +24,21 @@ export async function rscBuildClient(clientEntryFiles: Record) { // unintended consequences on CSS processing ensureProcessDirWeb() + if (!rwPaths.web.entryClient) { + throw new Error('Missing web/src/entry.client') + } + const clientBuildOutput = await viteBuild({ - // configFile: viteConfigPath, - root: rwPaths.web.src, - envPrefix: 'REDWOOD_ENV_', - publicDir: path.join(rwPaths.web.base, 'public'), envFile: false, - define: getEnvVarDefinitions(), - plugins: [ - react({ - babel: { - ...getWebSideDefaultBabelConfig({ - forVite: true, - forRscClient: true, - }), - }, - }), - rscIndexPlugin(), - ], build: { outDir: rwPaths.web.distClient, emptyOutDir: true, // Needed because `outDir` is not inside `root` rollupOptions: { onwarn: onWarn, input: { - main: rwPaths.web.html, + // @MARK: temporary hack to find the entry client so we can get the + // index.css bundle but we don't actually want this on an rsc page! + 'rwjs-client-entry': rwPaths.web.entryClient, // we need this, so that the output contains rsc-specific bundles // for the client-only components. They get loaded once the page is // rendered diff --git a/packages/vite/src/rsc/rscBuildClientEntriesFile.ts b/packages/vite/src/rsc/rscBuildClientEntriesFile.ts index 618680a896c8..ec911017c300 100644 --- a/packages/vite/src/rsc/rscBuildClientEntriesFile.ts +++ b/packages/vite/src/rsc/rscBuildClientEntriesFile.ts @@ -11,6 +11,9 @@ import type { rscBuildForServer } from './rscBuildForServer' * `web/dist/rsc/entries.js` file. * Only used by the RSC worker. */ +// TODO(RSC_DC): This function should eventually be removed. +// The dev server will need this implemented as a Vite plugin, +// so worth waiting till implementation to swap out and just include the plugin for the prod build export function rscBuildClientEntriesMappings( clientBuildOutput: Awaited>, serverBuildOutput: Awaited>, @@ -25,6 +28,7 @@ export function rscBuildClientEntriesMappings( const clientEntries: Record = {} for (const item of clientBuildOutput) { const { name, fileName } = item + const entryFile = name && // TODO (RSC) Can't we just compare the names? `item.name === name` diff --git a/packages/vite/src/rsc/rscBuildForServer.ts b/packages/vite/src/rsc/rscBuildForServer.ts index 5b3c66188fb2..2f2f13ea44a9 100644 --- a/packages/vite/src/rsc/rscBuildForServer.ts +++ b/packages/vite/src/rsc/rscBuildForServer.ts @@ -1,12 +1,9 @@ import path from 'node:path' -import react from '@vitejs/plugin-react' import { build as viteBuild } from 'vite' -import { getWebSideDefaultBabelConfig } from '@redwoodjs/babel-config' import { getPaths } from '@redwoodjs/project-config' -import { getEnvVarDefinitions } from '../lib/envVarDefinitions' import { onWarn } from '../lib/onWarn' import { rscTransformPlugin } from './rscVitePlugins' @@ -40,12 +37,11 @@ export async function rscBuildForServer( // TODO (RSC): No redwood-vite plugin, add it in here const rscServerBuildOutput = await viteBuild({ - // ...configFileConfig, - root: rwPaths.web.src, - envPrefix: 'REDWOOD_ENV_', - publicDir: path.join(rwPaths.web.base, 'public'), envFile: false, - define: getEnvVarDefinitions(), + legacy: { + // @MARK: for the worker, we're building ESM! (not CJS) + buildSsrCjsExternalHeuristics: false, + }, ssr: { // Externalize everything except packages with files that have // 'use client' in them (which are the files in `clientEntryFiles`) @@ -86,13 +82,6 @@ export async function rscBuildForServer( }, }, plugins: [ - react({ - babel: { - ...getWebSideDefaultBabelConfig({ - forVite: true, - }), - }, - }), // The rscTransformPlugin maps paths like // /Users/tobbe/.../rw-app/node_modules/@tobbe.dev/rsc-test/dist/rsc-test.es.js // to diff --git a/packages/vite/src/rsc/rscVitePlugins.ts b/packages/vite/src/rsc/rscVitePlugins.ts index c01cceac79e7..a9e4323938f2 100644 --- a/packages/vite/src/rsc/rscVitePlugins.ts +++ b/packages/vite/src/rsc/rscVitePlugins.ts @@ -6,24 +6,6 @@ import type { Plugin } from 'vite' import * as RSDWNodeLoader from '../react-server-dom-webpack/node-loader' import type { ResolveFunction } from '../react-server-dom-webpack/node-loader' -import { rscWebpackShims } from './rscWebpackShims' - -// Used in Step 2 of the build process, for the client bundle -export function rscIndexPlugin(): Plugin { - return { - name: 'rsc-index-plugin', - async transformIndexHtml() { - return [ - { - tag: 'script', - children: rscWebpackShims, - injectTo: 'body', - }, - ] - }, - } -} - export function rscTransformPlugin( clientEntryFiles: Record ): Plugin { diff --git a/packages/vite/src/runFeServer.ts b/packages/vite/src/runFeServer.ts index ba2c2d6c233c..486565eada90 100644 --- a/packages/vite/src/runFeServer.ts +++ b/packages/vite/src/runFeServer.ts @@ -67,10 +67,7 @@ export async function runFeServer() { ).default const clientBuildManifestUrl = url.pathToFileURL( - path.join( - rscEnabled ? rwPaths.web.distClient : rwPaths.web.dist, - 'client-build-manifest.json' - ) + path.join(rwPaths.web.distClient, 'client-build-manifest.json') ).href const clientBuildManifest: ViteBuildManifest = ( await import(clientBuildManifestUrl, { with: { type: 'json' } }) @@ -82,9 +79,13 @@ export async function runFeServer() { console.log('='.repeat(80)) } + // @MARK: Surely there's a better way than this! const clientEntry = Object.values(clientBuildManifest).find( (manifestItem) => { - return manifestItem.isEntry + // For RSC builds, we pass in many Vite entries, so we need to find it differently. + return rscEnabled + ? manifestItem.file.includes('rwjs-client-entry-') + : manifestItem.isEntry } ) @@ -96,10 +97,7 @@ export async function runFeServer() { // For CF workers, we'd need an equivalent of this app.use( '/assets', - express.static( - rscEnabled ? rwPaths.web.distClient : rwPaths.web.dist + '/assets', - { index: false } - ) + express.static(rwPaths.web.distClient + '/assets', { index: false }) ) // 2. Proxy the api server @@ -136,33 +134,17 @@ export async function runFeServer() { ? route.matchRegexString : route.pathDefinition - if (!rscEnabled) { - const routeHandler = await createReactStreamingHandler({ - route, - clientEntryPath, - getStylesheetLinks, - }) - - // Wrap with whatg/server adapter. Express handler -> Fetch API handler - app.get(expressPathDef, createServerAdapter(routeHandler)) - } else { - console.log('expressPathDef', expressPathDef) - - // This is for RSC only. And only for now, until we have SSR working - // with RSC. This maps /, /about, etc to index.html - app.get(expressPathDef, (req, res, next) => { - // Serve index.html for all routes, to let client side routing take - // over - req.url = '/' - // Without this, we get a flash of a url with a trailing slash. Still - // works, but doesn't look nice - // For example, if we navigate to /about we'll see a flash of /about/ - // before returning to /about - req.originalUrl = '/' - - return express.static(rwPaths.web.distClient)(req, res, next) - }) - } + // TODO(RSC_DC): RSC is rendering blank page, try using this function for initial render + const routeHandler = await createReactStreamingHandler({ + route, + clientEntryPath, + getStylesheetLinks, + }) + + console.log('Attaching streaming handler for route', route.pathDefinition) + + // Wrap with whatg/server adapter. Express handler -> Fetch API handler + app.get(expressPathDef, createServerAdapter(routeHandler)) } // Mounting middleware at /rw-rsc will strip /rw-rsc from req.url @@ -182,21 +164,7 @@ export async function runFeServer() { }) ) - // Serve static assets that aren't covered by any of the above routes or middleware - // Note: That the order here is important and that we are explicitly preventing access - // to the server dist folder - // TODO: In the future, we should explicitly serve `web/dist/client` and `web/dist/rsc` - // and simply not serve the `web/dist/server` folder - app.use(`/${path.basename(rwPaths.web.distServer)}/*`, (_req, res, _next) => { - return res - .status(403) - .end('403 Forbidden: Access to server dist is forbidden') - }) - app.use( - express.static(rscEnabled ? rwPaths.web.distClient : rwPaths.web.dist, { - index: false, - }) - ) + app.use(express.static(rwPaths.web.distClient, { index: false })) app.listen(rwConfig.web.port) console.log( diff --git a/packages/vite/src/streaming/createReactStreamingHandler.ts b/packages/vite/src/streaming/createReactStreamingHandler.ts index b72f62759c18..b3fbcf73ce1e 100644 --- a/packages/vite/src/streaming/createReactStreamingHandler.ts +++ b/packages/vite/src/streaming/createReactStreamingHandler.ts @@ -6,7 +6,7 @@ import type { ViteDevServer } from 'vite' import { defaultAuthProviderState } from '@redwoodjs/auth' import type { RWRouteManifestItem } from '@redwoodjs/internal' -import { getAppRouteHook, getConfig, getPaths } from '@redwoodjs/project-config' +import { getAppRouteHook, getPaths } from '@redwoodjs/project-config' import { matchPath } from '@redwoodjs/router' import type { TagDescriptor } from '@redwoodjs/web' @@ -41,24 +41,10 @@ export const createReactStreamingHandler = async ( let fallbackDocumentImport: any if (isProd) { - // TODO (RSC) Consolidate paths, so we can have the same code for SSR and RSC - if (getConfig().experimental?.rsc?.enabled) { - entryServerImport = await import( - makeFilePath( - path.join(rwPaths.web.distRsc, 'assets', 'entry.server.js') - ) - ) - fallbackDocumentImport = await import( - makeFilePath(path.join(rwPaths.web.distServer, 'assets', 'Document.js')) - ) - } else { - entryServerImport = await import( - makeFilePath(rwPaths.web.distEntryServer) - ) - fallbackDocumentImport = await import( - makeFilePath(rwPaths.web.distDocumentServer) - ) - } + entryServerImport = await import(makeFilePath(rwPaths.web.distEntryServer)) + fallbackDocumentImport = await import( + makeFilePath(rwPaths.web.distDocumentServer) + ) } // @NOTE: we are returning a FetchAPI handler @@ -129,6 +115,9 @@ export const createReactStreamingHandler = async ( metaTags = routeHookOutput.meta + // @MARK @TODO(RSC_DC): the entry path for RSC will be different, + // because we don't want to inject a full bundle, just a slice of it + // I'm not sure what though.... const jsBundles = [ clientEntryPath, // @NOTE: must have slash in front bundle && '/' + bundle, diff --git a/packages/vite/src/streaming/streamHelpers.ts b/packages/vite/src/streaming/streamHelpers.ts index 53f9082cf5f6..47a6e847fccd 100644 --- a/packages/vite/src/streaming/streamHelpers.ts +++ b/packages/vite/src/streaming/streamHelpers.ts @@ -18,6 +18,7 @@ import { } from '@redwoodjs/web/dist/components/ServerInject' import type { MiddlewareResponse } from '../middleware/MiddlewareResponse' +import { rscWebpackShims } from '../rsc/rscWebpackShims' import { createBufferedTransformStream } from './transforms/bufferedTransform' import { createTimeoutTransform } from './transforms/cancelTimeoutTransform' @@ -126,7 +127,7 @@ export async function reactRenderToStreamResponse( bootstrapScriptContent: // Only insert assetMap if clientside JS will be loaded jsBundles.length > 0 - ? `window.__REDWOOD__ASSET_MAP = ${assetMap}` + ? `window.__REDWOOD__ASSET_MAP = ${assetMap}; ${rscWebpackShims}` : undefined, bootstrapModules: jsBundles, }