From 562210f3e083ddd40cd6e5109ef9cfd65182d1e6 Mon Sep 17 00:00:00 2001 From: Leo McArdle Date: Fri, 12 Jul 2024 14:49:40 +0000 Subject: [PATCH] enhance(cloud-function): ad-hoc SSR https://mozilla-hub.atlassian.net/browse/MP-1349 inline env at build time into ssr bundle --- .github/workflows/dev-build.yml | 3 -- .github/workflows/prod-build.yml | 3 -- .github/workflows/stage-build.yml | 3 -- .github/workflows/test-build.yml | 3 +- cloud-function/package.json | 3 +- cloud-function/src/app.ts | 4 ++ cloud-function/src/handlers/proxy-content.ts | 27 +++------- cloud-function/src/handlers/render-html.ts | 52 ++++++++++++++++++++ cloud-function/src/headers.ts | 25 +++++++++- cloud-function/tsconfig.json | 3 +- package.json | 2 +- ssr/webpack.config.js | 4 ++ 12 files changed, 98 insertions(+), 34 deletions(-) create mode 100644 cloud-function/src/handlers/render-html.ts diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 26d3f2b0a474..ec3241714bc1 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -159,9 +159,6 @@ jobs: # Generate sitemap index file yarn build --sitemap-index - # SSR all pages - yarn render:html - # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json diff --git a/.github/workflows/prod-build.yml b/.github/workflows/prod-build.yml index b2134db8c9e1..e628c3d480a8 100644 --- a/.github/workflows/prod-build.yml +++ b/.github/workflows/prod-build.yml @@ -286,9 +286,6 @@ jobs: # Generate sitemap index file yarn build --sitemap-index - # SSR all pages - yarn render:html - # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml index 2e513a546a5b..6b5f8e1a94d8 100644 --- a/.github/workflows/stage-build.yml +++ b/.github/workflows/stage-build.yml @@ -302,9 +302,6 @@ jobs: # Generate sitemap index file yarn build --sitemap-index - # SSR all pages - yarn render:html - # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 4f8c96e0af0e..0fc17ed348f5 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -194,7 +194,7 @@ jobs: yarn build --sitemap-index # SSR all pages - yarn render:html + # yarn render:html # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json @@ -240,6 +240,7 @@ jobs: run: | npm ci npm run build-redirects + npm run build-canonicals - name: Deploy Function if: ${{ ! vars.SKIP_FUNCTION }} diff --git a/cloud-function/package.json b/cloud-function/package.json index 2ff6b8639f34..efb16c8c80c1 100644 --- a/cloud-function/package.json +++ b/cloud-function/package.json @@ -12,8 +12,9 @@ "build-canonicals": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node src/build-canonicals.ts", "build-redirects": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node src/build-redirects.ts", "copy-internal": "rm -rf ./src/internal && cp -R ../libs ./src/internal", + "copy-ssr": "mkdir -p ./src/internal/ssr && cp ../ssr/dist/main.js ./src/internal/ssr/", "gcp-build": "npm run build", - "prepare": "([ ! -e ../libs ] || npm run copy-internal)", + "prepare": "([ ! -e ../libs ] || npm run copy-internal) && ([ ! -e ../ssr/dist ] || npm run copy-ssr)", "proxy": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node src/proxy.ts", "server": "npm run build && functions-framework --target=mdnHandler", "server:watch": "nodemon --exec npm run server", diff --git a/cloud-function/src/app.ts b/cloud-function/src/app.ts index 1f7221f70d4f..eecaf1a91642 100644 --- a/cloud-function/src/app.ts +++ b/cloud-function/src/app.ts @@ -23,6 +23,7 @@ import { resolveRunnerHtml } from "./middlewares/resolve-runner-html.js"; import { proxyRunner } from "./handlers/proxy-runner.js"; import { stripForwardedHostHeaders } from "./middlewares/stripForwardedHostHeaders.js"; import { proxyPong } from "./handlers/proxy-pong.js"; +import { handleRenderHTML } from "./handlers/render-html.js"; const router = Router(); router.use(stripForwardedHostHeaders); @@ -88,6 +89,7 @@ router.get( redirectTrailingSlash, redirectMovedPages, resolveIndexHTML, + handleRenderHTML, proxyContent ); router.get( @@ -96,6 +98,7 @@ router.get( redirectLocale, redirectEnforceTrailingSlash, resolveIndexHTML, + handleRenderHTML, proxyContent ); // MDN Plus, static pages, etc. @@ -106,6 +109,7 @@ router.get( redirectLocale, redirectTrailingSlash, resolveIndexHTML, + handleRenderHTML, proxyContent ); router.all("*", notFound); diff --git a/cloud-function/src/handlers/proxy-content.ts b/cloud-function/src/handlers/proxy-content.ts index 55ad0e31c793..62522ad3923f 100644 --- a/cloud-function/src/handlers/proxy-content.ts +++ b/cloud-function/src/handlers/proxy-content.ts @@ -1,18 +1,14 @@ -/* eslint-disable n/no-unsupported-features/node-builtins */ import { createProxyMiddleware, fixRequestBody, responseInterceptor, } from "http-proxy-middleware"; -import { withContentResponseHeaders } from "../headers.js"; +import { withProxiedContentResponseHeaders } from "../headers.js"; import { Source, sourceUri } from "../env.js"; import { PROXY_TIMEOUT } from "../constants.js"; import { isLiveSampleURL } from "../utils.js"; - -const NOT_FOUND_PATH = "en-us/_spas/404.html"; - -let notFoundBuffer: ArrayBuffer; +import { renderHTMLForContext } from "./render-html.js"; const target = sourceUri(Source.content); @@ -27,23 +23,16 @@ export const proxyContent = createProxyMiddleware({ proxyReq: fixRequestBody, proxyRes: responseInterceptor( async (responseBuffer, proxyRes, req, res) => { - withContentResponseHeaders(proxyRes, req, res); if (proxyRes.statusCode === 404 && !isLiveSampleURL(req.url ?? "")) { - const tryHtml = await fetch( - `${target}${req.url?.slice(1)}/index.html` + const html = await renderHTMLForContext( + req, + res, + `${target}${req.url?.slice(1)}/index.json` ); - if (tryHtml.ok) { - res.statusCode = 200; - res.setHeader("Content-Type", "text/html"); - return Buffer.from(await tryHtml.arrayBuffer()); - } else if (!notFoundBuffer) { - const response = await fetch(`${target}${NOT_FOUND_PATH}`); - notFoundBuffer = await response.arrayBuffer(); - } - res.setHeader("Content-Type", "text/html"); - return Buffer.from(notFoundBuffer); + return Buffer.from(html); } + withProxiedContentResponseHeaders(proxyRes, req, res); return responseBuffer; } ), diff --git a/cloud-function/src/handlers/render-html.ts b/cloud-function/src/handlers/render-html.ts new file mode 100644 index 000000000000..c43f48e88d25 --- /dev/null +++ b/cloud-function/src/handlers/render-html.ts @@ -0,0 +1,52 @@ +/* eslint-disable n/no-unsupported-features/node-builtins */ +import type { NextFunction, Request, Response } from "express"; +import type { IncomingMessage, ServerResponse } from "node:http"; + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +import { renderHTML } from "../internal/ssr/main.js"; +import { sourceUri, Source } from "../env.js"; +import { withRenderedContentResponseHeaders } from "../headers.js"; + +const target = sourceUri(Source.content); + +export async function handleRenderHTML( + req: Request, + res: Response, + next: NextFunction +) { + if (req.url.endsWith("/index.html")) { + const html = await renderHTMLForContext( + req, + res, + target.replace(/\/$/, "") + req.url.replace(/html$/, "json") + ); + res.send(html).end(); + } else { + next(); + } +} + +export async function renderHTMLForContext( + req: IncomingMessage, + res: ServerResponse, + contextUrl: string +) { + res.setHeader("Content-Type", "text/html"); + res.setHeader("X-MDN-SSR", "true"); + try { + const contextRes = await fetch(contextUrl); + if (!contextRes.ok) { + throw new Error(contextRes.status.toString()); + } + const context = await contextRes.json(); + res.statusCode = 200; + withRenderedContentResponseHeaders(req, res); + return renderHTML(context); + } catch { + res.statusCode = 404; + withRenderedContentResponseHeaders(req, res); + const context = { url: req.url, pageNotFound: true }; + return renderHTML(context); + } +} diff --git a/cloud-function/src/headers.ts b/cloud-function/src/headers.ts index 9deeebc3fa95..900ecc024856 100644 --- a/cloud-function/src/headers.ts +++ b/cloud-function/src/headers.ts @@ -14,7 +14,7 @@ const NO_CACHE_VALUE = "no-store, must-revalidate"; const HASHED_REGEX = /\.[a-f0-9]{8,32}\./; -export function withContentResponseHeaders( +export function withProxiedContentResponseHeaders( proxyRes: IncomingMessage, req: IncomingMessage, res: ServerResponse @@ -56,6 +56,29 @@ export function withContentResponseHeaders( return res; } +export function withRenderedContentResponseHeaders( + req: IncomingMessage, + res: ServerResponse +) { + if (res.headersSent) { + console.warn( + `Cannot set content response headers. Headers already sent for: ${req.url}` + ); + return; + } + + const url = req.url ?? ""; + + setContentResponseHeaders((name, value) => res.setHeader(name, value), {}); + + const cacheControl = getCacheControl(res.statusCode ?? 0, url); + if (cacheControl) { + res.setHeader("Cache-Control", cacheControl); + } + + return res; +} + function getCacheControl(statusCode: number, url: string) { if ( statusCode === 404 || diff --git a/cloud-function/tsconfig.json b/cloud-function/tsconfig.json index fae5d6bfe79b..2ef9f9f851ff 100644 --- a/cloud-function/tsconfig.json +++ b/cloud-function/tsconfig.json @@ -19,8 +19,7 @@ "noPropertyAccessFromIndexSignature": true, "noUncheckedIndexedAccess": true, "noUnusedLocals": true, - "noUnusedParameters": true, - "checkJs": true + "noUnusedParameters": true }, "ts-node": { "esm": true, diff --git a/package.json b/package.json index 96820b228bd9..6b09b72d4c5f 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "build:docs": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/cli.ts -n", "build:glean": "cd client && cross-env VIRTUAL_ENV=venv glean translate src/telemetry/metrics.yaml src/telemetry/pings.yaml -f typescript -o src/telemetry/generated", "build:prepare": "yarn build:client && yarn build:ssr && yarn tool popularities && yarn tool spas && yarn tool gather-git-history && yarn tool build-robots-txt", - "build:ssr": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node ssr/prepare.ts && cd ssr && webpack --mode=production", + "build:ssr": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node ssr/prepare.ts && cd ssr && cross-env NODE_ENV=production webpack --mode=production", "build:sw": "cd client/pwa && yarn && yarn build:prod", "build:sw-dev": "cd client/pwa && yarn && yarn build", "check:tsc": "find . -name 'tsconfig.json' ! -wholename '**/node_modules/**' -print0 | xargs -n1 -P 2 -0 sh -c 'cd `dirname $0` && echo \"🔄 $(pwd)\" && npx tsc --noEmit && echo \"☑️ $(pwd)\" || exit 255'", diff --git a/ssr/webpack.config.js b/ssr/webpack.config.js index 64a4195de08d..42e7cd9204a4 100644 --- a/ssr/webpack.config.js +++ b/ssr/webpack.config.js @@ -1,6 +1,9 @@ import { fileURLToPath } from "node:url"; import nodeExternals from "webpack-node-externals"; import webpack from "webpack"; +import getClientEnvironment from "../client/config/env.js"; + +const env = getClientEnvironment(); const config = { context: fileURLToPath(new URL(".", import.meta.url)), @@ -94,6 +97,7 @@ const config = { new webpack.optimize.LimitChunkCountPlugin({ maxChunks: 1, }), + new webpack.DefinePlugin(env.stringified), ], experiments: { outputModule: true,