fix: add missing fs method rewrites to handle fetchRemoteFile in dsg/…

…ssr engine (#38822)

* test(adapters-e2e): enable placeholder tests in ssr remote-file

* fix(gatsby-adapter-netlify): bundling file-cdn

* fix(gatsby): set pathPrefix in engines

* fix(gatsby): add missing fs method rewrites to handle fetchRemoteFile in dsg/ssr engine

(cherry picked from commit bbdddd7)

e2e-tests/adapters/cypress/e2e/remote-file.cy.ts

@@ -29,7 +29,7 @@ const configs = [
   {
     title: `remote-file (SSR, Page Query)`,
     pagePath: `/routes/ssr/remote-file/`,
-    placeholders: false,
+    placeholders: true,
   },
 ]
 

e2e-tests/adapters/src/pages/routes/ssr/remote-file.jsx

@@ -4,7 +4,7 @@ import React from "react"
 import { GatsbyImage } from "gatsby-plugin-image"
 import Layout from "../../../components/layout"
 
-const RemoteFile = ({ data }) => {
+const RemoteFile = ({ data, serverData }) => {
   return (
     <Layout>
       {data.allMyRemoteImage.nodes.map(node => {
@@ -54,6 +54,7 @@ const RemoteFile = ({ data }) => {
           </div>
         )
       })}
+      <pre>{JSON.stringify(serverData, null, 2)}</pre>
     </Layout>
   )
 }
@@ -65,8 +66,7 @@ export const pageQuery = graphql`
         id
         url
         filename
-        # FILE_CDN is not supported in SSR/DSG yet
-        # publicUrl
+        publicUrl
         resize(width: 100) {
           height
           width
@@ -75,23 +75,17 @@ export const pageQuery = graphql`
         fixed: gatsbyImage(
           layout: FIXED
           width: 100
-          # only NONE placeholder is supported in SSR/DSG
-          # placeholder: DOMINANT_COLOR
-          placeholder: NONE
+          placeholder: DOMINANT_COLOR
         )
         constrained: gatsbyImage(
           layout: CONSTRAINED
           width: 300
-          # only NONE placeholder is supported in SSR/DSG
-          # placeholder: DOMINANT_COLOR
-          placeholder: NONE
+          placeholder: BLURRED
         )
         constrained_traced: gatsbyImage(
           layout: CONSTRAINED
           width: 300
-          # only NONE placeholder is supported in SSR/DSG
-          # placeholder: DOMINANT_COLOR
-          placeholder: NONE
+          placeholder: TRACED_SVG
         )
         full: gatsbyImage(layout: FULL_WIDTH, width: 500, placeholder: NONE)
       }
@@ -109,3 +103,11 @@ export const pageQuery = graphql`
 `
 
 export default RemoteFile
+
+export function getServerData() {
+  return {
+    props: {
+      ssr: true,
+    },
+  }
+}

packages/gatsby-adapter-netlify/src/file-cdn-url-generator.ts

@@ -1,4 +1,4 @@
-import crypto from "crypto"
+import { createHash } from "crypto"
 import { basename } from "path"
 
 import type { FileCdnUrlGeneratorFn, FileCdnSourceImage } from "gatsby"
@@ -21,8 +21,7 @@ export const generateFileUrl: FileCdnUrlGeneratorFn = function generateFileUrl(
     baseURL.searchParams.append(`cd`, source.internal.contentDigest)
   } else {
     baseURL = new URL(
-      `${placeholderOrigin}${pathPrefix}/_gatsby/file/${crypto
-        .createHash(`md5`)
+      `${placeholderOrigin}${pathPrefix}/_gatsby/file/${createHash(`md5`)
         .update(source.url)
         .digest(`hex`)}/${basename(source.filename)}`
     )

packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts

@@ -129,6 +129,11 @@ export async function createGraphqlEngineBundle(
   reporter: Reporter,
   isVerbose?: boolean
 ): Promise<webpack.Compilation | undefined> {
+  const state = store.getState()
+  const pathPrefix = state.program.prefixPaths
+    ? state.config.pathPrefix ?? ``
+    : ``
+
   const schemaSnapshotString = await fs.readFile(
     path.join(rootDir, `.cache`, `schema.gql`),
     `utf-8`
@@ -151,7 +156,7 @@ export async function createGraphqlEngineBundle(
 
   // We force a specific lmdb binary module if we detected a broken lmdb installation or if we detect the presence of an adapter
   let forcedLmdbBinaryModule: string | undefined = undefined
-  if (store.getState().adapter.instance) {
+  if (state.adapter.instance) {
     forcedLmdbBinaryModule = `${lmdbPackage}/node.abi83.glibc.node`
   }
   // We always force the binary if we've installed an alternative path
@@ -317,6 +322,7 @@ export async function createGraphqlEngineBundle(
         "process.env.GATSBY_SKIP_WRITING_SCHEMA_TO_FILE": `true`,
         "process.env.NODE_ENV": JSON.stringify(`production`),
         SCHEMA_SNAPSHOT: JSON.stringify(schemaSnapshotString),
+        PATH_PREFIX: JSON.stringify(pathPrefix),
         "process.env.GATSBY_LOGGER": JSON.stringify(`yurnalist`),
         "process.env.GATSBY_SLICES": JSON.stringify(
           !!process.env.GATSBY_SLICES

packages/gatsby/src/schema/graphql-engine/entry.ts

@@ -43,12 +43,34 @@ export class GraphQLEngine {
     this.getRunner()
   }
 
+  private setupPathPrefix(pathPrefix: string): void {
+    if (pathPrefix) {
+      store.dispatch({
+        type: `SET_PROGRAM`,
+        payload: {
+          prefixPaths: true,
+        },
+      })
+
+      store.dispatch({
+        type: `SET_SITE_CONFIG`,
+        payload: {
+          ...store.getState().config,
+          pathPrefix,
+        },
+      })
+    }
+  }
+
   private async _doGetRunner(): Promise<GraphQLRunner> {
     await tracerReadyPromise
 
     const wrapActivity = reporter.phantomActivity(`Initializing GraphQL Engine`)
     wrapActivity.start()
     try {
+      // @ts-ignore PATH_PREFIX is being "inlined" by bundler
+      this.setupPathPrefix(PATH_PREFIX)
+
       // @ts-ignore SCHEMA_SNAPSHOT is being "inlined" by bundler
       store.dispatch(actions.createTypes(SCHEMA_SNAPSHOT))
 

packages/gatsby/src/utils/page-ssr-module/lambda.ts

@@ -10,7 +10,7 @@ import { promisify } from "util"
 
 import type { IGatsbyPage } from "../../internal"
 import type { ISSRData } from "./entry"
-import { link } from "linkfs"
+import { link, rewritableMethods as linkRewritableMethods } from "linkfs"
 
 const cdnDatastore = `%CDN_DATASTORE_PATH%`
 const PATH_PREFIX = `%PATH_PREFIX%`
@@ -30,7 +30,7 @@ function setupFsWrapper(): string {
     global.__GATSBY.root = TEMP_DIR
 
     // TODO: don't hardcode this
-    const cacheDir = `/var/task/.cache`
+    const cacheDir = `${process.cwd()}/.cache`
 
     // we need to rewrite fs
     const rewrites = [
@@ -47,25 +47,132 @@ function setupFsWrapper(): string {
       to: TEMP_CACHE_DIR,
       rewrites,
     })
+
+    // copied from https://github.com/streamich/linkfs/blob/master/src/index.ts#L126-L142
+    function mapPathUsingRewrites(fsPath: fs.PathLike): string {
+      let filename = path.resolve(String(fsPath))
+      for (const [from, to] of rewrites) {
+        if (filename.indexOf(from) === 0) {
+          const rootRegex = /(?:^[a-zA-Z]:\\$)|(?:^\/$)/ // C:\ vs /
+          const isRoot = from.match(rootRegex)
+          const baseRegex = `^(` + from.replace(/\\/g, `\\\\`) + `)`
+
+          if (isRoot) {
+            const regex = new RegExp(baseRegex)
+            filename = filename.replace(regex, () => to + path.sep)
+          } else {
+            const regex = new RegExp(baseRegex + `(\\\\|/|$)`)
+            filename = filename.replace(regex, (_match, _p1, p2) => to + p2)
+          }
+        }
+      }
+      return filename
+    }
+
+    function applyRename<
+      T = typeof import("fs") | typeof import("fs").promises
+    >(fsToRewrite: T, lfs: T, method: "rename" | "renameSync"): void {
+      const original = fsToRewrite[method]
+      if (original) {
+        // @ts-ignore - complains about __promisify__ which doesn't actually exist in runtime
+        lfs[method] = (
+          ...args: Parameters<typeof import("fs")["rename" | "renameSync"]>
+        ): ReturnType<typeof import("fs")["rename" | "renameSync"]> => {
+          args[0] = mapPathUsingRewrites(args[0])
+          args[1] = mapPathUsingRewrites(args[1])
+          // @ts-ignore - can't decide which signature this is, but we just preserve the original
+          // signature here and mostly care about first 2 arguments being PathLike
+          return original.apply(fsToRewrite, args)
+        }
+      }
+    }
+
+    // linkfs doesn't handle following methods, so we need to add them manually
+    linkRewritableMethods.push(`createWriteStream`, `createReadStream`, `rm`)
+
+    function createLinkedFS<
+      T = typeof import("fs") | typeof import("fs").promises
+    >(fsToRewrite: T): T {
+      const linkedFS = link(fsToRewrite, rewrites) as T
+
+      // linkfs doesn't handle `to` argument in `rename` and `renameSync` methods
+      applyRename(fsToRewrite, linkedFS, `rename`)
+      applyRename(fsToRewrite, linkedFS, `renameSync`)
+
+      return linkedFS
+    }
+
     // Alias the cache dir paths to the temp dir
-    const lfs = link(fs, rewrites) as typeof import("fs")
+    const lfs = createLinkedFS(fs)
 
     // linkfs doesn't pass across the `native` prop, which graceful-fs needs
     for (const key in lfs) {
       if (Object.hasOwnProperty.call(fs[key], `native`)) {
         lfs[key].native = fs[key].native
       }
     }
-
-    const dbPath = path.join(TEMP_CACHE_DIR, `data`, `datastore`)
-
     // 'promises' is not initially linked within the 'linkfs'
     // package, and is needed by underlying Gatsby code (the
     // @graphql-tools/code-file-loader)
-    lfs.promises = link(fs.promises, rewrites)
+    lfs.promises = createLinkedFS(fs.promises)
+
+    const originalWritesStream = fs.WriteStream
+    function LinkedWriteStream(
+      this: fs.WriteStream,
+      ...args: Parameters<(typeof fs)["createWriteStream"]>
+    ): fs.WriteStream {
+      args[0] = mapPathUsingRewrites(args[0])
+      args[1] =
+        typeof args[1] === `string`
+          ? {
+              flags: args[1],
+              // @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options)
+              fs: lfs,
+            }
+          : {
+              ...(args[1] || {}),
+              // @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options)
+              fs: lfs,
+            }
+
+      // @ts-ignore TS doesn't like extending prototype "classes"
+      return originalWritesStream.apply(this, args)
+    }
+    LinkedWriteStream.prototype = Object.create(originalWritesStream.prototype)
+    // @ts-ignore TS doesn't like extending prototype "classes"
+    lfs.WriteStream = LinkedWriteStream
+
+    const originalReadStream = fs.ReadStream
+    function LinkedReadStream(
+      this: fs.ReadStream,
+      ...args: Parameters<(typeof fs)["createReadStream"]>
+    ): fs.ReadStream {
+      args[0] = mapPathUsingRewrites(args[0])
+      args[1] =
+        typeof args[1] === `string`
+          ? {
+              flags: args[1],
+              // @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options)
+              fs: lfs,
+            }
+          : {
+              ...(args[1] || {}),
+              // @ts-ignore there is `fs` property in options doh (https://nodejs.org/api/fs.html#fscreatewritestreampath-options)
+              fs: lfs,
+            }
+
+      // @ts-ignore TS doesn't like extending prototype "classes"
+      return originalReadStream.apply(this, args)
+    }
+    LinkedReadStream.prototype = Object.create(originalReadStream.prototype)
+    // @ts-ignore TS doesn't like extending prototype "classes"
+    lfs.ReadStream = LinkedReadStream
+
+    const dbPath = path.join(TEMP_CACHE_DIR, `data`, `datastore`)
 
     // Gatsby uses this instead of fs if present
     // eslint-disable-next-line no-underscore-dangle
+    // @ts-ignore __promisify__ stuff
     global._fsWrapper = lfs
 
     if (!cdnDatastore) {