fix: add images from frontmatter as well (#4156)

* ci workflow gating on ts-sdk changes

* test this; expect not to run, but other CI failure

* expect to trigger

* update CI

* expect to not trigger

* revert ruby change, working as expected

* added filepaths from frontmatter

* ut working, weird format

* painful greymatter caching

* small refactor and cleanup

* fix linter bug

* linter?

* added more tags and tests

* format

* prettier vs lint

* maybe better lint?

* beat linter

* beat linter

* get rid of useless block

packages/cli/docs-markdown-utils/package.json

@@ -27,6 +27,7 @@
     "depcheck": "depcheck"
   },
   "dependencies": {
+    "@fern-api/fdr-sdk": "0.98.16-3955e989a",
     "@fern-api/fs-utils": "workspace:*",
     "@fern-api/task-context": "workspace:*",
     "gray-matter": "^4.0.3",

packages/cli/docs-markdown-utils/src/__test__/parseImagePaths.test.ts

@@ -101,6 +101,52 @@ describe("parseImagePaths", () => {
         `);
     });
 
+    it("should parse url from frontmatter json", () => {
+        const page = '---\nimage: { type: "url", value: "https://someurl.com" }\n---';
+        const result = parseImagePaths(page, PATHS);
+        expect(result.filepaths).toEqual([]);
+        expect(result.markdown.trim()).toEqual("---\nimage:\n  type: url\n  value: 'https://someurl.com'\n---");
+    });
+
+    it("should parse url from frontmatter yaml", () => {
+        const page = '---\nimage:\n  type: url\n  value: "https://someurl.com"\n---';
+        const result = parseImagePaths(page, PATHS);
+        expect(result.filepaths).toEqual([]);
+        expect(result.markdown.trim()).toEqual("---\nimage:\n  type: url\n  value: 'https://someurl.com'\n---");
+    });
+
+    it("should parse url from frontmatter text", () => {
+        const page = '---\nimage: "https://someurl.com"\n---';
+        const result = parseImagePaths(page, PATHS);
+        expect(result.filepaths).toEqual([]);
+        expect(result.markdown.trim()).toEqual("---\nimage:\n  type: url\n  value: 'https://someurl.com'\n---");
+    });
+
+    it("should parse images from frontmatter text", () => {
+        const page = '---\nimage: "path/to/image.png"\n---';
+        const result = parseImagePaths(page, PATHS);
+        expect(result.filepaths).toEqual(["/Volume/git/fern/my/docs/folder/path/to/image.png"]);
+        expect(result.markdown.trim()).toEqual(
+            "---\nimage:\n  type: fileId\n  value: /Volume/git/fern/my/docs/folder/path/to/image.png\n---"
+        );
+    });
+
+    it("should parse og:images from frontmatter text", () => {
+        const page = '---\nog:image: "path/to/image.png"\n---';
+        const result = parseImagePaths(page, PATHS);
+        expect(result.filepaths).toEqual(["/Volume/git/fern/my/docs/folder/path/to/image.png"]);
+        expect(result.markdown.trim()).toEqual(
+            "---\n'og:image':\n  type: fileId\n  value: /Volume/git/fern/my/docs/folder/path/to/image.png\n---"
+        );
+    });
+
+    it("should parse the same result when run twice for frontmatter text", () => {
+        const page = '---\nimage: "path/to/image.png"\n---';
+        const result = parseImagePaths(page, PATHS);
+        const result2 = parseImagePaths(page, PATHS);
+        expect(result.markdown).toEqual(result.markdown);
+    });
+
     it("should parse image with alt on multiple lines", () => {
         const page = "This is a test page with an image ![image with \n new line in alt](path/to/image.png)";
         const result = parseImagePaths(page, PATHS);

packages/cli/docs-markdown-utils/src/parseImagePaths.ts

@@ -1,3 +1,4 @@
+import { DocsV1Write } from "@fern-api/fdr-sdk";
 import { AbsoluteFilePath, dirname, relative, RelativeFilePath, resolve } from "@fern-api/fs-utils";
 import { TaskContext } from "@fern-api/task-context";
 import grayMatter from "gray-matter";
@@ -30,11 +31,23 @@ export function parseImagePaths(
     filepaths: AbsoluteFilePath[];
     markdown: string;
 } {
-    const { content, data } = grayMatter(markdown);
+    // Don't remove {}! https://github.com/jonschlinkert/gray-matter/issues/43#issuecomment-318258919
+    const { content, data } = grayMatter(markdown, {});
     let replacedContent = content;
 
     const filepaths = new Set<AbsoluteFilePath>();
 
+    function mapImage(image: string | undefined) {
+        const resolvedPath = resolvePath(image, metadata);
+        if (resolvedPath != null) {
+            filepaths.add(resolvedPath);
+            return resolvedPath;
+        }
+        return;
+    }
+
+    visitFrontmatterImages(data, ["image", "og:image", "og:logo", "twitter:image"], mapImage);
+
     const tree = fromMarkdown(content, {
         extensions: [mdx()],
         mdastExtensions: [mdxFromMarkdown()]
@@ -118,7 +131,7 @@ export function parseImagePaths(
             }
         }
 
-        if (replaced === original) {
+        if (replaced === original && filepaths.size === 0) {
             return;
         }
 
@@ -169,7 +182,7 @@ export function replaceImagePathsAndUrls(
     metadata: AbsolutePathMetadata,
     context: TaskContext
 ): string {
-    const { content, data } = grayMatter(markdown);
+    const { content, data } = grayMatter(markdown, {});
     let replacedContent = content;
 
     const tree = fromMarkdown(content, {
@@ -179,6 +192,23 @@ export function replaceImagePathsAndUrls(
 
     let offset = 0;
 
+    function mapImage(image: string | undefined) {
+        if (image != null && !isExternalUrl(image) && !isDataUrl(image)) {
+            try {
+                const fileId = fileIdsMap.get(AbsoluteFilePath.of(image));
+                if (fileId != null) {
+                    return `file:${fileId}`;
+                }
+            } catch (e) {
+                // do nothing
+                return;
+            }
+        }
+        return;
+    }
+
+    visitFrontmatterImages(data, ["image", "og:image", "og:logo", "twitter:image"], mapImage);
+
     visit(tree, (node) => {
         if (node.position == null) {
             return;
@@ -188,15 +218,9 @@ export function replaceImagePathsAndUrls(
         let replaced = original;
 
         function replaceSrc(src: string | undefined) {
-            if (src != null && !isExternalUrl(src) && !isDataUrl(src)) {
-                try {
-                    const fileId = fileIdsMap.get(AbsoluteFilePath.of(src));
-                    if (fileId != null) {
-                        replaced = replaced.replace(src, `file:${fileId}`);
-                    }
-                } catch (e) {
-                    // do nothing
-                }
+            const imageSrc = mapImage(src);
+            if (src && imageSrc) {
+                replaced = replaced.replace(src, imageSrc);
             }
         }
 
@@ -347,3 +371,36 @@ function trimAnchor(text: unknown): string | undefined {
     }
     return text.replace(/#.*$/, "");
 }
+
+function visitFrontmatterImages(
+    data: Record<string, string | DocsV1Write.FileIdOrUrl>,
+    keys: string[],
+    mapImage: (image: string | undefined) => string | undefined
+) {
+    for (const key of keys) {
+        const value = data[key];
+        if (value != null) {
+            // realtime validation, this also assumes there can be other stuff in the object, but we only care about the valid keys
+            if (typeof value === "object") {
+                if (value.type === "fileId") {
+                    data[key] = {
+                        ...value,
+                        value: mapImage(value.value) ?? value.value
+                    };
+                }
+            } else if (typeof value === "string") {
+                const mappedImage = mapImage(value);
+                data[key] = mappedImage
+                    ? {
+                          type: "fileId",
+                          value: mappedImage
+                      }
+                    : {
+                          type: "url",
+                          value
+                      };
+            }
+            // else do nothing
+        }
+    }
+}

yarn.lock

@@ -3563,6 +3563,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@fern-api/docs-markdown-utils@workspace:packages/cli/docs-markdown-utils"
   dependencies:
+    "@fern-api/fdr-sdk": 0.98.16-3955e989a
     "@fern-api/fs-utils": "workspace:*"
     "@fern-api/task-context": "workspace:*"
     "@types/diff": ^5.2.1