validate front matter (#1054)

* validate front matter

* normalizeFrontMatter

* fix title: null

* pretty error in preview, crash on build

* simplify

* simplify 2

* treat invalid frontmatter as content

---------

Co-authored-by: Mike Bostock <mbostock@gmail.com>

src/build.ts

@@ -73,7 +73,7 @@ export async function build(
     const start = performance.now();
     const source = await readFile(sourcePath, "utf8");
     const page = parseMarkdown(source, options);
-    if (page?.data?.draft) {
+    if (page.data.draft) {
       effects.logger.log(faint("(skipped)"));
       continue;
     }

src/config.ts

@@ -104,10 +104,10 @@ function readPages(root: string, md: MarkdownIt): Page[] {
   if (cachedPages?.key === key) return cachedPages.pages;
   const pages: Page[] = [];
   for (const {file, source} of files) {
-    const parsed = parseMarkdownMetadata(source, {path: file, md});
-    if (parsed?.data?.draft) continue;
+    const {data, title} = parseMarkdownMetadata(source, {path: file, md});
+    if (data.draft) continue;
     const name = basename(file, ".md");
-    const page = {path: join("/", dirname(file), name), name: parsed.title ?? "Untitled"};
+    const page = {path: join("/", dirname(file), name), name: title ?? "Untitled"};
     if (name === "index") pages.unshift(page);
     else pages.push(page);
   }
@@ -199,7 +199,7 @@ function normalizeBase(base: any): string {
   return base;
 }
 
-function normalizeTheme(spec: any): string[] {
+export function normalizeTheme(spec: any): string[] {
   return resolveTheme(typeof spec === "string" ? [spec] : spec === null ? [] : Array.from(spec, String));
 }
 
@@ -254,19 +254,24 @@ function normalizeToc(spec: any): TableOfContents {
   return {label, show};
 }
 
-export function mergeToc(spec: any, toc: TableOfContents): TableOfContents {
-  let {label = toc.label, show = toc.show} = typeof spec !== "object" ? {show: spec} : spec ?? {};
-  label = String(label);
-  show = Boolean(show);
+export function mergeToc(spec: Partial<TableOfContents> = {}, toc: TableOfContents): TableOfContents {
+  const {label = toc.label, show = toc.show} = spec;
   return {label, show};
 }
 
-export function mergeStyle(path: string, style: any, theme: any, defaultStyle: null | Style): null | Style {
+export function mergeStyle(
+  path: string,
+  style: string | null | undefined,
+  theme: string[] | undefined,
+  defaultStyle: null | Style
+): null | Style {
   return style === undefined && theme === undefined
     ? defaultStyle
     : style === null
     ? null // disable
     : style !== undefined
-    ? {path: resolvePath(path, String(style))}
-    : {theme: normalizeTheme(theme)};
+    ? {path: resolvePath(path, style)}
+    : theme === undefined
+    ? defaultStyle
+    : {theme};
 }

src/frontMatter.ts

@@ -0,0 +1,69 @@
+import matter from "gray-matter";
+import {normalizeTheme} from "./config.js";
+import {yellow} from "./tty.js";
+
+export interface FrontMatter {
+  title?: string | null;
+  toc?: {show?: boolean; label?: string};
+  style?: string | null;
+  theme?: string[];
+  index?: boolean;
+  keywords?: string[];
+  draft?: boolean;
+  sidebar?: boolean;
+  sql?: {[key: string]: string};
+}
+
+export function readFrontMatter(input: string): {content: string; data: FrontMatter} {
+  try {
+    const {content, data} = matter(input, {});
+    return {content, data: normalizeFrontMatter(data)};
+  } catch (error: any) {
+    if ("mark" in error) {
+      console.warn(`${yellow("Invalid front matter")}: ${error.reason}`);
+      return {data: {}, content: input};
+    }
+    throw error;
+  }
+}
+
+export function normalizeFrontMatter(spec: any = {}): FrontMatter {
+  const frontMatter: FrontMatter = {};
+  if (spec == null || typeof spec !== "object") return frontMatter;
+  const {title, sidebar, toc, index, keywords, draft, sql, style, theme} = spec;
+  if (title !== undefined) frontMatter.title = stringOrNull(title);
+  if (sidebar !== undefined) frontMatter.sidebar = Boolean(sidebar);
+  if (toc !== undefined) frontMatter.toc = normalizeToc(toc);
+  if (index !== undefined) frontMatter.index = Boolean(index);
+  if (keywords !== undefined) frontMatter.keywords = normalizeKeywords(keywords);
+  if (draft !== undefined) frontMatter.draft = Boolean(draft);
+  if (sql !== undefined) frontMatter.sql = normalizeSql(sql);
+  if (style !== undefined) frontMatter.style = stringOrNull(style);
+  if (theme !== undefined) frontMatter.theme = normalizeTheme(theme);
+  return frontMatter;
+}
+
+function stringOrNull(spec: unknown): string | null {
+  return spec == null ? null : String(spec);
+}
+
+function normalizeToc(spec: unknown): {show?: boolean; label?: string} {
+  if (spec == null) return {show: false};
+  if (typeof spec !== "object") return {show: Boolean(spec)};
+  const {show, label} = spec as {show: unknown; label: unknown};
+  const toc: FrontMatter["toc"] = {};
+  if (show !== undefined) toc.show = Boolean(show);
+  if (label !== undefined) toc.label = String(label);
+  return toc;
+}
+
+function normalizeKeywords(spec: unknown): string[] {
+  return spec == null ? [] : typeof spec === "string" ? [spec] : Array.from(spec as any, String);
+}
+
+function normalizeSql(spec: unknown): {[key: string]: string} {
+  if (spec == null || typeof spec !== "object") return {};
+  const sql: {[key: string]: string} = {};
+  for (const key in spec) sql[key] = String(spec[key]);
+  return sql;
+}

src/markdown.ts

@@ -1,7 +1,6 @@
 /* eslint-disable import/no-named-as-default-member */
 import {createHash} from "node:crypto";
 import {extname} from "node:path/posix";
-import matter from "gray-matter";
 import he from "he";
 import MarkdownIt from "markdown-it";
 import type {RuleCore} from "markdown-it/lib/parser_core.js";
@@ -10,6 +9,8 @@ import type {RenderRule} from "markdown-it/lib/renderer.js";
 import MarkdownItAnchor from "markdown-it-anchor";
 import type {Config} from "./config.js";
 import {mergeStyle} from "./config.js";
+import type {FrontMatter} from "./frontMatter.js";
+import {readFrontMatter} from "./frontMatter.js";
 import {rewriteHtmlPaths} from "./html.js";
 import {parseInfo} from "./info.js";
 import type {JavaScriptNode} from "./javascript/parse.js";
@@ -31,12 +32,12 @@ export interface MarkdownPage {
   header: string | null;
   body: string;
   footer: string | null;
-  data: {[key: string]: any} | null;
+  data: FrontMatter;
   style: string | null;
   code: MarkdownCode[];
 }
 
-export interface ParseContext {
+interface ParseContext {
   code: MarkdownCode[];
   startLine: number;
   currentLine: number;
@@ -326,7 +327,7 @@ export function createMarkdownIt({
 
 export function parseMarkdown(input: string, options: ParseOptions): MarkdownPage {
   const {md, path} = options;
-  const {content, data} = matter(input, {});
+  const {content, data} = readFrontMatter(input);
   const code: MarkdownCode[] = [];
   const context: ParseContext = {code, startLine: 0, currentLine: 0, path};
   const tokens = md.parse(content, context);
@@ -336,8 +337,8 @@ export function parseMarkdown(input: string, options: ParseOptions): MarkdownPag
     header: getHtml("header", data, options),
     body,
     footer: getHtml("footer", data, options),
-    data: isEmpty(data) ? null : data,
-    title: data.title ?? findTitle(tokens) ?? null,
+    data,
+    title: data.title !== undefined ? data.title : findTitle(tokens),
     style: getStyle(data, options),
     code
   };
@@ -346,10 +347,13 @@ export function parseMarkdown(input: string, options: ParseOptions): MarkdownPag
 /** Like parseMarkdown, but optimized to return only metadata. */
 export function parseMarkdownMetadata(input: string, options: ParseOptions): Pick<MarkdownPage, "data" | "title"> {
   const {md, path} = options;
-  const {content, data} = matter(input, {});
+  const {content, data} = readFrontMatter(input);
   return {
-    data: isEmpty(data) ? null : data,
-    title: data.title ?? findTitle(md.parse(content, {code: [], startLine: 0, currentLine: 0, path})) ?? null
+    data,
+    title:
+      data.title !== undefined
+        ? data.title
+        : findTitle(md.parse(content, {code: [], startLine: 0, currentLine: 0, path}))
   };
 }
 
@@ -367,7 +371,7 @@ function getHtml(
     : null;
 }
 
-function getStyle(data: Record<string, any>, {path, style = null}: ParseOptions): string | null {
+function getStyle(data: FrontMatter, {path, style = null}: ParseOptions): string | null {
   try {
     style = mergeStyle(path, data.style, data.theme, style);
   } catch (error) {
@@ -382,14 +386,8 @@ function getStyle(data: Record<string, any>, {path, style = null}: ParseOptions)
     : `observablehq:theme-${style.theme.join(",")}.css`;
 }
 
-// TODO Use gray-matter’s parts.isEmpty, but only when it’s accurate.
-function isEmpty(object) {
-  for (const key in object) return false;
-  return true;
-}
-
 // TODO Make this smarter.
-function findTitle(tokens: ReturnType<MarkdownIt["parse"]>): string | undefined {
+function findTitle(tokens: ReturnType<MarkdownIt["parse"]>): string | null {
   for (const [i, token] of tokens.entries()) {
     if (token.type === "heading_open" && token.tag === "h1") {
       const next = tokens[i + 1];
@@ -404,4 +402,5 @@ function findTitle(tokens: ReturnType<MarkdownIt["parse"]>): string | undefined
       }
     }
   }
+  return null;
 }

src/preview.ts

@@ -435,7 +435,7 @@ function getFiles({files, resolveFile}: Resolvers): Map<string, string> {
 }
 
 function getTables({data}: MarkdownPage): Map<string, string> {
-  return new Map(Object.entries(data?.sql ?? {}));
+  return new Map(Object.entries(data.sql ?? {}));
 }
 
 type CodePatch = {removed: string[]; added: string[]};

src/render.ts

@@ -27,9 +27,8 @@ export async function renderPage(page: MarkdownPage, options: RenderOptions & Re
   const {data} = page;
   const {base, path, title, preview} = options;
   const {loaders, resolvers = await getResolvers(page, options)} = options;
-  const sidebar = data?.sidebar !== undefined ? Boolean(data.sidebar) : options.sidebar;
-  const toc = mergeToc(data?.toc, options.toc);
-  const draft = Boolean(data?.draft);
+  const {draft = false, sidebar = options.sidebar} = data;
+  const toc = mergeToc(data.toc, options.toc);
   const {files, resolveFile, resolveImport} = resolvers;
   return String(html`<!DOCTYPE html>
 <meta charset="utf-8">${path === "/404" ? html`\n<base href="${preview ? "/" : base}">` : ""}
@@ -86,13 +85,13 @@ ${html.unsafe(rewriteHtml(page.body, resolvers))}</main>${renderFooter(page.foot
 `);
 }
 
-function registerTables(sql: Record<string, any>, options: RenderOptions): string {
+function registerTables(sql: Record<string, string>, options: RenderOptions): string {
   return Object.entries(sql)
     .map(([name, source]) => registerTable(name, source, options))
     .join("\n");
 }
 
-function registerTable(name: string, source: any, {path}: RenderOptions): string {
+function registerTable(name: string, source: string, {path}: RenderOptions): string {
   return `registerTable(${JSON.stringify(name)}, ${
     isAssetPath(source)
       ? `FileAttachment(${JSON.stringify(resolveRelativePath(path, source))})`

src/resolvers.ts

@@ -112,7 +112,7 @@ export async function getResolvers(
   }
 
   // Add SQL sources.
-  if (page.data?.sql) {
+  if (page.data.sql) {
     for (const source of Object.values(page.data.sql)) {
       files.add(String(source));
     }

test/config-test.ts

@@ -178,11 +178,9 @@ describe("mergeToc(spec, toc)", () => {
     const toc = config({pages: [], toc: true}, root).toc;
     assert.deepStrictEqual(mergeToc({show: false}, toc), {label: "Contents", show: false});
     assert.deepStrictEqual(mergeToc({label: "On this page"}, toc), {label: "On this page", show: true});
-    assert.deepStrictEqual(mergeToc(false, toc), {label: "Contents", show: false});
-    assert.deepStrictEqual(mergeToc(true, toc), {label: "Contents", show: true});
-    assert.deepStrictEqual(mergeToc(undefined, toc), {label: "Contents", show: true});
-    assert.deepStrictEqual(mergeToc(null, toc), {label: "Contents", show: true});
-    assert.deepStrictEqual(mergeToc(0, toc), {label: "Contents", show: false});
-    assert.deepStrictEqual(mergeToc(1, toc), {label: "Contents", show: true});
+    assert.deepStrictEqual(mergeToc({label: undefined}, toc), {label: "Contents", show: true});
+    assert.deepStrictEqual(mergeToc({show: true}, toc), {label: "Contents", show: true});
+    assert.deepStrictEqual(mergeToc({show: undefined}, toc), {label: "Contents", show: true});
+    assert.deepStrictEqual(mergeToc({}, toc), {label: "Contents", show: true});
   });
 });

test/frontMatter-test.ts

@@ -0,0 +1,88 @@
+import assert from "node:assert";
+import {normalizeFrontMatter} from "../src/frontMatter.js";
+
+describe("normalizeFrontMatter(spec)", () => {
+  it("returns the empty object for an undefined, null, empty spec", () => {
+    assert.deepStrictEqual(normalizeFrontMatter(), {});
+    assert.deepStrictEqual(normalizeFrontMatter(undefined), {});
+    assert.deepStrictEqual(normalizeFrontMatter(null), {});
+    assert.deepStrictEqual(normalizeFrontMatter(false), {});
+    assert.deepStrictEqual(normalizeFrontMatter(true), {});
+    assert.deepStrictEqual(normalizeFrontMatter({}), {});
+    assert.deepStrictEqual(normalizeFrontMatter(42), {});
+  });
+  it("coerces the title to a string or null", () => {
+    assert.deepStrictEqual(normalizeFrontMatter({title: 42}), {title: "42"});
+    assert.deepStrictEqual(normalizeFrontMatter({title: undefined}), {});
+    assert.deepStrictEqual(normalizeFrontMatter({title: null}), {title: null});
+    assert.deepStrictEqual(normalizeFrontMatter({title: ""}), {title: ""});
+    assert.deepStrictEqual(normalizeFrontMatter({title: "foo"}), {title: "foo"});
+    assert.deepStrictEqual(normalizeFrontMatter({title: {toString: () => "foo"}}), {title: "foo"});
+  });
+  it("coerces the toc to {show?, label?}", () => {
+    assert.deepStrictEqual(normalizeFrontMatter({toc: false}), {toc: {show: false}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: true}), {toc: {show: true}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: null}), {toc: {show: false}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: ""}), {toc: {show: false}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: 42}), {toc: {show: true}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: {}}), {toc: {}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: {show: 1}}), {toc: {show: true}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: {show: 0}}), {toc: {show: false}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: {show: null}}), {toc: {show: false}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: {show: undefined}}), {toc: {}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: {label: null}}), {toc: {label: "null"}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: {label: false}}), {toc: {label: "false"}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: {label: 42}}), {toc: {label: "42"}});
+    assert.deepStrictEqual(normalizeFrontMatter({toc: {label: {toString: () => "foo"}}}), {toc: {label: "foo"}});
+  });
+  it("coerces index to a boolean", () => {
+    assert.deepStrictEqual(normalizeFrontMatter({index: undefined}), {});
+    assert.deepStrictEqual(normalizeFrontMatter({index: null}), {index: false});
+    assert.deepStrictEqual(normalizeFrontMatter({index: 0}), {index: false});
+    assert.deepStrictEqual(normalizeFrontMatter({index: 1}), {index: true});
+    assert.deepStrictEqual(normalizeFrontMatter({index: true}), {index: true});
+    assert.deepStrictEqual(normalizeFrontMatter({index: false}), {index: false});
+  });
+  it("coerces sidebar to a boolean", () => {
+    assert.deepStrictEqual(normalizeFrontMatter({sidebar: undefined}), {});
+    assert.deepStrictEqual(normalizeFrontMatter({sidebar: null}), {sidebar: false});
+    assert.deepStrictEqual(normalizeFrontMatter({sidebar: 0}), {sidebar: false});
+    assert.deepStrictEqual(normalizeFrontMatter({sidebar: 1}), {sidebar: true});
+    assert.deepStrictEqual(normalizeFrontMatter({sidebar: true}), {sidebar: true});
+    assert.deepStrictEqual(normalizeFrontMatter({sidebar: false}), {sidebar: false});
+  });
+  it("coerces draft to a boolean", () => {
+    assert.deepStrictEqual(normalizeFrontMatter({draft: undefined}), {});
+    assert.deepStrictEqual(normalizeFrontMatter({draft: null}), {draft: false});
+    assert.deepStrictEqual(normalizeFrontMatter({draft: 0}), {draft: false});
+    assert.deepStrictEqual(normalizeFrontMatter({draft: 1}), {draft: true});
+    assert.deepStrictEqual(normalizeFrontMatter({draft: true}), {draft: true});
+    assert.deepStrictEqual(normalizeFrontMatter({draft: false}), {draft: false});
+  });
+  it("coerces keywords to an array of strings", () => {
+    assert.deepStrictEqual(normalizeFrontMatter({keywords: undefined}), {});
+    assert.deepStrictEqual(normalizeFrontMatter({keywords: null}), {keywords: []});
+    assert.deepStrictEqual(normalizeFrontMatter({keywords: []}), {keywords: []});
+    assert.deepStrictEqual(normalizeFrontMatter({keywords: [1, 2]}), {keywords: ["1", "2"]});
+    assert.deepStrictEqual(normalizeFrontMatter({keywords: "test"}), {keywords: ["test"]});
+    assert.deepStrictEqual(normalizeFrontMatter({keywords: ""}), {keywords: [""]});
+    assert.deepStrictEqual(normalizeFrontMatter({keywords: "foo, bar"}), {keywords: ["foo, bar"]});
+    assert.deepStrictEqual(normalizeFrontMatter({keywords: [1, "foo"]}), {keywords: ["1", "foo"]});
+    assert.deepStrictEqual(normalizeFrontMatter({keywords: new Set([1, "foo"])}), {keywords: ["1", "foo"]});
+  });
+  it("coerces sql to a Record<string, string>", () => {
+    assert.deepStrictEqual(normalizeFrontMatter({sql: undefined}), {});
+    assert.deepStrictEqual(normalizeFrontMatter({sql: null}), {sql: {}});
+    assert.deepStrictEqual(normalizeFrontMatter({sql: 0}), {sql: {}});
+    assert.deepStrictEqual(normalizeFrontMatter({sql: 1}), {sql: {}});
+    assert.deepStrictEqual(normalizeFrontMatter({sql: false}), {sql: {}});
+    assert.deepStrictEqual(normalizeFrontMatter({sql: {foo: 1}}), {sql: {foo: "1"}});
+    assert.deepStrictEqual(normalizeFrontMatter({sql: {foo: null}}), {sql: {foo: "null"}});
+    assert.deepStrictEqual(normalizeFrontMatter({sql: {foo: "bar"}}), {sql: {foo: "bar"}});
+    assert.deepStrictEqual(normalizeFrontMatter({sql: {foo: []}}), {sql: {foo: ""}});
+    assert.deepStrictEqual(normalizeFrontMatter({sql: {foo: {toString: () => "bar"}}}), {sql: {foo: "bar"}});
+  });
+  it("ignores unknown properties", () => {
+    assert.deepStrictEqual(normalizeFrontMatter({foo: 42}), {});
+  });
+});

test/input/yaml-frontmatter.md

@@ -1,6 +1,7 @@
 ---
 title: YAML
-style:
+style: custom.css
+keywords:
   - one
   - two
 ---