robust placeholder

src/markdown.ts

@@ -2,8 +2,6 @@
 import {createHash} from "node:crypto";
 import he from "he";
 import MarkdownIt from "markdown-it";
-import type {RuleCore} from "markdown-it/lib/parser_core.js";
-import type {RuleInline} from "markdown-it/lib/parser_inline.js";
 import type {RenderRule} from "markdown-it/lib/renderer.js";
 import MarkdownItAnchor from "markdown-it-anchor";
 import type {Config} from "./config.js";
@@ -15,6 +13,7 @@ import {parseInfo} from "./info.js";
 import type {JavaScriptNode} from "./javascript/parse.js";
 import {parseJavaScript} from "./javascript/parse.js";
 import {isAssetPath, relativePath} from "./path.js";
+import {parsePlaceholder} from "./placeholder.js";
 import {transpileSql} from "./sql.js";
 import {transpileTag} from "./tag.js";
 import {InvalidThemeError} from "./theme.js";
@@ -36,10 +35,8 @@ export interface MarkdownPage {
   code: MarkdownCode[];
 }
 
-interface ParseContext {
+export interface ParseContext {
   code: MarkdownCode[];
-  startLine: number;
-  currentLine: number;
   path: string;
 }
 
@@ -103,7 +100,6 @@ function makeFenceRenderer(baseRenderer: RenderRule): RenderRule {
       source = isFalse(attributes.run) ? undefined : getLiveSource(token.content, tag, attributes);
       if (source != null) {
         const id = uniqueCodeId(context, source);
-        // TODO const sourceLine = context.startLine + context.currentLine;
         const node = parseJavaScript(source, {path});
         context.code.push({id, node});
         html += `<div class="observablehq observablehq--block">${
@@ -123,159 +119,28 @@ function makeFenceRenderer(baseRenderer: RenderRule): RenderRule {
   };
 }
 
-const CODE_DOLLAR = 36;
-const CODE_BRACEL = 123;
-const CODE_BRACER = 125;
-const CODE_BACKSLASH = 92;
-const CODE_QUOTE = 34;
-const CODE_SINGLE_QUOTE = 39;
-const CODE_BACKTICK = 96;
-
-function parsePlaceholder(content: string, replacer: (i: number, j: number) => void) {
-  let afterDollar = false;
-  for (let j = 0, n = content.length; j < n; ++j) {
-    const cj = content.charCodeAt(j);
-    if (cj === CODE_BACKSLASH) {
-      ++j; // skip next character
-      continue;
-    }
-    if (cj === CODE_DOLLAR) {
-      afterDollar = true;
-      continue;
-    }
-    if (afterDollar) {
-      if (cj === CODE_BRACEL) {
-        let quote = 0; // TODO detect comments, too
-        let braces = 0;
-        let k = j + 1;
-        inner: for (; k < n; ++k) {
-          const ck = content.charCodeAt(k);
-          if (ck === CODE_BACKSLASH) {
-            ++k;
-            continue;
-          }
-          if (quote) {
-            if (ck === quote) quote = 0;
-            continue;
-          }
-          switch (ck) {
-            case CODE_QUOTE:
-            case CODE_SINGLE_QUOTE:
-            case CODE_BACKTICK:
-              quote = ck;
-              break;
-            case CODE_BRACEL:
-              ++braces;
-              break;
-            case CODE_BRACER:
-              if (--braces < 0) {
-                replacer(j - 1, k + 1);
-                break inner;
-              }
-              break;
-          }
-        }
-        j = k;
+function preparePlaceholders(input: string, context: ParseContext): string {
+  const outputs: string[] = [];
+  for (const {type, value} of parsePlaceholder(input)) {
+    if (type === "content") {
+      outputs.push(value);
+    } else if (type === "code") {
+      const id = uniqueCodeId(context, value);
+      try {
+        const node = parseJavaScript(value, {path: context.path, inline: true});
+        context.code.push({id, node});
+        outputs.push(`<o-loading></o-loading><!--:${id}:-->`);
+      } catch (error) {
+        if (!(error instanceof SyntaxError)) throw error;
+        outputs.push(
+          `<span class="observablehq--inspect observablehq--error" style="display: block;">SyntaxError: ${he.escape(
+            error.message
+          )}</span>`
+        );
       }
-      afterDollar = false;
     }
   }
-}
-
-function transformPlaceholderBlock(token) {
-  const input = token.content;
-  if (/^\s*<script[\s>]/.test(input)) return [token]; // ignore <script> elements
-  const output: any[] = [];
-  let i = 0;
-  parsePlaceholder(input, (j, k) => {
-    output.push({...token, level: i > 0 ? token.level + 1 : token.level, content: input.slice(i, j)});
-    output.push({type: "placeholder", level: token.level + 1, content: input.slice(j + 2, k - 1)});
-    i = k;
-  });
-  if (i === 0) return [token];
-  else if (i < input.length) output.push({...token, content: input.slice(i), nesting: -1});
-  return output;
-}
-
-const transformPlaceholderInline: RuleInline = (state, silent) => {
-  if (silent || state.pos + 2 > state.posMax) return false;
-  const marker1 = state.src.charCodeAt(state.pos);
-  const marker2 = state.src.charCodeAt(state.pos + 1);
-  if (!(marker1 === CODE_DOLLAR && marker2 === CODE_BRACEL)) return false;
-  let quote = 0;
-  let braces = 0;
-  for (let pos = state.pos + 2; pos < state.posMax; ++pos) {
-    const code = state.src.charCodeAt(pos);
-    if (code === CODE_BACKSLASH) {
-      ++pos; // skip next character
-      continue;
-    }
-    if (quote) {
-      if (code === quote) quote = 0;
-      continue;
-    }
-    switch (code) {
-      case CODE_QUOTE:
-      case CODE_SINGLE_QUOTE:
-      case CODE_BACKTICK:
-        quote = code;
-        break;
-      case CODE_BRACEL:
-        ++braces;
-        break;
-      case CODE_BRACER:
-        if (--braces < 0) {
-          const token = state.push("placeholder", "", 0);
-          token.content = state.src.slice(state.pos + 2, pos);
-          state.pos = pos + 1;
-          return true;
-        }
-        break;
-    }
-  }
-  return false;
-};
-
-const transformPlaceholderCore: RuleCore = (state) => {
-  const input = state.tokens;
-  const output: any[] = [];
-  for (const token of input) {
-    switch (token.type) {
-      case "html_block":
-        output.push(...transformPlaceholderBlock(token));
-        break;
-      default:
-        output.push(token);
-        break;
-    }
-  }
-  state.tokens = output;
-};
-
-function makePlaceholderRenderer(): RenderRule {
-  return (tokens, idx, options, context: ParseContext) => {
-    const {path} = context;
-    const token = tokens[idx];
-    const id = uniqueCodeId(context, token.content);
-    try {
-      // TODO sourceLine: context.startLine + context.currentLine
-      const node = parseJavaScript(token.content, {path, inline: true});
-      context.code.push({id, node});
-      return `<o-loading></o-loading><!--:${id}:-->`;
-    } catch (error) {
-      if (!(error instanceof SyntaxError)) throw error;
-      return `<span class="observablehq--inspect observablehq--error" style="display: block;">SyntaxError: ${he.escape(
-        error.message
-      )}</span>`;
-    }
-  };
-}
-
-function makeSoftbreakRenderer(baseRenderer: RenderRule): RenderRule {
-  return (tokens, idx, options, context: ParseContext, self) => {
-    context.currentLine++;
-    return baseRenderer(tokens, idx, options, context, self);
-  };
+  return outputs.join("");
 }
 
 export interface ParseOptions {
@@ -302,20 +167,16 @@ export function createMarkdownIt({
   const md = MarkdownIt({html: true, linkify, typographer, quotes});
   if (linkify) md.linkify.set({fuzzyLink: false, fuzzyEmail: false});
   md.use(MarkdownItAnchor);
-  md.inline.ruler.push("placeholder", transformPlaceholderInline);
-  md.core.ruler.before("linkify", "placeholder", transformPlaceholderCore);
-  md.renderer.rules.placeholder = makePlaceholderRenderer();
   md.renderer.rules.fence = makeFenceRenderer(md.renderer.rules.fence!);
-  md.renderer.rules.softbreak = makeSoftbreakRenderer(md.renderer.rules.softbreak!);
   return markdownIt === undefined ? md : markdownIt(md);
 }
 
 export function parseMarkdown(input: string, options: ParseOptions): MarkdownPage {
   const {md, path} = options;
   const {content, data} = readFrontMatter(input);
   const code: MarkdownCode[] = [];
-  const context: ParseContext = {code, startLine: 0, currentLine: 0, path};
-  const tokens = md.parse(content, context);
+  const context: ParseContext = {code, path};
+  const tokens = md.parse(preparePlaceholders(content, context), context);
   const body = md.renderer.render(tokens, md.options, context); // Note: mutates code!
   const title = data.title !== undefined ? data.title : findTitle(tokens);
   return {
@@ -336,10 +197,7 @@ export function parseMarkdownMetadata(input: string, options: ParseOptions): Pic
   const {content, data} = readFrontMatter(input);
   return {
     data,
-    title:
-      data.title !== undefined
-        ? data.title
-        : findTitle(md.parse(content, {code: [], startLine: 0, currentLine: 0, path}))
+    title: data.title !== undefined ? data.title : findTitle(md.parse(content, {code: [], path}))
   };
 }