From 1c7402ae47ce0c3b7244872f49231ee7e80703b6 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Tue, 7 Nov 2023 13:53:25 -0800 Subject: [PATCH 1/2] dot, tex, and mermaid blocks (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dot & tex blocks * add tests * getLiveSource * mermaid --------- Co-authored-by: Philippe Rivière --- public/client.js | 53 +++++++++++ src/markdown.ts | 27 ++++-- src/render.ts | 2 + src/tag.js | 170 ++++++++++++++++++++++++++++++++++ test/input/dot-graphviz.md | 13 +++ test/input/mermaid.md | 7 ++ test/input/tex-block.md | 3 + test/output/dot-graphviz.html | 1 + test/output/dot-graphviz.json | 27 ++++++ test/output/mermaid.html | 1 + test/output/mermaid.json | 27 ++++++ test/output/tex-block.html | 1 + test/output/tex-block.json | 27 ++++++ test/tag-test.js | 17 ++++ 14 files changed, 369 insertions(+), 7 deletions(-) create mode 100644 src/tag.js create mode 100644 test/input/dot-graphviz.md create mode 100644 test/input/mermaid.md create mode 100644 test/input/tex-block.md create mode 100644 test/output/dot-graphviz.html create mode 100644 test/output/dot-graphviz.json create mode 100644 test/output/mermaid.html create mode 100644 test/output/mermaid.json create mode 100644 test/output/tex-block.html create mode 100644 test/output/tex-block.json create mode 100644 test/tag-test.js diff --git a/public/client.js b/public/client.js index 35c2f75f9..70afb0d48 100644 --- a/public/client.js +++ b/public/client.js @@ -74,7 +74,60 @@ function recommendedLibraries() { link.href = "https://cdn.jsdelivr.net/gh/observablehq/inputs/src/style.css"; document.head.append(link); return inputs; + }, + dot, + mermaid + }; +} + +// TODO Incorporate this into the standard library. +async function dot() { + const {instance} = await import("https://cdn.jsdelivr.net/npm/@viz-js/viz/+esm"); + const viz = await instance(); + return function dot(strings) { + let string = strings[0] + ""; + let i = 0; + let n = arguments.length; + while (++i < n) string += arguments[i] + "" + strings[i]; + const svg = viz.renderSVGElement(string, { + graphAttributes: { + bgcolor: "none" + }, + nodeAttributes: { + color: "#00000101", + fontcolor: "#00000101", + fontname: "var(--sans-serif)", + fontsize: "12" + }, + edgeAttributes: { + color: "#00000101" + } + }); + for (const e of svg.querySelectorAll("[stroke='#000001'][stroke-opacity='0.003922']")) { + e.setAttribute("stroke", "currentColor"); + e.removeAttribute("stroke-opacity"); + } + for (const e of svg.querySelectorAll("[fill='#000001'][fill-opacity='0.003922']")) { + e.setAttribute("fill", "currentColor"); + e.removeAttribute("fill-opacity"); } + svg.remove(); + svg.style = "max-width: 100%; height: auto;"; + return svg; + }; +} + +// TODO Incorporate this into the standard library. +async function mermaid() { + let nextId = 0; + const {default: mer} = await import("https://cdn.jsdelivr.net/npm/mermaid/+esm"); + mer.initialize({startOnLoad: false, securityLevel: "loose", theme: "neutral"}); + return async function mermaid() { + const div = document.createElement("div"); + div.innerHTML = (await mer.render(`mermaid-${++nextId}`, String.raw.apply(String, arguments))).svg; + const svg = div.firstChild; + svg.remove(); + return svg; }; } diff --git a/src/markdown.ts b/src/markdown.ts index 656e22ac6..964f288b3 100644 --- a/src/markdown.ts +++ b/src/markdown.ts @@ -13,6 +13,7 @@ import {readFile} from "node:fs/promises"; import {pathFromRoot} from "./files.js"; import {computeHash} from "./hash.js"; import {transpileJavaScript, type FileReference, type ImportReference, type Transpile} from "./javascript.js"; +import {transpileTag} from "./tag.js"; export interface ReadMarkdownResult { contents: string; @@ -77,26 +78,38 @@ function uniqueCodeId(context: ParseContext, content: string): string { return id; } +function getLiveSource(content, language, option) { + return option === "no-run" + ? undefined + : language === "js" + ? content + : language === "tex" + ? transpileTag(content, "tex.block", true) + : language === "dot" + ? transpileTag(content, "dot", false) + : language === "mermaid" + ? transpileTag(content, "await mermaid", false) + : undefined; +} + function makeFenceRenderer(root: string, baseRenderer: RenderRule): RenderRule { return (tokens, idx, options, context: ParseContext, self) => { const token = tokens[idx]; const [language, option] = token.info.split(" "); let result = ""; let count = 0; - if (language === "js" && option !== "no-run") { + const source = getLiveSource(token.content, language, option); + if (source != null) { const id = uniqueCodeId(context, token.content); - const transpile = transpileJavaScript(token.content, { - id, - root, - sourceLine: context.startLine + context.currentLine - }); + const sourceLine = context.startLine + context.currentLine; + const transpile = transpileJavaScript(source, {id, root, sourceLine}); extendPiece(context, {code: [transpile]}); if (transpile.files) context.files.push(...transpile.files); if (transpile.imports) context.imports.push(...transpile.imports); result += `
\n`; count++; } - if (language !== "js" || option === "show" || option === "no-run") { + if (source == null || option === "show") { result += baseRenderer(tokens, idx, options, context, self); count++; } diff --git a/src/render.ts b/src/render.ts index 7611ffbd9..2f89016ad 100644 --- a/src/render.ts +++ b/src/render.ts @@ -119,6 +119,8 @@ function getImportPreloads(parseResult: ParseResult): Iterable { if (inputs.has("Plot")) specifiers.add("npm:@observablehq/plot"); if (inputs.has("htl") || inputs.has("html") || inputs.has("svg") || inputs.has("Inputs")) specifiers.add("npm:htl"); if (inputs.has("Inputs")) specifiers.add("npm:@observablehq/inputs"); + if (inputs.has("dot")) specifiers.add("npm:@viz-js/viz"); + if (inputs.has("mermaid")) specifiers.add("npm:mermaid").add("npm:d3"); const preloads: string[] = []; for (const specifier of specifiers) { const resolved = resolveImport(specifier); diff --git a/src/tag.js b/src/tag.js new file mode 100644 index 000000000..9b6a55084 --- /dev/null +++ b/src/tag.js @@ -0,0 +1,170 @@ +import {Parser, TokContext, tokTypes as tt} from "acorn"; +import {Sourcemap} from "./sourcemap.js"; + +const CODE_DOLLAR = 36; +const CODE_BACKSLASH = 92; +const CODE_BACKTICK = 96; +const CODE_BRACEL = 123; + +export function transpileTag(input, tag = "", raw = false) { + const options = {ecmaVersion: 13, sourceType: "module"}; + const template = TemplateParser.parse(input, options); + const source = new Sourcemap(input); + escapeTemplateElements(source, template, raw); + source.insertLeft(template.start, tag + "`"); + source.insertRight(template.end, "`"); + return String(source); +} + +class TemplateParser extends Parser { + constructor(...args) { + super(...args); + // Initialize the type so that we're inside a backQuote + this.type = tt.backQuote; + this.exprAllowed = false; + } + initialContext() { + // Provide our custom TokContext + return [o_tmpl]; + } + parseTopLevel(body) { + // Fix for nextToken calling finishToken(tt.eof) + if (this.type === tt.eof) this.value = ""; + // Based on acorn.Parser.parseTemplate + const isTagged = true; + body.expressions = []; + let curElt = this.parseTemplateElement({isTagged}); + body.quasis = [curElt]; + while (this.type !== tt.eof) { + this.expect(tt.dollarBraceL); + body.expressions.push(this.parseExpression()); + this.expect(tt.braceR); + body.quasis.push((curElt = this.parseTemplateElement({isTagged}))); + } + curElt.tail = true; + this.next(); + this.finishNode(body, "TemplateLiteral"); + this.expect(tt.eof); + return body; + } +} + +// Based on acorn’s q_tmpl. We will use this to initialize the +// parser context so our `readTemplateToken` override is called. +// `readTemplateToken` is based on acorn's `readTmplToken` which +// is used inside template literals. Our version allows backQuotes. +const o_tmpl = new TokContext( + "`", // token + true, // isExpr + true, // preserveSpace + (parser) => readTemplateToken.call(parser) // override +); + +// This is our custom override for parsing a template that allows backticks. +// Based on acorn's readInvalidTemplateToken. +function readTemplateToken() { + out: for (; this.pos < this.input.length; this.pos++) { + switch (this.input.charCodeAt(this.pos)) { + case CODE_BACKSLASH: { + if (this.pos < this.input.length - 1) ++this.pos; // not a terminal slash + break; + } + case CODE_DOLLAR: { + if (this.input.charCodeAt(this.pos + 1) === CODE_BRACEL) { + if (this.pos === this.start && this.type === tt.invalidTemplate) { + this.pos += 2; + return this.finishToken(tt.dollarBraceL); + } + break out; + } + break; + } + } + } + return this.finishToken(tt.invalidTemplate, this.input.slice(this.start, this.pos)); +} + +function escapeTemplateElements(source, {quasis}, raw) { + for (const quasi of quasis) { + if (raw) { + interpolateBacktick(source, quasi); + } else { + escapeBacktick(source, quasi); + escapeBackslash(source, quasi); + } + } + if (raw) interpolateTerminalBackslash(source); +} + +function escapeBacktick(source, {start, end}) { + const input = source._input; + for (let i = start; i < end; ++i) { + if (input.charCodeAt(i) === CODE_BACKTICK) { + source.insertRight(i, "\\"); + } + } +} + +function interpolateBacktick(source, {start, end}) { + const input = source._input; + let oddBackslashes = false; + for (let i = start; i < end; ++i) { + switch (input.charCodeAt(i)) { + case CODE_BACKSLASH: { + oddBackslashes = !oddBackslashes; + break; + } + case CODE_BACKTICK: { + if (!oddBackslashes) { + let j = i + 1; + while (j < end && input.charCodeAt(j) === CODE_BACKTICK) ++j; + source.replaceRight(i, j, `\${'${"`".repeat(j - i)}'}`); + i = j - 1; + } + // fall through + } + default: { + oddBackslashes = false; + break; + } + } + } +} + +function escapeBackslash(source, {start, end}) { + const input = source._input; + let afterDollar = false; + let oddBackslashes = false; + for (let i = start; i < end; ++i) { + switch (input.charCodeAt(i)) { + case CODE_DOLLAR: { + afterDollar = true; + oddBackslashes = false; + break; + } + case CODE_BACKSLASH: { + oddBackslashes = !oddBackslashes; + if (afterDollar && input.charCodeAt(i + 1) === CODE_BRACEL) continue; + if (oddBackslashes && input.charCodeAt(i + 1) === CODE_DOLLAR && input.charCodeAt(i + 2) === CODE_BRACEL) + continue; + source.insertRight(i, "\\"); + break; + } + default: { + afterDollar = false; + oddBackslashes = false; + break; + } + } + } +} + +function interpolateTerminalBackslash(source) { + const input = source._input; + let oddBackslashes = false; + for (let i = input.length - 1; i >= 0; i--) { + if (input.charCodeAt(i) === CODE_BACKSLASH) oddBackslashes = !oddBackslashes; + else break; + } + if (oddBackslashes) source.replaceRight(input.length - 1, input.length, "${'\\\\'}"); +} diff --git a/test/input/dot-graphviz.md b/test/input/dot-graphviz.md new file mode 100644 index 000000000..ed5dbd469 --- /dev/null +++ b/test/input/dot-graphviz.md @@ -0,0 +1,13 @@ +```dot +digraph D { + + A [shape=diamond] + B [shape=box] + C [shape=circle] + + A -> B [style=dashed] + A -> C + A -> D [penwidth=5, arrowhead=none] + +} +``` \ No newline at end of file diff --git a/test/input/mermaid.md b/test/input/mermaid.md new file mode 100644 index 000000000..780c06f53 --- /dev/null +++ b/test/input/mermaid.md @@ -0,0 +1,7 @@ +```mermaid +graph TD; + A-->B; + A-->C; + B-->D; + C-->D; +``` diff --git a/test/input/tex-block.md b/test/input/tex-block.md new file mode 100644 index 000000000..bc3ff3b33 --- /dev/null +++ b/test/input/tex-block.md @@ -0,0 +1,3 @@ +```tex +\int_0^1 (x + y)dx +``` diff --git a/test/output/dot-graphviz.html b/test/output/dot-graphviz.html new file mode 100644 index 000000000..ab530f313 --- /dev/null +++ b/test/output/dot-graphviz.html @@ -0,0 +1 @@ +
diff --git a/test/output/dot-graphviz.json b/test/output/dot-graphviz.json new file mode 100644 index 000000000..25de78a28 --- /dev/null +++ b/test/output/dot-graphviz.json @@ -0,0 +1,27 @@ +{ + "data": null, + "title": null, + "files": [], + "imports": [], + "pieces": [ + { + "type": "html", + "id": "", + "cellIds": [ + "391a3a26" + ], + "html": "
\n" + } + ], + "cells": [ + { + "type": "cell", + "id": "391a3a26", + "inputs": [ + "dot", + "display" + ], + "body": "(dot,display) => {\ndisplay((\ndot`digraph D {\n\n A [shape=diamond]\n B [shape=box]\n C [shape=circle]\n\n A -> B [style=dashed]\n A -> C\n A -> D [penwidth=5, arrowhead=none]\n\n}\n`\n))\n}" + } + ] +} \ No newline at end of file diff --git a/test/output/mermaid.html b/test/output/mermaid.html new file mode 100644 index 000000000..6f875a10d --- /dev/null +++ b/test/output/mermaid.html @@ -0,0 +1 @@ +
diff --git a/test/output/mermaid.json b/test/output/mermaid.json new file mode 100644 index 000000000..4e17658f2 --- /dev/null +++ b/test/output/mermaid.json @@ -0,0 +1,27 @@ +{ + "data": null, + "title": null, + "files": [], + "imports": [], + "pieces": [ + { + "type": "html", + "id": "", + "cellIds": [ + "1ffa6d4f" + ], + "html": "
\n" + } + ], + "cells": [ + { + "type": "cell", + "id": "1ffa6d4f", + "inputs": [ + "mermaid", + "display" + ], + "body": "async (mermaid,display) => {\ndisplay((\nawait mermaid`graph TD;\n A-->B;\n A-->C;\n B-->D;\n C-->D;\n`\n))\n}" + } + ] +} \ No newline at end of file diff --git a/test/output/tex-block.html b/test/output/tex-block.html new file mode 100644 index 000000000..fe5200008 --- /dev/null +++ b/test/output/tex-block.html @@ -0,0 +1 @@ +
diff --git a/test/output/tex-block.json b/test/output/tex-block.json new file mode 100644 index 000000000..4fe25aaba --- /dev/null +++ b/test/output/tex-block.json @@ -0,0 +1,27 @@ +{ + "data": null, + "title": null, + "files": [], + "imports": [], + "pieces": [ + { + "type": "html", + "id": "", + "cellIds": [ + "5ceee90d" + ], + "html": "
\n" + } + ], + "cells": [ + { + "type": "cell", + "id": "5ceee90d", + "inputs": [ + "tex", + "display" + ], + "body": "(tex,display) => {\ndisplay((\ntex.block`\\int_0^1 (x + y)dx\n`\n))\n}" + } + ] +} \ No newline at end of file diff --git a/test/tag-test.js b/test/tag-test.js new file mode 100644 index 000000000..6ab3cc8e0 --- /dev/null +++ b/test/tag-test.js @@ -0,0 +1,17 @@ +import assert from "node:assert"; +import {transpileTag} from "../src/tag.js"; + +describe("transpileTag(input)", () => { + it("bare template literal", () => { + assert.strictEqual(transpileTag("1 + 2"), "`1 + 2`"); + }); + it("tagged template literal", () => { + assert.strictEqual(transpileTag("1 + 2", "tag"), "tag`1 + 2`"); + }); + it("embedded expression", () => { + assert.strictEqual(transpileTag("1 + ${2}"), "`1 + ${2}`"); + }); + it("escaped embedded expression", () => { + assert.strictEqual(transpileTag("1 + $\\{2}"), "`1 + $\\{2}`"); + }); +}); From d8f57101cddaece49ebe9d31d9a7ff01b6d46d10 Mon Sep 17 00:00:00 2001 From: Cindy <37343722+cinxmo@users.noreply.github.com> Date: Tue, 7 Nov 2023 18:45:36 -0500 Subject: [PATCH 2/2] Support relative imports (#40) Co-authored-by: Mike Bostock --- src/build.ts | 2 +- src/javascript.ts | 28 +++++--- src/javascript/features.js | 28 +++----- src/javascript/fetches.js | 6 +- src/javascript/imports.ts | 56 +++++++++++----- src/markdown.ts | 22 +++--- src/navigation.ts | 2 +- src/preview.ts | 4 +- src/render.ts | 17 ++--- test/input/imports/bar.js | 3 + test/input/{ => imports}/dynamic-import.js | 0 test/input/imports/illegal-import.js | 3 + test/input/imports/other/foo.js | 3 + test/input/{ => imports}/static-import-npm.js | 0 test/input/{ => imports}/static-import.js | 0 .../imports/transitive-dynamic-import.js | 1 + .../input/imports/transitive-static-import.js | 1 + test/isLocalImport-test.ts | 58 ++++++++++++++++ test/javascript-test.ts | 67 ++++++++++++++++--- test/markdown-test.ts | 3 +- test/output/imports/dynamic-import.js | 4 ++ test/output/imports/illegal-import.js | 6 ++ .../output/{ => imports}/static-import-npm.js | 0 test/output/{ => imports}/static-import.js | 2 +- .../imports/transitive-dynamic-import.js | 4 ++ .../imports/transitive-static-import.js | 4 ++ test/output/local-fetch.json | 4 +- 27 files changed, 242 insertions(+), 86 deletions(-) create mode 100644 test/input/imports/bar.js rename test/input/{ => imports}/dynamic-import.js (100%) create mode 100644 test/input/imports/illegal-import.js create mode 100644 test/input/imports/other/foo.js rename test/input/{ => imports}/static-import-npm.js (100%) rename test/input/{ => imports}/static-import.js (100%) create mode 100644 test/input/imports/transitive-dynamic-import.js create mode 100644 test/input/imports/transitive-static-import.js create mode 100644 test/isLocalImport-test.ts create mode 100644 test/output/imports/dynamic-import.js create mode 100644 test/output/imports/illegal-import.js rename test/output/{ => imports}/static-import-npm.js (100%) rename test/output/{ => imports}/static-import.js (62%) create mode 100644 test/output/imports/transitive-dynamic-import.js create mode 100644 test/output/imports/transitive-static-import.js diff --git a/src/build.ts b/src/build.ts index 7eff211b6..e69ccc6ae 100644 --- a/src/build.ts +++ b/src/build.ts @@ -34,7 +34,7 @@ async function build(context: CommandContext) { pages, resolver }); - files.push(...render.files.map((f) => join(sourceFile, "..", f.name))); + files.push(...render.files.map((f) => f.name)); await prepareOutput(outputPath); await writeFile(outputPath, render.html); } diff --git a/src/javascript.ts b/src/javascript.ts index 0c713a249..ce6ce6653 100644 --- a/src/javascript.ts +++ b/src/javascript.ts @@ -1,4 +1,4 @@ -import {Parser, tokTypes, type Options} from "acorn"; +import {Parser, tokTypes, type Node, type Options} from "acorn"; import mime from "mime"; import {findAwaits} from "./javascript/awaits.js"; import {findDeclarations} from "./javascript/declarations.js"; @@ -20,6 +20,7 @@ export interface FileReference { export interface ImportReference { name: string; + type: "global" | "local"; } export interface Transpile { @@ -36,13 +37,14 @@ export interface Transpile { export interface ParseOptions { id: string; root: string; + sourcePath: string; inline?: boolean; sourceLine?: number; globals?: Set; } export function transpileJavaScript(input: string, options: ParseOptions): Transpile { - const {id} = options; + const {id, root, sourcePath} = options; try { const node = parseJavaScript(input, options); const databases = node.features.filter((f) => f.type === "DatabaseClient").map((f) => ({name: f.name})); @@ -57,8 +59,8 @@ export function transpileJavaScript(input: string, options: ParseOptions): Trans output.insertRight(input.length, "\n))"); inputs.push("display"); } - rewriteImports(output, node); - rewriteFetches(output, node); + rewriteImports(output, node, root, sourcePath); + rewriteFetches(output, node, root, sourcePath); return { id, ...(inputs.length ? {inputs} : null), @@ -99,8 +101,18 @@ function trim(output: Sourcemap, input: string): void { export const parseOptions: Options = {ecmaVersion: 13, sourceType: "module"}; -export function parseJavaScript(input: string, options: ParseOptions) { - const {globals = defaultGlobals, inline = false, root} = options; +export interface JavaScriptNode { + body: Node; + declarations: {name: string}[] | null; + references: {name: string}[]; + features: {type: unknown; name: string}[]; + imports: {type: "global" | "local"; name: string}[]; + expression: boolean; + async: boolean; +} + +function parseJavaScript(input: string, options: ParseOptions): JavaScriptNode { + const {globals = defaultGlobals, inline = false, root, sourcePath} = options; // First attempt to parse as an expression; if this fails, parse as a program. let expression = maybeParseExpression(input, parseOptions); if (expression?.type === "ClassExpression" && expression.id) expression = null; // treat named class as program @@ -109,8 +121,8 @@ export function parseJavaScript(input: string, options: ParseOptions) { const body = expression ?? (Parser.parse(input, parseOptions) as any); const references = findReferences(body, globals, input); const declarations = expression ? null : findDeclarations(body, globals, input); - const features = findFeatures(body, references, input); - const imports = findImports(body, root); + const {imports, features: importFeatures} = findImports(body, root, sourcePath); + const features = [...importFeatures, ...findFeatures(body, root, sourcePath, references, input)]; return { body, declarations, diff --git a/src/javascript/features.js b/src/javascript/features.js index 007b1277e..acd2e1baf 100644 --- a/src/javascript/features.js +++ b/src/javascript/features.js @@ -1,7 +1,9 @@ import {simple} from "acorn-walk"; import {syntaxError} from "./syntaxError.js"; +import {isLocalImport} from "./imports.ts"; +import {dirname, join} from "node:path"; -export function findFeatures(node, references, input) { +export function findFeatures(node, root, sourcePath, references, input) { const features = []; simple(node, { @@ -11,10 +13,9 @@ export function findFeatures(node, references, input) { arguments: args, arguments: [arg] } = node; - // Promote fetches with static literals to file attachment references. - if (isLocalFetch(node, references)) { - features.push({type: "FileAttachment", name: getStringLiteralValue(arg)}); + if (isLocalFetch(node, references, root, sourcePath)) { + features.push({type: "FileAttachment", name: join(dirname(sourcePath), getStringLiteralValue(arg))}); return; } @@ -33,27 +34,14 @@ export function findFeatures(node, references, input) { if (args.length !== 1 || !isStringLiteral(arg)) { throw syntaxError(`${callee.name} requires a single literal string argument`, node, input); } - features.push({type: callee.name, name: getStringLiteralValue(arg)}); - }, - // Promote dynamic imports with static literals to file attachment references. - ImportExpression: findImport, - ImportDeclaration: findImport - }); - - function findImport(node) { - if (isStringLiteral(node.source)) { - const value = getStringLiteralValue(node.source); - if (value.startsWith("./")) { - features.push({type: "FileAttachment", name: value}); - } } - } + }); return features; } -export function isLocalFetch(node, references) { +export function isLocalFetch(node, references, root, sourcePath) { if (node.type !== "CallExpression") return false; const { callee, @@ -65,7 +53,7 @@ export function isLocalFetch(node, references) { !references.includes(callee) && arg && isStringLiteral(arg) && - getStringLiteralValue(arg).startsWith("./") + isLocalImport(getStringLiteralValue(arg), root, sourcePath) ); } diff --git a/src/javascript/fetches.js b/src/javascript/fetches.js index 9b2b82c2d..979a6a294 100644 --- a/src/javascript/fetches.js +++ b/src/javascript/fetches.js @@ -1,10 +1,10 @@ import {simple} from "acorn-walk"; import {isLocalFetch} from "./features.js"; -export function rewriteFetches(output, root) { - simple(root.body, { +export function rewriteFetches(output, rootNode, root, sourcePath) { + simple(rootNode.body, { CallExpression(node) { - if (isLocalFetch(node, root.references)) { + if (isLocalFetch(node, rootNode.references, root, sourcePath)) { output.insertLeft(node.arguments[0].start + 3, "_file/"); } } diff --git a/src/javascript/imports.ts b/src/javascript/imports.ts index f6d9fa204..6b0e24ea2 100644 --- a/src/javascript/imports.ts +++ b/src/javascript/imports.ts @@ -1,12 +1,14 @@ import {Parser} from "acorn"; +import type {Node} from "acorn"; import {simple} from "acorn-walk"; import {readFileSync} from "node:fs"; -import {dirname, join, relative, resolve} from "node:path"; -import {parseOptions} from "../javascript.js"; +import {dirname, join} from "node:path"; +import {type JavaScriptNode, parseOptions} from "../javascript.js"; import {getStringLiteralValue, isStringLiteral} from "./features.js"; -export function findImports(body, root) { - const imports: {name: string}[] = []; +export function findImports(body: Node, root: string, sourcePath: string) { + const imports: {name: string; type: "global" | "local"}[] = []; + const features: {name: string; type: string}[] = []; const paths = new Set(); simple(body, { @@ -19,18 +21,24 @@ export function findImports(body, root) { function findImport(node) { if (isStringLiteral(node.source)) { const value = getStringLiteralValue(node.source); - if (value.startsWith("./")) findLocalImports(join(root, value)); - imports.push({name: value}); + if (isLocalImport(value, root, sourcePath)) { + findLocalImports(join(dirname(sourcePath), value)); + } else { + imports.push({name: value, type: "global"}); + } } } // If this is an import of a local ES module, recursively parse the module to // find transitive imports. + // path is the full URI path without /_file function findLocalImports(path) { if (paths.has(path)) return; paths.add(path); + imports.push({type: "local", name: path}); + features.push({type: "FileAttachment", name: path}); try { - const input = readFileSync(path, "utf-8"); + const input = readFileSync(join(root, path), "utf-8"); const program = Parser.parse(input, parseOptions); simple(program, { ImportDeclaration: findLocalImport, @@ -44,38 +52,41 @@ export function findImports(body, root) { function findLocalImport(node) { if (isStringLiteral(node.source)) { const value = getStringLiteralValue(node.source); - if (value.startsWith("./")) { - const subpath = resolve(dirname(path), value); + if (isLocalImport(value, root, path)) { + const subpath = join(dirname(path), value); findLocalImports(subpath); - imports.push({name: `./${relative(root, subpath)}`}); } else { - imports.push({name: value}); + imports.push({name: value, type: "global"}); + // non-local imports don't need to be promoted to file attachments } } } } - return imports; + return {imports, features}; } // TODO parallelize multiple static imports -// TODO need to know the local path of the importing notebook; this assumes it’s in the root -export function rewriteImports(output, root) { - simple(root.body, { +export function rewriteImports(output: any, rootNode: JavaScriptNode, root: string, sourcePath: string) { + simple(rootNode.body, { ImportExpression(node: any) { if (isStringLiteral(node.source)) { const value = getStringLiteralValue(node.source); output.replaceLeft( node.source.start, node.source.end, - JSON.stringify(value.startsWith("./") ? `/_file/${value.slice(2)}` : resolveImport(value)) + JSON.stringify( + isLocalImport(value, root, sourcePath) + ? join("/_file/", join(dirname(sourcePath), value)) + : resolveImport(value) + ) ); } }, ImportDeclaration(node: any) { if (isStringLiteral(node.source)) { const value = getStringLiteralValue(node.source); - root.async = true; + rootNode.async = true; output.replaceLeft( node.start, node.end, @@ -86,7 +97,9 @@ export function rewriteImports(output, root) { ? node.specifiers.find(isNamespaceSpecifier).local.name : "{}" } = await import(${JSON.stringify( - value.startsWith("./") ? `/_file/${value.slice(2)}` : resolveImport(value) + isLocalImport(value, root, sourcePath) + ? join("/_file/", join(dirname(sourcePath), value)) + : resolveImport(value) )});` ); } @@ -102,6 +115,13 @@ function rewriteImportSpecifier(node) { : `${node.imported.name}: ${node.local.name}`; } +export function isLocalImport(value: string, root: string, sourcePath: string): boolean { + return ( + ["./", "../", "/"].some((prefix) => value.startsWith(prefix)) && + join(root + "/", dirname(sourcePath), value).startsWith(root) + ); +} + function isNamespaceSpecifier(node) { return node.type === "ImportNamespaceSpecifier"; } diff --git a/src/markdown.ts b/src/markdown.ts index 964f288b3..6220b5f4a 100644 --- a/src/markdown.ts +++ b/src/markdown.ts @@ -92,7 +92,7 @@ function getLiveSource(content, language, option) { : undefined; } -function makeFenceRenderer(root: string, baseRenderer: RenderRule): RenderRule { +function makeFenceRenderer(root: string, baseRenderer: RenderRule, sourcePath: string): RenderRule { return (tokens, idx, options, context: ParseContext, self) => { const token = tokens[idx]; const [language, option] = token.info.split(" "); @@ -102,7 +102,12 @@ function makeFenceRenderer(root: string, baseRenderer: RenderRule): RenderRule { if (source != null) { const id = uniqueCodeId(context, token.content); const sourceLine = context.startLine + context.currentLine; - const transpile = transpileJavaScript(source, {id, root, sourceLine}); + const transpile = transpileJavaScript(source, { + id, + root, + sourcePath, + sourceLine + }); extendPiece(context, {code: [transpile]}); if (transpile.files) context.files.push(...transpile.files); if (transpile.imports) context.imports.push(...transpile.imports); @@ -248,13 +253,14 @@ const transformPlaceholderCore: RuleCore = (state) => { state.tokens = output; }; -function makePlaceholderRenderer(root: string): RenderRule { +function makePlaceholderRenderer(root: string, sourcePath: string): RenderRule { return (tokens, idx, options, context: ParseContext) => { const id = uniqueCodeId(context, tokens[idx].content); const token = tokens[idx]; const transpile = transpileJavaScript(token.content, { id, root, + sourcePath, inline: true, sourceLine: context.startLine + context.currentLine }); @@ -344,7 +350,7 @@ function toParseCells(pieces: RenderPiece[]): CellPiece[] { return cellPieces; } -export function parseMarkdown(source: string, root: string): ParseResult { +export function parseMarkdown(source: string, root: string, sourcePath: string): ParseResult { const parts = matter(source); // TODO: We need to know what line in the source the markdown starts on and pass that // as startLine in the parse context below. @@ -364,8 +370,8 @@ export function parseMarkdown(source: string, root: string): ParseResult { md.use(MarkdownItAnchor, {permalink: MarkdownItAnchor.permalink.headerLink({class: "observablehq-header-anchor"})}); md.inline.ruler.push("placeholder", transformPlaceholderInline); md.core.ruler.before("linkify", "placeholder", transformPlaceholderCore); - md.renderer.rules.placeholder = makePlaceholderRenderer(root); - md.renderer.rules.fence = makeFenceRenderer(root, md.renderer.rules.fence!); + md.renderer.rules.placeholder = makePlaceholderRenderer(root, sourcePath); + md.renderer.rules.fence = makeFenceRenderer(root, md.renderer.rules.fence!, sourcePath); md.renderer.rules.softbreak = makeSoftbreakRenderer(md.renderer.rules.softbreak!); md.renderer.render = renderIntoPieces(md.renderer, root); const context: ParseContext = {files: [], imports: [], pieces: [], startLine: 0, currentLine: 0}; @@ -454,6 +460,6 @@ export function diffMarkdown({parse: prevParse}: ReadMarkdownResult, {parse: nex } export async function readMarkdown(path: string, root: string): Promise { - const contents = await readFile(path, "utf-8"); - return {contents, parse: parseMarkdown(contents, root), hash: computeHash(contents)}; + const contents = await readFile(pathFromRoot(path, root)!, "utf-8"); + return {contents, parse: parseMarkdown(contents, root, path), hash: computeHash(contents)}; } diff --git a/src/navigation.ts b/src/navigation.ts index 8f2f153e4..41030b18b 100644 --- a/src/navigation.ts +++ b/src/navigation.ts @@ -12,7 +12,7 @@ export async function readPages(root: string): Promise CellPiece; } export function renderPreview(source: string, options: RenderOptions): Render { - const parseResult = parseMarkdown(source, options.root); + const parseResult = parseMarkdown(source, options.root, options.path); return { html: render(parseResult, {...options, preview: true, hash: computeHash(source)}), files: parseResult.files, @@ -26,7 +26,7 @@ export function renderPreview(source: string, options: RenderOptions): Render { } export function renderServerless(source: string, options: RenderOptions): Render { - const parseResult = parseMarkdown(source, options.root); + const parseResult = parseMarkdown(source, options.root, options.path); return { html: render(parseResult, options), files: parseResult.files, @@ -59,10 +59,6 @@ ${ } ${Array.from(getImportPreloads(parseResult)) - .concat( - parseResult.imports.filter(({name}) => name.startsWith("./")).map(({name}) => `/_file/${name.slice(2)}`), - parseResult.cells.some((cell) => cell.databases?.length) ? "/_observablehq/database.js" : [] - ) .map((href) => ``) .join("\n")}