Merge branch 'main' into fil/validate-sql-frontmatter

docs/deploying.md

@@ -0,0 +1,100 @@
+# Deploying
+
+When time comes to share your project, you have many options for deploying it for others to see. Framework is compatible with many static site hosts and automation environments. In this guide we’ll focus on deploying to Observable manually, then with GitHub Actions.
+
+## Manually deploying to Observable
+
+If you don’t already have a project to deploy, you can create one by following [getting-started](./getting-started). First, make sure that your project builds without error:
+
+```sh
+$ npm run build
+```
+
+Once that is done you can deploy to Observable with the command
+
+```sh
+$ npm run deploy
+```
+
+The first time you run this command, you will be prompted for details needed to set up the project on the server, such as the project's _slug_ (which determines its URL), and the access level. If you don’t already have an Observable account or aren’t signed in, this command will also guide you through setting that up.
+
+When the deploy command finishes, it prints a link to observablehq.cloud where you can view your deployed project. If you choose “private” as the access level, you can now share that link with anyone who is a member of  your workspace. If you chose “public”, you can share that link with anyone and they’ll be able to see your Framework project.
+
+<div class="note">The deploy command creates a file at <code>docs/.observablehq/deploy.json</code> with information on where to deploy the project. It is required for automated deploys. You should commit it to git to make it available to GitHub Actions. (If you have configured a source root besides <code>docs/</code>, the file will be placed there instead.)</div>
+
+## Automated deploys to Observable
+
+To set up automatic deploys, we’ll be using [GitHub actions](https://github.com/features/actions). In your git repository, create and commit a file at `.github/workflows/deploy.yml`. Here is a starting example:
+
+```yaml
+name: Deploy
+on:
+  # Run this workflow whenever a new commit is pushed to main.
+  push: {branches: [main]}
+  # Run this workflow once per day, at 10:15 UTC
+  schedule: [{cron: "15 10 * * *"}]
+  # Run this workflow when triggered manually in GitHub's UI.
+  workflow_dispatch: {}
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+      - run: npm ci
+      - run: npm run build
+      - name: Deploy to Observable Cloud
+        # This parameter to `--message` will use the latest commit message
+        run: npm run deploy -- --message "$(git log -1 --pretty=%s)"
+        env:
+          # Authentication information. See below for how to set this up.
+          OBSERVABLE_TOKEN: ${{ secrets.OBSERVABLE_TOKEN }}
+```
+
+When deploying automatically, you won’t be able to login with a browser the way you did for manual deploys. Instead, you will authenticate via the environment variable `OBSERVABLE_TOKEN`, using an API key from Observable.
+
+To create a token, go to https://observablehq.com and open your workspace settings. Choose “API keys”. From there, create a new key, and assign it the "Deploy new versions of projects" scope.
+
+That token is the equivalent of a password giving write access to your hosted project. **Do not commit it to git** (and, if it is exposed in any way, take a minute to revoke it and create a new one instead—or contact support).
+
+To pass this information securely to the Github action (so it can effectively be authorized to deploy the project to Observable), we’ll use GitHub secrets. Sign in to the settings of your GitHub account, and add a secret named `OBSERVABLE_TOKEN`. See [GitHub’s documentation](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions) for more information about secrets.
+
+This `deploy.yml` will automatically build and deploy your project once per day (to keep your data up-to-date), as well as whenever you push a new version of the code to your repository (so you can make changes at any time).
+
+### Caching
+
+If some of your data loaders take a long time to run, or simply don’t need to be updated every time you modify the code, you can set up caching to automatically re-use existing data from previous builds. To do that, add the following steps to your `deploy.yml` before you run `build`:
+
+```yaml
+jobs:
+  deploy:
+    steps:
+      # ...
+      - id: date
+        run: echo "date=$(TZ=America/Los_Angeles date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+      - id: cache-data
+        uses: actions/cache@v4
+        with:
+          path: docs/.observablehq/cache
+          key: data-${{ hashFiles('docs/data/*') }}-${{ steps.date.outputs.date }}
+      - if: steps.cache-data.outputs.cache-hit == 'true'
+        run: find docs/.observablehq/cache -type f -exec touch {} +
+      # ...
+```
+
+This uses one cache per calendar day (in the “America/Los_Angeles” time zone). If you deploy multiple times in a day, the results of your data loaders will be reused on the second and subsequent runs. You can customize the `date` and `cache-data` steps to change the cadence of the caching. For example you could use `date +'%Y-%U'` to cache data for a week or `date +'%Y-%m-%dT%H` to cache it for only an hour.
+
+<div class="note">You’ll need to change the paths used in this config if <code>observablehq.config.js</code> points to a different <code>root</code>.</div>
+
+## Deploying to other services
+
+The output of Observable Framework is set of static files that can be hosted by many services. To build a hostable copy of your project, run:
+
+```sh
+$ npm run build
+```
+
+Then you can upload the contents of your `dist/` directory to your static webhost of choice. Some webhosts may need the `cleanUrls` option <a href="https://github.com/observablehq/framework/releases/tag/v1.3.0" target="_blank" class="observablehq-version-badge" data-version="^1.3.0" title="Added in v1.3.0"></a> set to false in your project configuration file. For details and other options, see [configuration](./config).

docs/getting-started.md

@@ -649,9 +649,9 @@ json.dump(forecast, sys.stdout)
 
 To write the data loader in R, name it <code>forecast.json.R</code>. Or as shell script, <code>forecast.json.sh</code>. You get the idea. See [Data loaders: Routing](./loaders#routing) for more. The beauty of this approach is that you can leverage the strengths (and libraries) of multiple languages, and still get instant updates in the browser as you develop.
 
-### Deploying via GitHub Actions
+### Deploying automatically
 
-You can schedule builds and deploy your project automatically on commit, or on a schedule. See <a href="https://github.com/observablehq/framework/blob/main/.github/workflows/deploy.yml">this documentation site’s deploy.yml</a> for an example.
+You can schedule builds and deploy your project automatically on commit, or on a schedule. See [deploying](./deploying) for more details.
 
 ### Ask for help, or share your feedback
 

observablehq.config.ts

@@ -10,6 +10,7 @@ export default {
     {name: "SQL", path: "/sql"},
     {name: "Themes", path: "/themes"},
     {name: "Configuration", path: "/config"},
+    {name: "Deploying", path: "/deploying"},
     {
       name: "JavaScript",
       open: false,

src/build.ts

@@ -10,7 +10,7 @@ import {transpileModule} from "./javascript/transpile.js";
 import type {Logger, Writer} from "./logger.js";
 import type {MarkdownPage} from "./markdown.js";
 import {parseMarkdown} from "./markdown.js";
-import {populateNpmCache, resolveNpmImport, resolveNpmSpecifier} from "./npm.js";
+import {extractNpmSpecifier, populateNpmCache, resolveNpmImport} from "./npm.js";
 import {isPathImport, relativePath, resolvePath} from "./path.js";
 import {renderPage} from "./render.js";
 import type {Resolvers} from "./resolvers.js";
@@ -176,7 +176,7 @@ export async function build(
   // doesn’t let you pass in a resolver.
   for (const path of globalImports) {
     if (!path.startsWith("/_npm/")) continue; // skip _observablehq
-    effects.output.write(`${faint("copy")} npm:${resolveNpmSpecifier(path)} ${faint("→")} `);
+    effects.output.write(`${faint("copy")} npm:${extractNpmSpecifier(path)} ${faint("→")} `);
     const sourcePath = await populateNpmCache(root, path); // TODO effects
     await effects.copyFile(sourcePath, path);
   }

src/client/preview.js

@@ -1,4 +1,3 @@
-import {registerTable} from "npm:@observablehq/duckdb";
 import {FileAttachment, registerFile} from "npm:@observablehq/stdlib";
 import {main, runtime, undefine} from "./main.js";
 import {enableCopyButtons} from "./pre.js";
@@ -17,7 +16,7 @@ export function open({hash, eval: compile} = {}) {
     send({type: "hello", path: location.pathname, hash});
   };
 
-  socket.onmessage = (event) => {
+  socket.onmessage = async (event) => {
     const message = JSON.parse(event.data);
     console.info("↓", message);
     switch (message.type) {
@@ -84,11 +83,14 @@ export function open({hash, eval: compile} = {}) {
         for (const file of message.files.added) {
           registerFile(file.name, file);
         }
-        for (const name of message.tables.removed) {
-          registerTable(name, null);
-        }
-        for (const table of message.tables.added) {
-          registerTable(table.name, FileAttachment(table.path));
+        if (message.tables.removed.length || message.tables.added.length) {
+          const {registerTable} = await import("npm:@observablehq/duckdb");
+          for (const name of message.tables.removed) {
+            registerTable(name, null);
+          }
+          for (const table of message.tables.added) {
+            registerTable(table.name, FileAttachment(table.path));
+          }
         }
         if (message.tables.removed.length || message.tables.added.length) {
           const sql = main._resolve("sql");

src/html.ts

@@ -3,7 +3,7 @@ import he from "he";
 import hljs from "highlight.js";
 import type {DOMWindow} from "jsdom";
 import {JSDOM, VirtualConsole} from "jsdom";
-import {relativePath, resolveLocalPath} from "./path.js";
+import {isAssetPath, relativePath, resolveLocalPath} from "./path.js";
 
 const ASSET_PROPERTIES: readonly [selector: string, src: string][] = [
   ["a[href][download]", "href"],
@@ -17,55 +17,98 @@ const ASSET_PROPERTIES: readonly [selector: string, src: string][] = [
   ["video[src]", "src"]
 ];
 
-export function isAssetPath(specifier: string): boolean {
-  return !/^(\w+:|#)/.test(specifier);
+export function isJavaScript({type}: HTMLScriptElement): boolean {
+  if (!type) return true;
+  type = type.toLowerCase();
+  return type === "text/javascript" || type === "application/javascript" || type === "module";
 }
 
 export function parseHtml(html: string): DOMWindow {
   return new JSDOM(`<!DOCTYPE html><body>${html}`, {virtualConsole: new VirtualConsole()}).window;
 }
 
-export function findAssets(html: string, path: string): Set<string> {
+interface Assets {
+  files: Set<string>;
+  localImports: Set<string>;
+  globalImports: Set<string>;
+  staticImports: Set<string>;
+}
+
+export function findAssets(html: string, path: string): Assets {
   const {document} = parseHtml(html);
-  const assets = new Set<string>();
+  const files = new Set<string>();
+  const localImports = new Set<string>();
+  const globalImports = new Set<string>();
+  const staticImports = new Set<string>();
 
-  const maybeAsset = (specifier: string): void => {
+  const maybeFile = (specifier: string): void => {
     if (!isAssetPath(specifier)) return;
     const localPath = resolveLocalPath(path, specifier);
     if (!localPath) return console.warn(`non-local asset path: ${specifier}`);
-    assets.add(relativePath(path, localPath));
+    files.add(relativePath(path, localPath));
   };
 
   for (const [selector, src] of ASSET_PROPERTIES) {
     for (const element of document.querySelectorAll(selector)) {
-      const source = decodeURIComponent(element.getAttribute(src)!);
+      const source = decodeURI(element.getAttribute(src)!);
       if (src === "srcset") {
         for (const s of parseSrcset(source)) {
-          maybeAsset(s);
+          maybeFile(s);
         }
       } else {
-        maybeAsset(source);
+        maybeFile(source);
       }
     }
   }
 
-  return assets;
+  for (const script of document.querySelectorAll<HTMLScriptElement>("script[src]")) {
+    let src = script.getAttribute("src")!;
+    if (isJavaScript(script)) {
+      if (isAssetPath(src)) {
+        const localPath = resolveLocalPath(path, src);
+        if (!localPath) {
+          console.warn(`non-local asset path: ${src}`);
+          continue;
+        }
+        localImports.add((src = relativePath(path, localPath)));
+      } else {
+        globalImports.add(src);
+      }
+      if (script.getAttribute("type")?.toLowerCase() === "module") {
+        staticImports.add(src); // modulepreload
+      }
+    } else {
+      maybeFile(src);
+    }
+  }
+
+  return {files, localImports, globalImports, staticImports};
+}
+
+interface HtmlResolvers {
+  resolveFile?: (specifier: string) => string;
+  resolveScript?: (specifier: string) => string;
 }
 
-export function rewriteHtml(html: string, resolve: (specifier: string) => string = String): string {
+export function rewriteHtml(html: string, {resolveFile = String, resolveScript = String}: HtmlResolvers): string {
   const {document} = parseHtml(html);
 
-  const maybeResolve = (specifier: string): string => {
-    return isAssetPath(specifier) ? resolve(specifier) : specifier;
+  const maybeResolveFile = (specifier: string): string => {
+    return isAssetPath(specifier) ? resolveFile(specifier) : specifier;
   };
 
   for (const [selector, src] of ASSET_PROPERTIES) {
     for (const element of document.querySelectorAll(selector)) {
-      const source = decodeURIComponent(element.getAttribute(src)!);
-      element.setAttribute(src, src === "srcset" ? resolveSrcset(source, maybeResolve) : maybeResolve(source));
+      const source = decodeURI(element.getAttribute(src)!);
+      element.setAttribute(src, src === "srcset" ? resolveSrcset(source, maybeResolveFile) : maybeResolveFile(source));
     }
   }
 
+  for (const script of document.querySelectorAll<HTMLScriptElement>("script[src]")) {
+    const src = decodeURI(script.getAttribute("src")!);
+    script.setAttribute("src", (isJavaScript(script) ? resolveScript : maybeResolveFile)(src));
+  }
+
   // Syntax highlighting for <code> elements. The code could contain an inline
   // expression within, or other HTML, but we only highlight text nodes that are
   // direct children of code elements.

src/javascript/imports.ts

@@ -71,7 +71,7 @@ export function findImports(body: Node, path: string, input: string): ImportRefe
   function findImport(node: ImportNode | ExportNode) {
     const source = node.source;
     if (!source || !isStringLiteral(source)) return;
-    const name = decodeURIComponent(getStringLiteralValue(source));
+    const name = decodeURI(getStringLiteralValue(source));
     const method = node.type === "ImportExpression" ? "dynamic" : "static";
     if (isPathImport(name)) {
       const localPath = resolveLocalPath(path, name);
@@ -85,7 +85,7 @@ export function findImports(body: Node, path: string, input: string): ImportRefe
   function findImportMetaResolve(node: CallExpression) {
     const source = node.arguments[0];
     if (!isImportMetaResolve(node) || !isStringLiteral(source)) return;
-    const name = decodeURIComponent(getStringLiteralValue(source));
+    const name = decodeURI(getStringLiteralValue(source));
     if (isPathImport(name)) {
       const localPath = resolveLocalPath(path, name);
       if (!localPath) throw syntaxError(`non-local import: ${name}`, node, input); // prettier-ignore

src/markdown.ts

@@ -334,7 +334,7 @@ export function parseMarkdown(input: string, {md, path, style: configStyle}: Par
   const code: MarkdownCode[] = [];
   const context: ParseContext = {code, startLine: 0, currentLine: 0, path};
   const tokens = md.parse(content, context);
-  const html = md.renderer.render(tokens, md.options, context); // Note: mutates code, assets!
+  const html = md.renderer.render(tokens, md.options, context); // Note: mutates code!
   const style = getStylesheet(path, data, configStyle);
   return {
     html,

src/npm.ts

@@ -80,7 +80,7 @@ export async function populateNpmCache(root: string, path: string): Promise<stri
   let promise = npmRequests.get(path);
   if (promise) return promise; // coalesce concurrent requests
   promise = (async function () {
-    const specifier = resolveNpmSpecifier(path);
+    const specifier = extractNpmSpecifier(path);
     const href = `https://cdn.jsdelivr.net/npm/${specifier}`;
     process.stdout.write(`npm:${specifier} ${faint("→")} `);
     const response = await fetch(href);
@@ -117,7 +117,7 @@ export async function getDependencyResolver(
 ): Promise<(specifier: string) => string> {
   const body = parseProgram(input);
   const dependencies = new Set<string>();
-  const {name, range} = parseNpmSpecifier(resolveNpmSpecifier(path));
+  const {name, range} = parseNpmSpecifier(extractNpmSpecifier(path));
 
   simple(body, {
     ImportDeclaration: findImport,
@@ -172,7 +172,7 @@ export async function getDependencyResolver(
   return (specifier: string) => {
     if (!specifier.startsWith("/npm/")) return specifier;
     if (resolutions.has(specifier)) specifier = resolutions.get(specifier)!;
-    else specifier = `/_npm/${specifier.slice("/npm/".length)}${specifier.endsWith("/+esm") ? ".js" : ""}`;
+    else specifier = fromJsDelivrPath(specifier);
     return relativePath(path, specifier);
   };
 }
@@ -249,14 +249,14 @@ export async function resolveNpmImport(root: string, specifier: string): Promise
       ? "dist/echarts.esm.min.js"
       : "+esm"
   } = parseNpmSpecifier(specifier);
-  return `/_npm/${name}@${await resolveNpmVersion(root, {name, range})}/${path.replace(/\+esm$/, "+esm.js")}`;
+  return `/_npm/${name}@${await resolveNpmVersion(root, {name, range})}/${path.replace(/\+esm$/, "_esm.js")}`;
 }
 
 const npmImportsCache = new Map<string, Promise<ImportReference[]>>();
 
 /**
  * Resolves the direct dependencies of the specified npm path, such as
- * "/_npm/d3@7.8.5/+esm.js", returning the corresponding set of npm paths.
+ * "/_npm/d3@7.8.5/_esm.js", returning the corresponding set of npm paths.
  */
 export async function resolveNpmImports(root: string, path: string): Promise<ImportReference[]> {
   if (!path.startsWith("/_npm/")) throw new Error(`invalid npm path: ${path}`);
@@ -278,6 +278,20 @@ export async function resolveNpmImports(root: string, path: string): Promise<Imp
   return promise;
 }
 
-export function resolveNpmSpecifier(path: string): string {
-  return path.replace(/^\/_npm\//, "").replace(/\/\+esm\.js$/, "/+esm");
+/**
+ * Given a local npm path such as "/_npm/d3@7.8.5/_esm.js", returns the
+ * corresponding npm specifier such as "d3@7.8.5".
+ */
+export function extractNpmSpecifier(path: string): string {
+  if (!path.startsWith("/_npm/")) throw new Error(`invalid npm path: ${path}`);
+  return path.replace(/^\/_npm\//, "").replace(/\/_esm\.js$/, "/+esm");
+}
+
+/**
+ * Given a jsDelivr path such as "/npm/d3@7.8.5/+esm", returns the corresponding
+ * local path such as "/_npm/d3@7.8.5/_esm.js".
+ */
+export function fromJsDelivrPath(path: string): string {
+  if (!path.startsWith("/npm/")) throw new Error(`invalid jsDelivr path: ${path}`);
+  return path.replace(/^\/npm\//, "/_npm/").replace(/\/\+esm$/, "/_esm.js");
 }