duckdb 1.29.0; self-host extensions

docs/lib/duckdb.md

@@ -105,3 +105,7 @@ const sql = DuckDBClient.sql({quakes: `https://earthquake.usgs.gov/earthquakes/f
 ```sql echo
 SELECT * FROM quakes ORDER BY updated DESC;
 ```
+
+## Extensions
+
+DuckDB’s [extensions](../sql#extensions) are supported.

docs/sql.md

@@ -206,3 +206,51 @@ Inputs.table(await sql([`SELECT * FROM gaia WHERE source_id IN (${[source_ids]})
 When interpolating values into SQL queries, be careful to avoid [SQL injection](https://en.wikipedia.org/wiki/SQL_injection) by properly escaping or sanitizing user input. The example above is safe only because `source_ids` are known to be numeric.
 
 </div>
+
+## Extensions
+
+DuckDB has a flexible extension mechanism that allows for dynamically loading extensions. These may extend DuckDB's functionality by providing support for additional file formats, introducing new types, and domain-specific functionality.
+
+Framework can download and host the extensions of your choice. By default, only "json" and "parquet" are self-hosted, but you can add more by specifying them in the [configuration](../config). The self-hosted extensions are served from the `/_duckdb/` directory with a content-hashed URL, ensuring optimal performance and allowing you to work offline and from a server you control.
+
+The self-hosted extensions are immediately available in all the [sql](./sql) code blocks. For instance, all these queries work instantly once you have the "json", "spatial" and "h3" extensions configured:
+
+```sql echo
+SELECT bbox FROM read_json('https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson');
+```
+
+```sql echo run=false
+SELECT ST_Area('POLYGON((0 0, 0 1, 1 1, 1 0, 0 0))'::GEOMETRY) as area;
+```
+
+```sql echo run=false
+SELECT format('{:x}', h3_latlng_to_cell(37.77, -122.43, 9)) AS cell_id;
+```
+
+If you load an extension that is not self-hosted, DuckDB falls back to loading it directly from the core or community servers. For example, this documentation does not have the "inet" extension configured for self-hosting. If you inspect the network tab in your browser, you can see that it gets autoloaded from the official core extensions repository.
+
+```sql echo
+SELECT '127.0.0.1'::INET AS ipv4, '2001:db8:3c4d::/48'::INET AS ipv6;
+```
+
+At present Framework does not know which extensions your code is using. As indicated above, you have to inspect the network activity in your browser to see if that is the case, and you can then decide to add them to your configuration for self-hosting. (In the future, the preview server might be able to raise a warning if the list is incomplete. If you are interested in this feature, please upvote #issueTK.)
+
+<div class="tip">
+
+You can also initialize a custom [DuckDBClient](./lib/duckdb) with a custom list of extensions. For example, the `sql2` tagged template literal below does not load "json" or "parquet", and installs and loads "h3" directly from the community extensions repository:
+
+```js echo run=false
+const sql2 = await DuckDBClient.sql();
+await sql2`INSTALL h3 FROM community;`
+await sql2`LOAD h3;`
+```
+
+You can use it in JavaScript to return the cell [`892830828a3ffff`](https://h3geo.org/#hex=892830828a3ffff):
+
+```js run=false
+sql2`SELECT format('{:x}', h3_latlng_to_cell(37.77, -122.43, 9)) AS id;`
+```
+
+</div>
+
+These features are tied to DuckDB wasm’s 1.29 version, and strongly dependent on its development cycle.

src/build.ts

@@ -9,7 +9,7 @@ import {findModule, getModuleHash, readJavaScript} from "./javascript/module.js"
 import {transpileModule} from "./javascript/transpile.js";
 import type {Logger, Writer} from "./logger.js";
 import type {MarkdownPage} from "./markdown.js";
-import {populateNpmCache, resolveNpmImport, rewriteNpmImports} from "./npm.js";
+import {populateNpmCache, resolveDuckDBExtension, resolveNpmImport, rewriteNpmImports} from "./npm.js";
 import {isAssetPath, isPathImport, relativePath, resolvePath, within} from "./path.js";
 import {renderModule, renderPage} from "./render.js";
 import type {Resolvers} from "./resolvers.js";
@@ -53,7 +53,7 @@ export async function build(
   {config}: BuildOptions,
   effects: BuildEffects = new FileBuildEffects(config.output, join(config.root, ".observablehq", "cache"))
 ): Promise<void> {
-  const {root, loaders} = config;
+  const {root, loaders, duckdb} = config;
   Telemetry.record({event: "build", step: "start"});
 
   // Prepare for build (such as by emptying the existing output root).
@@ -201,8 +201,8 @@ export async function build(
   }
 
   // Copy over global assets (e.g., minisearch.json, DuckDB’s WebAssembly).
-  // Anything in _observablehq also needs a content hash, but anything in _npm
-  // or _node does not (because they are already necessarily immutable).
+  // Anything in _observablehq also needs a content hash, but anything in _npm,
+  // _node or _duckdb does not (because they are already necessarily immutable).
   for (const path of globalImports) {
     if (path.endsWith(".js")) continue;
     const sourcePath = join(cacheRoot, path);
@@ -218,6 +218,20 @@ export async function build(
     }
   }
 
+  // Write the DuckDB extensions manifest.
+  if (globalImports.has("/_observablehq/stdlib/duckdb.js")) {
+    const path = join("_observablehq", "duckdb_manifest.json");
+    effects.output.write(`${faint("duckdb manifest")} `);
+    const manifest = await Promise.all(
+      Object.entries(duckdb.extensions).map(async ([name, source]) => [
+        name,
+        {ref: dirname(dirname(dirname(await resolveDuckDBExtension(root, source)))), load: true}
+      ])
+    );
+    await effects.writeFile(path, JSON.stringify(manifest));
+    effects.logger.log(path);
+  }
+
   // Compute the hashes for global modules. By computing the hash on the file in
   // the cache root, this takes into consideration the resolved exact versions
   // of npm and node imports for transitive dependencies.

src/client/stdlib/duckdb.js

@@ -169,6 +169,7 @@ export class DuckDBClient {
       config = {...config, query: {...config.query, castBigIntToDouble: true}};
     }
     await db.open(config);
+    await registerExtensions(db);
     await Promise.all(Object.entries(sources).map(([name, source]) => insertSource(db, name, source)));
     return new DuckDBClient(db);
   }
@@ -182,6 +183,22 @@ Object.defineProperty(DuckDBClient.prototype, "dialect", {
   value: "duckdb"
 });
 
+async function registerExtensions(db) {
+  const connection = await db.connect();
+  try {
+    const extensions = await fetch(import.meta.resolve("observablehq:duckdb_manifest.json")).then((r) => r.json());
+    await Promise.all(
+      extensions.map(([name, {ref, load}]) =>
+        connection
+          .query(`INSTALL ${name} FROM '${import.meta.resolve(`../../${ref}`)}'`)
+          .then(() => load && connection.query(`LOAD ${name}`))
+      )
+    );
+  } finally {
+    await connection.close();
+  }
+}
+
 async function insertSource(database, name, source) {
   source = await source;
   if (isFileAttachment(source)) return insertFile(database, name, source);

src/config.ts

@@ -76,6 +76,10 @@ export interface SearchConfigSpec {
   index?: unknown;
 }
 
+export interface DuckDBConfig {
+  extensions: {[key: string]: string};
+}
+
 export interface Config {
   root: string; // defaults to src
   output: string; // defaults to dist
@@ -98,6 +102,7 @@ export interface Config {
   normalizePath: (path: string) => string;
   loaders: LoaderResolver;
   watchPath?: string;
+  duckdb: DuckDBConfig;
 }
 
 export interface ConfigSpec {
@@ -125,6 +130,7 @@ export interface ConfigSpec {
   quotes?: unknown;
   cleanUrls?: unknown;
   markdownIt?: unknown;
+  duckdb?: unknown;
 }
 
 interface ScriptSpec {
@@ -260,6 +266,7 @@ export function normalizeConfig(spec: ConfigSpec = {}, defaultRoot?: string, wat
   const search = spec.search == null || spec.search === false ? null : normalizeSearch(spec.search as any);
   const interpreters = normalizeInterpreters(spec.interpreters as any);
   const normalizePath = getPathNormalizer(spec.cleanUrls);
+  const duckdb = normalizeDuckDB(spec.duckdb as any);
 
   // If this path ends with a slash, then add an implicit /index to the
   // end of the path. Otherwise, remove the .html extension (we use clean
@@ -310,7 +317,8 @@ export function normalizeConfig(spec: ConfigSpec = {}, defaultRoot?: string, wat
     md,
     normalizePath,
     loaders: new LoaderResolver({root, interpreters}),
-    watchPath
+    watchPath,
+    duckdb
   };
   if (pages === undefined) Object.defineProperty(config, "pages", {get: () => readPages(root, md)});
   if (sidebar === undefined) Object.defineProperty(config, "sidebar", {get: () => config.pages.length > 0});
@@ -488,3 +496,19 @@ export function mergeStyle(
 export function stringOrNull(spec: unknown): string | null {
   return spec == null || spec === false ? null : String(spec);
 }
+
+function normalizeDuckDB(spec: unknown): DuckDBConfig {
+  const extensions = spec?.["extensions"] ?? ["json", "parquet"];
+  return {
+    extensions: Object.fromEntries(
+      Object.entries(
+        Array.isArray(extensions)
+          ? Object.fromEntries(extensions.map((name) => [name, true]))
+          : (spec as {[key: string]: string})
+      ).map(([name, value]) => [
+        name,
+        value === true ? `https://extensions.duckdb.org/v1.1.1/wasm_eh/${name}.duckdb_extension.wasm` : `${value}`
+      ])
+    )
+  };
+}

src/libraries.ts

@@ -1,3 +1,5 @@
+import type {DuckDBConfig} from "./config.js";
+
 export function getImplicitFileImports(methods: Iterable<string>): Set<string> {
   const set = setof(methods);
   const implicits = new Set<string>();
@@ -72,14 +74,15 @@ export function getImplicitStylesheets(imports: Iterable<string>): Set<string> {
  * library used by FileAttachment) we manually enumerate the needed additional
  * downloads here. TODO Support versioned imports, too, such as "npm:leaflet@1".
  */
-export function getImplicitDownloads(imports: Iterable<string>): Set<string> {
+export function getImplicitDownloads(imports: Iterable<string>, duckdb: DuckDBConfig): Set<string> {
   const set = setof(imports);
   const implicits = new Set<string>();
   if (set.has("npm:@observablehq/duckdb")) {
     implicits.add("npm:@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm");
     implicits.add("npm:@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js");
     implicits.add("npm:@duckdb/duckdb-wasm/dist/duckdb-eh.wasm");
     implicits.add("npm:@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js");
+    for (const [, url] of Object.entries(duckdb.extensions)) implicits.add(url);
   }
   if (set.has("npm:@observablehq/sqlite")) {
     implicits.add("npm:sql.js/dist/sql-wasm.js");

src/npm.ts

@@ -1,5 +1,6 @@
+import {createHash} from "node:crypto";
 import {existsSync} from "node:fs";
-import {mkdir, readFile, readdir, writeFile} from "node:fs/promises";
+import {copyFile, mkdir, readFile, readdir, writeFile} from "node:fs/promises";
 import {dirname, extname, join} from "node:path/posix";
 import type {CallExpression} from "acorn";
 import {simple} from "acorn-walk";
@@ -162,7 +163,7 @@ export async function getDependencyResolver(
         (name === "arquero" || name === "@uwdata/mosaic-core" || name === "@duckdb/duckdb-wasm") && depName === "apache-arrow" // prettier-ignore
           ? "latest" // force Arquero, Mosaic & DuckDB-Wasm to use the (same) latest version of Arrow
           : name === "@uwdata/mosaic-core" && depName === "@duckdb/duckdb-wasm"
-          ? "1.28.0" // force Mosaic to use the latest (stable) version of DuckDB-Wasm
+          ? "1.29.0" // force Mosaic to use the latest (stable) version of DuckDB-Wasm
           : pkg.dependencies?.[depName] ??
             pkg.devDependencies?.[depName] ??
             pkg.peerDependencies?.[depName] ??
@@ -248,9 +249,7 @@ async function resolveNpmVersion(root: string, {name, range}: NpmSpecifier): Pro
 export async function resolveNpmImport(root: string, specifier: string): Promise<string> {
   const {
     name,
-    range = name === "@duckdb/duckdb-wasm"
-      ? "1.28.0" // https://github.com/duckdb/duckdb-wasm/issues/1561
-      : undefined,
+    range = name === "@duckdb/duckdb-wasm" ? "1.29.0" : undefined,
     path = name === "mermaid"
       ? "dist/mermaid.esm.min.mjs/+esm"
       : name === "echarts"
@@ -316,3 +315,41 @@ export function fromJsDelivrPath(path: string): string {
   const subpath = parts.slice(i).join("/"); // "+esm" or "lite/+esm" or "lite.js/+esm"
   return `/_npm/${namever}/${subpath === "+esm" ? "_esm.js" : subpath.replace(/\/\+esm$/, "._esm.js")}`;
 }
+
+const downloadRequests = new Map<string, Promise<string>>();
+
+/**
+ * Given a URL such as
+ * https://extensions.duckdb.org/v1.1.1/wasm_eh/parquet.duckdb_extension.wasm,
+ * saves the file to _duckdb/{hash}/v1.1.1/wasm_eh/parquet.duckdb_extension.wasm
+ * and returns _duckdb/{hash} for DuckDB to INSTALL.
+ */
+export async function resolveDuckDBExtension(root: string, href: string): Promise<string> {
+  if (!href.startsWith("https://")) throw new Error(`invalid download path: ${href}`);
+  const {host, pathname} = new URL(href);
+  const cache = join(root, ".observablehq", "cache");
+  const outputPath = join(cache, host, pathname);
+  if (existsSync(outputPath)) return duckDBHash(outputPath, cache, pathname);
+  let promise = downloadRequests.get(outputPath);
+  if (promise) return promise; // coalesce concurrent requests
+  promise = (async () => {
+    console.log(`download: ${href} ${faint("→")} ${outputPath}`);
+    const response = await fetch(href);
+    if (!response.ok) throw new Error(`unable to fetch: ${href}`);
+    await mkdir(dirname(outputPath), {recursive: true});
+    await writeFile(outputPath, Buffer.from(await response.arrayBuffer()));
+    return duckDBHash(outputPath, cache, pathname);
+  })();
+  promise.catch(console.error).then(() => downloadRequests.delete(outputPath));
+  downloadRequests.set(outputPath, promise);
+  return promise;
+}
+
+async function duckDBHash(outputPath: string, cache: string, extension: string): Promise<string> {
+  const contents = await readFile(outputPath, "utf-8");
+  const dir = join("_duckdb", createHash("sha256").update(contents).digest("hex").slice(0, 8));
+  const targetPath = join(cache, dir, extension);
+  await mkdir(dirname(targetPath), {recursive: true});
+  await copyFile(outputPath, targetPath);
+  return join(dir, extension);
+}

src/preview.ts

@@ -25,6 +25,7 @@ import {findModule, readJavaScript} from "./javascript/module.js";
 import {transpileJavaScript, transpileModule} from "./javascript/transpile.js";
 import type {LoaderResolver} from "./loader.js";
 import type {MarkdownCode, MarkdownPage} from "./markdown.js";
+import {resolveDuckDBExtension} from "./npm.js";
 import {populateNpmCache} from "./npm.js";
 import {isPathImport, resolvePath} from "./path.js";
 import {renderModule, renderPage} from "./render.js";
@@ -118,7 +119,7 @@ export class PreviewServer {
 
   _handleRequest: RequestListener = async (req, res) => {
     const config = await this._readConfig();
-    const {root, loaders} = config;
+    const {root, loaders, duckdb} = config;
     if (this._verbose) console.log(faint(req.method!), req.url);
     const url = new URL(req.url!, "http://localhost");
     const {origin} = req.headers;
@@ -139,7 +140,15 @@ export class PreviewServer {
       } else if (pathname.startsWith("/_observablehq/") && pathname.endsWith(".css")) {
         const path = getClientPath(pathname.slice("/_observablehq/".length));
         end(req, res, await bundleStyles({path}), "text/css");
-      } else if (pathname.startsWith("/_node/") || pathname.startsWith("/_jsr/")) {
+      } else if (pathname === "/_observablehq/duckdb_manifest.json") {
+        const manifest = await Promise.all(
+          Object.entries(duckdb.extensions).map(async ([name, source]) => [
+            name,
+            {ref: dirname(dirname(dirname(await resolveDuckDBExtension(root, source)))), load: true}
+          ])
+        );
+        end(req, res, JSON.stringify(manifest), "text/json");
+      } else if (pathname.startsWith("/_node/") || pathname.startsWith("/_jsr/") || pathname.startsWith("/_duckdb/")) {
         send(req, pathname, {root: join(root, ".observablehq", "cache")}).pipe(res);
       } else if (pathname.startsWith("/_npm/")) {
         await populateNpmCache(root, pathname);
@@ -390,9 +399,9 @@ function handleWatch(socket: WebSocket, req: IncomingMessage, configPromise: Pro
     if (path.endsWith("/")) path += "index";
     path = join(dirname(path), basename(path, ".html"));
     config = await configPromise;
-    const {root, loaders, normalizePath} = config;
+    const {root, loaders, normalizePath, duckdb} = config;
     const page = await loaders.loadPage(path, {path, ...config});
-    const resolvers = await getResolvers(page, {root, path, loaders, normalizePath});
+    const resolvers = await getResolvers(page, {root, path, loaders, normalizePath, duckdb});
     if (resolvers.hash === initialHash) send({type: "welcome"});
     else return void send({type: "reload"});
     hash = resolvers.hash;

src/resolvers.ts

@@ -1,5 +1,6 @@
 import {createHash} from "node:crypto";
 import {extname, join} from "node:path/posix";
+import type {DuckDBConfig} from "./config.js";
 import {findAssets} from "./html.js";
 import {defaultGlobals} from "./javascript/globals.js";
 import {isJavaScript} from "./javascript/imports.js";
@@ -12,6 +13,7 @@ import type {LoaderResolver} from "./loader.js";
 import type {MarkdownPage} from "./markdown.js";
 import {extractNodeSpecifier, resolveNodeImport, resolveNodeImports} from "./node.js";
 import {extractNpmSpecifier, populateNpmCache, resolveNpmImport, resolveNpmImports} from "./npm.js";
+import {resolveDuckDBExtension} from "./npm.js";
 import {isAssetPath, isPathImport, parseRelativeUrl, relativePath, resolveLocalPath, resolvePath} from "./path.js";
 
 export interface Resolvers {
@@ -38,6 +40,7 @@ export interface ResolversConfig {
   normalizePath: (path: string) => string;
   globalStylesheets?: string[];
   loaders: LoaderResolver;
+  duckdb: DuckDBConfig;
 }
 
 const defaultImports = [
@@ -202,7 +205,7 @@ async function resolveResolvers(
     staticImports?: Iterable<string> | null;
     stylesheets?: Iterable<string> | null;
   },
-  {root, path, normalizePath, loaders}: ResolversConfig
+  {root, path, normalizePath, loaders, duckdb}: ResolversConfig
 ): Promise<Omit<Resolvers, "path" | "hash" | "assets" | "anchors" | "localLinks">> {
   const files = new Set<string>(initialFiles);
   const fileMethods = new Set<string>(initialFileMethods);
@@ -361,12 +364,15 @@ async function resolveResolvers(
 
   // Add implicit downloads. (This should be maybe be stored separately rather
   // than being tossed into global imports, but it works for now.)
-  for (const specifier of getImplicitDownloads(globalImports)) {
+  for (const specifier of getImplicitDownloads(globalImports, duckdb)) {
     globalImports.add(specifier);
     if (specifier.startsWith("npm:")) {
       const path = await resolveNpmImport(root, specifier.slice("npm:".length));
       resolutions.set(specifier, path);
       await populateNpmCache(root, path);
+    } else if (specifier.startsWith("https://")) {
+      const path = await resolveDuckDBExtension(root, specifier);
+      resolutions.set(specifier, path);
     } else if (!specifier.startsWith("observablehq:")) {
       throw new Error(`unhandled implicit download: ${specifier}`);
     }