jsr: imports

docs/imports.md

@@ -99,6 +99,18 @@ Unlike `npm:` imports, Node imports do not support semver ranges: the imported v
 
 Imports from `node_modules` are cached in `.observablehq/cache/_node` within your [source root](./config#root) (typically `src`). You shouldn’t need to clear this cache as it is automatically managed, but feel free to clear it you like.
 
+## JSR imports <a href="https://github.com/observablehq/framework/pulls/957" class="observablehq-version-badge" data-version="prerelease" title="Added in #957"></a>
+
+You can import a package from [JSR (the JavaScript Registry)](https://jsr.io/) using the `jsr:` protocol. When you import using `jsr:`, Framework automatically downloads and self-hosts the package. (As with `npm:` imports, and unlike Node imports, you don’t have to install from `jsr:` manually.) As an example, here the number three is computed using a [pseudorandom number generator](https://jsr.io/@std/random) from the [Deno Standard Library](https://deno.com/blog/std-on-jsr):
+
+```js echo
+import {randomIntegerBetween, randomSeeded} from "jsr:@std/random";
+
+const prng = randomSeeded(1n);
+
+display(randomIntegerBetween(1, 10, {prng}));
+```
+
 ## Local imports
 
 You can import [JavaScript](./javascript) and [TypeScript](./javascript#type-script) modules from local files. This is useful for organizing your code into modules that can be imported across multiple pages. You can also unit test your code and share code with other web applications.

package.json

@@ -43,6 +43,7 @@
       "test/build/src/bin/",
       "test/build/src/client/",
       "test/build/src/convert.js",
+      "test/build/src/jsr.js",
       "test/build/src/observableApiConfig.js",
       "test/build/src/preview.js"
     ],
@@ -83,10 +84,12 @@
     "minisearch": "^6.3.0",
     "open": "^10.1.0",
     "pkg-dir": "^8.0.0",
+    "resolve.exports": "^2.0.2",
     "rollup": "^4.6.0",
     "rollup-plugin-esbuild": "^6.1.0",
     "semver": "^7.5.4",
     "send": "^0.18.0",
+    "tar": "^6.2.0",
     "tar-stream": "^3.1.6",
     "tsx": "^4.7.1",
     "untildify": "^5.0.0",
@@ -105,6 +108,7 @@
     "@types/node": "^20.7.1",
     "@types/prompts": "^2.4.9",
     "@types/send": "^0.17.2",
+    "@types/tar": "^6.1.11",
     "@types/tar-stream": "^3.1.3",
     "@types/ws": "^8.5.6",
     "@typescript-eslint/eslint-plugin": "^7.2.0",

src/jsr.ts

@@ -0,0 +1,172 @@
+import {mkdir, readFile, writeFile} from "node:fs/promises";
+import {join} from "node:path/posix";
+import {Readable} from "node:stream";
+import {finished} from "node:stream/promises";
+import {globSync} from "glob";
+import {exports as resolveExports} from "resolve.exports";
+import {satisfies} from "semver";
+import {x} from "tar";
+import type {ImportReference} from "./javascript/imports.js";
+import {parseImports} from "./javascript/imports.js";
+import type {NpmSpecifier} from "./npm.js";
+import {formatNpmSpecifier, parseNpmSpecifier} from "./npm.js";
+import {initializeNpmVersionCache, resolveNpmImport, rewriteNpmImports} from "./npm.js";
+import {isPathImport} from "./path.js";
+import {faint} from "./tty.js";
+
+const jsrVersionCaches = new Map<string, Promise<Map<string, string[]>>>();
+const jsrVersionRequests = new Map<string, Promise<string>>();
+const jsrPackageRequests = new Map<string, Promise<void>>();
+const jsrResolveRequests = new Map<string, Promise<string>>();
+
+function getJsrVersionCache(root: string): Promise<Map<string, string[]>> {
+  let cache = jsrVersionCaches.get(root);
+  if (!cache) jsrVersionCaches.set(root, (cache = initializeNpmVersionCache(root, "_jsr")));
+  return cache;
+}
+
+/**
+ * Resolves the desired version of the specified JSR package, respecting the
+ * specifier’s range if any. If any satisfying packages already exist in the JSR
+ * import cache, the greatest satisfying cached version is returned. Otherwise,
+ * the desired version is resolved via JSR’s API, and then the package and all
+ * its transitive dependencies are downloaded from JSR (and npm if needed).
+ */
+async function resolveJsrVersion(root: string, {name, range}: NpmSpecifier): Promise<string> {
+  const cache = await getJsrVersionCache(root);
+  const versions = cache.get(name);
+  if (versions) for (const version of versions) if (!range || satisfies(version, range)) return version;
+  const href = `https://npm.jsr.io/@jsr/${name.replace(/^@/, "").replace(/\//, "__")}`;
+  let promise = jsrVersionRequests.get(href);
+  if (promise) return promise; // coalesce concurrent requests
+  promise = (async function () {
+    process.stdout.write(`jsr:${formatNpmSpecifier({name, range})} ${faint("→")} `);
+    const metaResponse = await fetch(href);
+    if (!metaResponse.ok) throw new Error(`unable to fetch: ${href}`);
+    const meta = await metaResponse.json();
+    let version: {version: string; dist: {tarball: string}} | undefined;
+    if (meta["dist-tags"][range ?? "latest"]) {
+      version = meta["versions"][meta["dist-tags"][range ?? "latest"]];
+    } else if (range) {
+      if (meta.versions[range]) {
+        version = meta.versions[range]; // exact match; ignore yanked
+      } else {
+        for (const key in meta.versions) {
+          if (satisfies(key, range) && !meta.versions[key].yanked) {
+            version = meta.versions[key];
+          }
+        }
+      }
+    }
+    if (!version) throw new Error(`unable to resolve version: ${formatNpmSpecifier({name, range})}`);
+    await fetchJsrPackage(root, name, version.version, version.dist.tarball);
+    process.stdout.write(`${version.version}\n`);
+    return version.version;
+  })();
+  promise.catch(console.error).then(() => jsrVersionRequests.delete(href));
+  jsrVersionRequests.set(href, promise);
+  return promise;
+}
+
+/**
+ * Fetches a package from the JSR registry, as well as its transitive
+ * dependencies from JSR and npm, rewriting any dependency imports as relative
+ * paths within the  import cache.
+ */
+async function fetchJsrPackage(root: string, name: string, version: string, tarball: string): Promise<void> {
+  const dir = join(root, ".observablehq", "cache", "_jsr", formatNpmSpecifier({name, range: version}));
+  let promise = jsrPackageRequests.get(dir);
+  if (promise) return promise;
+  promise = (async () => {
+    const tarballResponse = await fetch(tarball);
+    if (!tarballResponse.ok) throw new Error(`unable to fetch: ${tarball}`);
+    await mkdir(dir, {recursive: true});
+    await finished(Readable.fromWeb(tarballResponse.body as any).pipe(x({strip: 1, C: dir})));
+    await rewriteJsrImports(root, dir);
+  })();
+  jsrPackageRequests.set(dir, promise);
+  return promise;
+}
+
+/**
+ * Resolves the given JSR specifier, such as `@std/bytes@^1.0.0`, returning the
+ * path to the module such as `/_jsr/@std/bytes@1.0.2/mod.js`.
+ */
+export async function resolveJsrImport(root: string, specifier: string): Promise<string> {
+  let promise = jsrResolveRequests.get(specifier);
+  if (promise) return promise;
+  promise = (async function () {
+    const spec = parseNpmSpecifier(specifier);
+    const {name} = spec;
+    const version = await resolveJsrVersion(root, spec);
+    const dir = join(root, ".observablehq", "cache", "_jsr", formatNpmSpecifier({name, range: version}));
+    const info = JSON.parse(await readFile(join(dir, "package.json"), "utf8"));
+    const [path] = resolveExports(info, spec.path === undefined ? "." : `./${spec.path}`, {browser: true})!;
+    return join("/", "_jsr", `${name}@${version}`, path);
+  })();
+  jsrResolveRequests.set(specifier, promise);
+  return promise;
+}
+
+/**
+ * After downloading a package from JSR, this rewrites any transitive JSR and
+ * Node imports to use relative paths within the import cache. For example, if
+ * jsr:@std/streams depends on jsr:@std/bytes, this will replace an import of
+ * @jsr/std__bytes with a relative path to /_jsr/@std/bytes@1.0.2/mod.js.
+ */
+async function rewriteJsrImports(root: string, dir: string): Promise<void> {
+  const info = JSON.parse(await readFile(join(dir, "package.json"), "utf8"));
+  for (const path of globSync("**/*.js", {cwd: dir, nodir: true})) {
+    const input = await readFile(join(dir, path), "utf8");
+    const promises = new Map<string, Promise<string>>();
+    try {
+      rewriteNpmImports(input, (i) => {
+        if (i.startsWith("@jsr/")) {
+          const {name, path} = parseNpmSpecifier(i);
+          const range = resolveDependencyVersion(info, name);
+          const specifier = formatNpmSpecifier({name: `@${name.slice("@jsr/".length).replace(/__/, "/")}`, range, path}); // prettier-ignore
+          if (!promises.has(i)) promises.set(i, resolveJsrImport(root, specifier));
+        } else if (!isPathImport(i) && !/^[\w-]+:/.test(i)) {
+          const {name, path} = parseNpmSpecifier(i);
+          const range = resolveDependencyVersion(info, i);
+          const specifier = formatNpmSpecifier({name, range, path});
+          if (!promises.has(i)) promises.set(i, resolveNpmImport(root, specifier));
+        }
+      });
+    } catch {
+      continue; // ignore syntax errors
+    }
+    const resolutions = new Map<string, string>();
+    for (const [key, promise] of promises) resolutions.set(key, await promise);
+    const output = rewriteNpmImports(input, (i) => resolutions.get(i));
+    await writeFile(join(dir, path), output, "utf8");
+  }
+}
+
+type PackageDependencies = Record<string, string>;
+
+interface PackageInfo {
+  dependencies?: PackageDependencies;
+  devDependencies?: PackageDependencies;
+  peerDependencies?: PackageDependencies;
+  optionalDependencies?: PackageDependencies;
+  bundleDependencies?: PackageDependencies;
+  bundledDependencies?: PackageDependencies;
+}
+
+// https://docs.npmjs.com/cli/v10/configuring-npm/package-json
+function resolveDependencyVersion(info: PackageInfo, name: string): string | undefined {
+  return (
+    info.dependencies?.[name] ??
+    info.devDependencies?.[name] ??
+    info.peerDependencies?.[name] ??
+    info.optionalDependencies?.[name] ??
+    info.bundleDependencies?.[name] ??
+    info.bundledDependencies?.[name]
+  );
+}
+
+export async function resolveJsrImports(root: string, path: string): Promise<ImportReference[]> {
+  if (!path.startsWith("/_jsr/")) throw new Error(`invalid jsr path: ${path}`);
+  return parseImports(join(root, ".observablehq", "cache"), path);
+}

src/npm.ts

@@ -36,7 +36,7 @@ export function formatNpmSpecifier({name, range, path}: NpmSpecifier): string {
 }
 
 /** Rewrites /npm/ import specifiers to be relative paths to /_npm/. */
-export function rewriteNpmImports(input: string, resolve: (specifier: string) => string = String): string {
+export function rewriteNpmImports(input: string, resolve: (s: string) => string | void = () => undefined): string {
   const body = parseProgram(input);
   const output = new Sourcemap(input);
 
@@ -63,7 +63,8 @@ export function rewriteNpmImports(input: string, resolve: (specifier: string) =>
   function rewriteImportSource(source: StringLiteral) {
     const value = getStringLiteralValue(source);
     const resolved = resolve(value);
-    if (value !== resolved) output.replaceLeft(source.start, source.end, JSON.stringify(resolved));
+    if (resolved === undefined || value === resolved) return;
+    output.replaceLeft(source.start, source.end, JSON.stringify(resolved));
   }
 
   // TODO Preserve the source map, but download it too.
@@ -178,9 +179,9 @@ export async function getDependencyResolver(
   };
 }
 
-async function initializeNpmVersionCache(root: string): Promise<Map<string, string[]>> {
+export async function initializeNpmVersionCache(root: string, dir = "_npm"): Promise<Map<string, string[]>> {
   const cache = new Map<string, string[]>();
-  const cacheDir = join(root, ".observablehq", "cache", "_npm");
+  const cacheDir = join(root, ".observablehq", "cache", dir);
   try {
     for (const entry of await readdir(cacheDir)) {
       if (entry.startsWith("@")) {
@@ -211,7 +212,7 @@ const npmVersionRequests = new Map<string, Promise<string>>();
 
 function getNpmVersionCache(root: string): Promise<Map<string, string[]>> {
   let cache = npmVersionCaches.get(root);
-  if (!cache) npmVersionCaches.set(root, (cache = initializeNpmVersionCache(root)));
+  if (!cache) npmVersionCaches.set(root, (cache = initializeNpmVersionCache(root, "_npm")));
   return cache;
 }
 

src/preview.ts

@@ -139,7 +139,7 @@ export class PreviewServer {
       } else if (pathname.startsWith("/_observablehq/") && pathname.endsWith(".css")) {
         const path = getClientPath(pathname.slice("/_observablehq/".length));
         end(req, res, await bundleStyles({path}), "text/css");
-      } else if (pathname.startsWith("/_node/")) {
+      } else if (pathname.startsWith("/_node/") || pathname.startsWith("/_jsr/")) {
         send(req, pathname, {root: join(root, ".observablehq", "cache")}).pipe(res);
       } else if (pathname.startsWith("/_npm/")) {
         await populateNpmCache(root, pathname);

src/resolvers.ts

@@ -3,6 +3,7 @@ import {extname, join} from "node:path/posix";
 import {findAssets} from "./html.js";
 import {defaultGlobals} from "./javascript/globals.js";
 import {getFileHash, getModuleHash, getModuleInfo} from "./javascript/module.js";
+import {resolveJsrImport, resolveJsrImports} from "./jsr.js";
 import {getImplicitDependencies, getImplicitDownloads} from "./libraries.js";
 import {getImplicitFileImports, getImplicitInputImports} from "./libraries.js";
 import {getImplicitStylesheets} from "./libraries.js";
@@ -240,12 +241,15 @@ async function resolveResolvers(
     globalImports.add(i);
   }
 
-  // Resolve npm: and bare imports. This has the side-effect of populating the
-  // npm import cache with direct dependencies, and the node import cache with
-  // all transitive dependencies.
+  // Resolve npm:, jsr:, and bare imports. This has the side-effect of
+  // populating the npm import cache with direct dependencies, and the node
+  // and jsr import caches with all transitive dependencies.
   for (const i of globalImports) {
-    if (i.startsWith("npm:") && !builtins.has(i)) {
+    if (builtins.has(i)) continue;
+    if (i.startsWith("npm:")) {
       resolutions.set(i, await resolveNpmImport(root, i.slice("npm:".length)));
+    } else if (i.startsWith("jsr:")) {
+      resolutions.set(i, await resolveJsrImport(root, i.slice("jsr:".length)));
     } else if (!/^\w+:/.test(i)) {
       try {
         resolutions.set(i, await resolveNodeImport(root, i));
@@ -255,8 +259,8 @@ async function resolveResolvers(
     }
   }
 
-  // Follow transitive imports of npm and bare imports. This populates the
-  // remainder of the npm import cache.
+  // Follow transitive imports of npm:, jsr:, and bare imports. This populates
+  // the remainder of the import caches.
   for (const [key, value] of resolutions) {
     if (key.startsWith("npm:")) {
       for (const i of await resolveNpmImports(root, value)) {
@@ -267,6 +271,18 @@ async function resolveResolvers(
           resolutions.set(specifier, path);
         }
       }
+    } else if (key.startsWith("jsr:")) {
+      for (const i of await resolveJsrImports(root, value)) {
+        if (i.type === "local") {
+          const path = resolvePath(value, i.name);
+          let specifier: string;
+          if (path.startsWith("/_npm/")) specifier = `npm:${extractNpmSpecifier(path)}`;
+          else if (path.startsWith("/_jsr/")) specifier = `jsr:${path.slice("/_jsr/".length)}`;
+          else continue;
+          globalImports.add(specifier);
+          resolutions.set(specifier, path);
+        }
+      }
     } else if (!/^\w+:/.test(key)) {
       for (const i of await resolveNodeImports(root, value)) {
         if (i.type === "local") {
@@ -282,7 +298,7 @@ async function resolveResolvers(
   // Resolve transitive static npm: and bare imports.
   const staticResolutions = new Map<string, string>();
   for (const i of staticImports) {
-    if (i.startsWith("npm:") || !/^\w+:/.test(i)) {
+    if (i.startsWith("npm:") || i.startsWith("jsr:") || !/^\w+:/.test(i)) {
       const r = resolutions.get(i);
       if (r) staticResolutions.set(i, r);
     }
@@ -297,6 +313,18 @@ async function resolveResolvers(
           staticResolutions.set(specifier, path);
         }
       }
+    } else if (key.startsWith("jsr:")) {
+      for (const i of await resolveJsrImports(root, value)) {
+        if (i.type === "local" && i.method === "static") {
+          const path = resolvePath(value, i.name);
+          let specifier: string;
+          if (path.startsWith("/_npm/")) specifier = `npm:${extractNpmSpecifier(path)}`;
+          else if (path.startsWith("/_jsr/")) specifier = `jsr:${path.slice("/_jsr/".length)}`;
+          else continue;
+          staticImports.add(specifier);
+          staticResolutions.set(specifier, path);
+        }
+      }
     } else if (!/^\w+:/.test(key)) {
       for (const i of await resolveNodeImports(root, value)) {
         if (i.type === "local" && i.method === "static") {
@@ -316,6 +344,8 @@ async function resolveResolvers(
       const path = await resolveNpmImport(root, specifier.slice("npm:".length));
       resolutions.set(specifier, path);
       await populateNpmCache(root, path);
+    } else if (specifier.startsWith("jsr:")) {
+      // TODO jsr:
     }
   }
 
@@ -327,6 +357,8 @@ async function resolveResolvers(
       const path = await resolveNpmImport(root, specifier.slice("npm:".length));
       resolutions.set(specifier, path);
       await populateNpmCache(root, path);
+    } else if (specifier.startsWith("jsr:")) {
+      // TODO jsr:
     }
   }
 
@@ -410,11 +442,14 @@ export async function getModuleStaticImports(root: string, path: string): Promis
 
   // Collect transitive global imports.
   for (const i of globalImports) {
-    if (i.startsWith("npm:") && !builtins.has(i)) {
+    if (builtins.has(i)) continue;
+    if (i.startsWith("npm:")) {
       const p = await resolveNpmImport(root, i.slice("npm:".length));
       for (const o of await resolveNpmImports(root, p)) {
         if (o.type === "local") globalImports.add(`npm:${extractNpmSpecifier(resolvePath(p, o.name))}`);
       }
+    } else if (i.startsWith("jsr:")) {
+      // TODO jsr:
     } else if (!/^\w+:/.test(i)) {
       const p = await resolveNodeImport(root, i);
       for (const o of await resolveNodeImports(root, p)) {
@@ -446,6 +481,8 @@ export function getModuleResolver(
       ? relativePath(servePath, `/_observablehq/${specifier.slice("observablehq:".length)}${extname(specifier) ? "" : ".js"}`) // prettier-ignore
       : specifier.startsWith("npm:")
       ? relativePath(servePath, await resolveNpmImport(root, specifier.slice("npm:".length)))
+      : specifier.startsWith("jsr:")
+      ? relativePath(servePath, await resolveJsrImport(root, specifier.slice("jsr:".length)))
       : !/^\w+:/.test(specifier)
       ? relativePath(servePath, await resolveNodeImport(root, specifier))
       : specifier;