jsr: imports

docs/imports.md

@@ -99,6 +99,18 @@ Unlike `npm:` imports, Node imports do not support semver ranges: the imported v
 
 Imports from `node_modules` are cached in `.observablehq/cache/_node` within your [source root](./config#root) (typically `src`). You shouldn’t need to clear this cache as it is automatically managed, but feel free to clear it you like.
 
+## JSR imports <a href="https://github.com/observablehq/framework/pulls/957" class="observablehq-version-badge" data-version="prerelease" title="Added in #957"></a>
+
+You can import a package from [JSR (the JavaScript Registry)](https://jsr.io/) using the `jsr:` protocol. When you import using `jsr:`, Framework automatically downloads and self-hosts the package. (As with `npm:` imports, and unlike Node imports, you don’t have to install from `jsr:` manually.) As an example, here the number three is computed using a [pseudorandom number generator](https://jsr.io/@std/random) from the [Deno Standard Library](https://deno.com/blog/std-on-jsr):
+
+```js echo
+import {randomIntegerBetween, randomSeeded} from "jsr:@std/random";
+
+const prng = randomSeeded(1n);
+
+display(randomIntegerBetween(1, 10, {prng}));
+```
+
 ## Local imports
 
 You can import [JavaScript](./javascript) and [TypeScript](./javascript#type-script) modules from local files. This is useful for organizing your code into modules that can be imported across multiple pages. You can also unit test your code and share code with other web applications.

package.json

@@ -43,6 +43,7 @@
       "test/build/src/bin/",
       "test/build/src/client/",
       "test/build/src/convert.js",
+      "test/build/src/jsr.js",
       "test/build/src/observableApiConfig.js",
       "test/build/src/preview.js"
     ],
@@ -83,10 +84,12 @@
     "minisearch": "^6.3.0",
     "open": "^10.1.0",
     "pkg-dir": "^8.0.0",
+    "resolve.exports": "^2.0.2",
     "rollup": "^4.6.0",
     "rollup-plugin-esbuild": "^6.1.0",
     "semver": "^7.5.4",
     "send": "^0.18.0",
+    "tar": "^6.2.0",
     "tar-stream": "^3.1.6",
     "tsx": "^4.7.1",
     "untildify": "^5.0.0",
@@ -105,6 +108,7 @@
     "@types/node": "^20.7.1",
     "@types/prompts": "^2.4.9",
     "@types/send": "^0.17.2",
+    "@types/tar": "^6.1.11",
     "@types/tar-stream": "^3.1.3",
     "@types/ws": "^8.5.6",
     "@typescript-eslint/eslint-plugin": "^7.2.0",

src/jsr.ts

@@ -0,0 +1,191 @@
+import {mkdir, readFile, writeFile} from "node:fs/promises";
+import {join} from "node:path/posix";
+import {Readable} from "node:stream";
+import {finished} from "node:stream/promises";
+import {globSync} from "glob";
+import {exports as resolveExports} from "resolve.exports";
+import {rsort, satisfies} from "semver";
+import {x} from "tar";
+import type {ImportReference} from "./javascript/imports.js";
+import {parseImports} from "./javascript/imports.js";
+import type {NpmSpecifier} from "./npm.js";
+import {formatNpmSpecifier, parseNpmSpecifier} from "./npm.js";
+import {initializeNpmVersionCache, resolveNpmImport, rewriteNpmImports} from "./npm.js";
+import {isPathImport} from "./path.js";
+import {faint} from "./tty.js";
+
+const jsrVersionCaches = new Map<string, Promise<Map<string, string[]>>>();
+const jsrVersionRequests = new Map<string, Promise<string>>();
+const jsrPackageRequests = new Map<string, Promise<void>>();
+const jsrResolveRequests = new Map<string, Promise<string>>();
+
+function getJsrVersionCache(root: string): Promise<Map<string, string[]>> {
+  let cache = jsrVersionCaches.get(root);
+  if (!cache) jsrVersionCaches.set(root, (cache = initializeNpmVersionCache(root, "_jsr")));
+  return cache;
+}
+
+/**
+ * Resolves the desired version of the specified JSR package, respecting the
+ * specifier’s range if any. If any satisfying packages already exist in the JSR
+ * import cache, the greatest satisfying cached version is returned. Otherwise,
+ * the desired version is resolved via JSR’s API, and then the package and all
+ * its transitive dependencies are downloaded from JSR (and npm if needed).
+ */
+async function resolveJsrVersion(root: string, {name, range}: NpmSpecifier): Promise<string> {
+  const cache = await getJsrVersionCache(root);
+  const versions = cache.get(name);
+  if (versions) for (const version of versions) if (!range || satisfies(version, range)) return version;
+  const href = `https://npm.jsr.io/@jsr/${name.replace(/^@/, "").replace(/\//, "__")}`;
+  let promise = jsrVersionRequests.get(href);
+  if (promise) return promise; // coalesce concurrent requests
+  promise = (async function () {
+    process.stdout.write(`jsr:${formatNpmSpecifier({name, range})} ${faint("→")} `);
+    const metaResponse = await fetch(href);
+    if (!metaResponse.ok) throw new Error(`unable to fetch: ${href}`);
+    const meta = await metaResponse.json();
+    let info: {version: string; dist: {tarball: string}} | undefined;
+    if (meta["dist-tags"][range ?? "latest"]) {
+      info = meta["versions"][meta["dist-tags"][range ?? "latest"]];
+    } else if (range) {
+      if (meta.versions[range]) {
+        info = meta.versions[range]; // exact match; ignore yanked
+      } else {
+        for (const key in meta.versions) {
+          if (satisfies(key, range) && !meta.versions[key].yanked) {
+            info = meta.versions[key];
+          }
+        }
+      }
+    }
+    if (!info) throw new Error(`unable to resolve version: ${formatNpmSpecifier({name, range})}`);
+    const {version, dist} = info;
+    await fetchJsrPackage(root, name, version, dist.tarball);
+    cache.set(name, versions ? rsort(versions.concat(version)) : [version]);
+    process.stdout.write(`${version}\n`);
+    return version;
+  })();
+  promise.catch(console.error).then(() => jsrVersionRequests.delete(href));
+  jsrVersionRequests.set(href, promise);
+  return promise;
+}
+
+/**
+ * Fetches a package from the JSR registry, as well as its transitive
+ * dependencies from JSR and npm, rewriting any dependency imports as relative
+ * paths within the  import cache.
+ */
+async function fetchJsrPackage(root: string, name: string, version: string, tarball: string): Promise<void> {
+  const dir = join(root, ".observablehq", "cache", "_jsr", formatNpmSpecifier({name, range: version}));
+  let promise = jsrPackageRequests.get(dir);
+  if (promise) return promise;
+  promise = (async () => {
+    const tarballResponse = await fetch(tarball);
+    if (!tarballResponse.ok) throw new Error(`unable to fetch: ${tarball}`);
+    await mkdir(dir, {recursive: true});
+    await finished(Readable.fromWeb(tarballResponse.body as any).pipe(x({strip: 1, C: dir})));
+    await rewriteJsrImports(root, dir);
+  })();
+  promise.catch(console.error).then(() => jsrPackageRequests.delete(dir));
+  jsrPackageRequests.set(dir, promise);
+  return promise;
+}
+
+/**
+ * Resolves the given JSR specifier, such as `@std/bytes@^1.0.0`, returning the
+ * path to the module such as `/_jsr/@std/bytes@1.0.2/mod.js`. This function
+ * also allows JSR specifiers with a leading slash indicating an already-
+ * resolved path such as /@std/bytes@1.0.2/mod.js.
+ */
+export async function resolveJsrImport(root: string, specifier: string): Promise<string> {
+  if (specifier.startsWith("/")) return `/_jsr/${specifier.slice("/".length)}`;
+  let promise = jsrResolveRequests.get(specifier);
+  if (promise) return promise;
+  promise = (async function () {
+    const spec = parseNpmSpecifier(specifier);
+    const {name} = spec;
+    const version = await resolveJsrVersion(root, spec);
+    const dir = join(root, ".observablehq", "cache", "_jsr", formatNpmSpecifier({name, range: version}));
+    const info = JSON.parse(await readFile(join(dir, "package.json"), "utf8"));
+    const [path] = resolveExports(info, spec.path === undefined ? "." : `./${spec.path}`, {browser: true})!;
+    return join("/", "_jsr", `${name}@${version}`, path);
+  })();
+  // TODO delete request promise?
+  jsrResolveRequests.set(specifier, promise);
+  return promise;
+}
+
+/**
+ * After downloading a package from JSR, this rewrites any transitive JSR and
+ * Node imports to use relative paths within the import cache. For example, if
+ * jsr:@std/streams depends on jsr:@std/bytes, this will replace an import of
+ * @jsr/std__bytes with a relative path to /_jsr/@std/bytes@1.0.2/mod.js.
+ */
+async function rewriteJsrImports(root: string, dir: string): Promise<void> {
+  const info = JSON.parse(await readFile(join(dir, "package.json"), "utf8"));
+  for (const path of globSync("**/*.js", {cwd: dir, nodir: true})) {
+    const input = await readFile(join(dir, path), "utf8");
+    const promises = new Map<string, Promise<string>>();
+    try {
+      rewriteNpmImports(input, (i) => {
+        if (i.startsWith("@jsr/")) {
+          const {name, path} = parseNpmSpecifier(i);
+          const range = resolveDependencyVersion(info, name);
+          const specifier = formatNpmSpecifier({name: `@${name.slice("@jsr/".length).replace(/__/, "/")}`, range, path}); // prettier-ignore
+          if (!promises.has(i)) promises.set(i, resolveJsrImport(root, specifier));
+        } else if (!isPathImport(i) && !/^[\w-]+:/.test(i)) {
+          const {name, path} = parseNpmSpecifier(i);
+          const range = resolveDependencyVersion(info, i);
+          const specifier = formatNpmSpecifier({name, range, path});
+          if (!promises.has(i)) promises.set(i, resolveNpmImport(root, specifier));
+        }
+      });
+    } catch {
+      continue; // ignore syntax errors
+    }
+    const resolutions = new Map<string, string>();
+    for (const [key, promise] of promises) resolutions.set(key, await promise);
+    const output = rewriteNpmImports(input, (i) => resolutions.get(i));
+    await writeFile(join(dir, path), output, "utf8");
+  }
+}
+
+type PackageDependencies = Record<string, string>;
+
+interface PackageInfo {
+  dependencies?: PackageDependencies;
+  devDependencies?: PackageDependencies;
+  peerDependencies?: PackageDependencies;
+  optionalDependencies?: PackageDependencies;
+  bundleDependencies?: PackageDependencies;
+  bundledDependencies?: PackageDependencies;
+}
+
+// https://docs.npmjs.com/cli/v10/configuring-npm/package-json
+function resolveDependencyVersion(info: PackageInfo, name: string): string | undefined {
+  return (
+    info.dependencies?.[name] ??
+    info.devDependencies?.[name] ??
+    info.peerDependencies?.[name] ??
+    info.optionalDependencies?.[name] ??
+    info.bundleDependencies?.[name] ??
+    info.bundledDependencies?.[name]
+  );
+}
+
+export async function resolveJsrImports(root: string, path: string): Promise<ImportReference[]> {
+  if (!path.startsWith("/_jsr/")) throw new Error(`invalid jsr path: ${path}`);
+  return parseImports(join(root, ".observablehq", "cache"), path);
+}
+
+/**
+ * The conversion of JSR specifier (e.g., @std/random) to JSR path (e.g.,
+ * @std/random@0.1.0/between.js) is not invertible, so we can’t reconstruct the
+ * JSR specifier from the path; hence this method instead returns a specifier
+ * with a leading slash such as /@std/random@0.1.0/between.js that can be used
+ * to avoid re-resolving JSR specifiers.
+ */
+export function extractJsrSpecifier(path: string): string {
+  if (!path.startsWith("/_jsr/")) throw new Error(`invalid jsr path: ${path}`);
+  return path.slice("/_jsr".length);
+}

src/npm.ts

@@ -36,7 +36,7 @@ export function formatNpmSpecifier({name, range, path}: NpmSpecifier): string {
 }
 
 /** Rewrites /npm/ import specifiers to be relative paths to /_npm/. */
-export function rewriteNpmImports(input: string, resolve: (specifier: string) => string = String): string {
+export function rewriteNpmImports(input: string, resolve: (s: string) => string | void = () => undefined): string {
   const body = parseProgram(input);
   const output = new Sourcemap(input);
 
@@ -63,7 +63,8 @@ export function rewriteNpmImports(input: string, resolve: (specifier: string) =>
   function rewriteImportSource(source: StringLiteral) {
     const value = getStringLiteralValue(source);
     const resolved = resolve(value);
-    if (value !== resolved) output.replaceLeft(source.start, source.end, JSON.stringify(resolved));
+    if (resolved === undefined || value === resolved) return;
+    output.replaceLeft(source.start, source.end, JSON.stringify(resolved));
   }
 
   // TODO Preserve the source map, but download it too.
@@ -178,9 +179,9 @@ export async function getDependencyResolver(
   };
 }
 
-async function initializeNpmVersionCache(root: string): Promise<Map<string, string[]>> {
+export async function initializeNpmVersionCache(root: string, dir = "_npm"): Promise<Map<string, string[]>> {
   const cache = new Map<string, string[]>();
-  const cacheDir = join(root, ".observablehq", "cache", "_npm");
+  const cacheDir = join(root, ".observablehq", "cache", dir);
   try {
     for (const entry of await readdir(cacheDir)) {
       if (entry.startsWith("@")) {
@@ -211,7 +212,7 @@ const npmVersionRequests = new Map<string, Promise<string>>();
 
 function getNpmVersionCache(root: string): Promise<Map<string, string[]>> {
   let cache = npmVersionCaches.get(root);
-  if (!cache) npmVersionCaches.set(root, (cache = initializeNpmVersionCache(root)));
+  if (!cache) npmVersionCaches.set(root, (cache = initializeNpmVersionCache(root, "_npm")));
   return cache;
 }
 

src/preview.ts

@@ -139,7 +139,7 @@ export class PreviewServer {
       } else if (pathname.startsWith("/_observablehq/") && pathname.endsWith(".css")) {
         const path = getClientPath(pathname.slice("/_observablehq/".length));
         end(req, res, await bundleStyles({path}), "text/css");
-      } else if (pathname.startsWith("/_node/")) {
+      } else if (pathname.startsWith("/_node/") || pathname.startsWith("/_jsr/")) {
         send(req, pathname, {root: join(root, ".observablehq", "cache")}).pipe(res);
       } else if (pathname.startsWith("/_npm/")) {
         await populateNpmCache(root, pathname);