From 6dad7609c3b77fe46f46af6091f2404ab6120288 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Fri, 26 Jan 2024 08:51:24 +0100 Subject: [PATCH] chore: set up workspace publish from CI (#4210) --- .github/workflows/workspace_publish.yml | 43 +++++ _tools/convert_to_workspace.ts | 247 ++++++++++++++++++++++++ _tools/packages.ts | 63 ++++++ 3 files changed, 353 insertions(+) create mode 100644 .github/workflows/workspace_publish.yml create mode 100644 _tools/convert_to_workspace.ts create mode 100644 _tools/packages.ts diff --git a/.github/workflows/workspace_publish.yml b/.github/workflows/workspace_publish.yml new file mode 100644 index 000000000000..3c19b16fe041 --- /dev/null +++ b/.github/workflows/workspace_publish.yml @@ -0,0 +1,43 @@ +name: workspace publish + +on: + push: + branches: [main, workspace_publish] + +env: + DENO_UNSTABLE_WORKSPACES: true + +jobs: + publish: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + permissions: + contents: read + id-token: write + + steps: + - name: Clone repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Deno + uses: denoland/setup-deno@v1 + + - name: Convert to workspace + run: deno run -A ./_tools/convert_to_workspace.ts + + - name: Format + run: deno fmt + + - name: Type check + run: deno test --unstable --no-run --doc + + - name: Publish (dry run) + if: startsWith(github.ref, 'refs/tags/') == false + run: deno publish --dry-run + + - name: Publish (real) + if: startsWith(github.ref, 'refs/tags/') + run: deno publish diff --git a/_tools/convert_to_workspace.ts b/_tools/convert_to_workspace.ts new file mode 100644 index 000000000000..bca2ed7bce1a --- /dev/null +++ b/_tools/convert_to_workspace.ts @@ -0,0 +1,247 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { discoverExportsByPackage, discoverPackages } from "./packages.ts"; +import { walk } from "../fs/walk.ts"; +import { + dirname, + fromFileUrl, + join, + relative, + toFileUrl, +} from "../path/mod.ts"; +import { VERSION } from "../version.ts"; + +const cwd = await Deno.realPath("."); + +await Deno.remove("./version.ts"); + +const toRemove = `1. Do not import symbols with an underscore in the name. + + Bad: + \`\`\`ts + import { _format } from "https://deno.land/std@$STD_VERSION/path/_common/format.ts"; + \`\`\` + +1. Do not import modules with an underscore in the path. + + Bad: + \`\`\`ts + import { filterInPlace } from "https://deno.land/std@$STD_VERSION/collections/_utils.ts"; + \`\`\` + +1. Do not import test modules or test data. + + Bad: + \`\`\`ts + import { test } from "https://deno.land/std@$STD_VERSION/front_matter/test.ts"; + \`\`\` +`; +let readme = await Deno.readTextFile("README.md"); +readme = readme.replace(toRemove, ""); +await Deno.writeTextFile("README.md", readme); + +let fileServer = await Deno.readTextFile("http/file_server.ts"); +fileServer = fileServer.replace( + `import { VERSION } from "../version.ts";`, + `import { version } from "./deno.json" with { type: "json" };`, +); +fileServer = fileServer.replaceAll("${VERSION}", "${version}"); +fileServer = fileServer.replace( + "https://deno.land/std/http/file_server.ts", + "jsr:@std/http@${version}/file_server", +); +await Deno.writeTextFile("http/file_server.ts", fileServer); + +let fileServerTest = await Deno.readTextFile("http/file_server_test.ts"); +fileServerTest = fileServerTest.replace( + `import { VERSION } from "../version.ts";`, + `import { version } from "./deno.json" with { type: "json" };`, +); +fileServerTest = fileServerTest.replaceAll("${VERSION}", "${version}"); +await Deno.writeTextFile("http/file_server_test.ts", fileServerTest); + +const packages = await discoverPackages(); +const exportsByPackage = await discoverExportsByPackage(packages); + +const allExports: string[] = []; +for (const [pkg, exports] of exportsByPackage.entries()) { + for (const [_, path] of exports) { + allExports.push(join(pkg, path)); + } +} + +// can't use a data url here because it's too long and won't work on windows +const tempFileText = allExports.map((path) => + `import "${toFileUrl(Deno.realPathSync(path))}";` +) + .join(""); +const tempFilePath = "temp_graph.ts"; +Deno.writeTextFileSync(tempFilePath, tempFileText); +const out = await new Deno.Command(Deno.execPath(), { + args: ["info", "--json", "--config", "deno.json", tempFilePath], +}).output(); +Deno.removeSync(tempFilePath); +const graph = JSON.parse(new TextDecoder().decode(out.stdout)); + +const pkgDeps = new Map>( + packages.map((pkg) => [pkg, new Set()]), +); +for (const { specifier, dependencies } of graph.modules) { + if (!specifier.startsWith("file://") || specifier.endsWith("temp_graph.ts")) { + continue; + } + const from = relative(cwd, fromFileUrl(specifier)).replaceAll("\\", "/"); + const fromPkg = from.split("/")[0]; + for (const dep of dependencies ?? []) { + if (dep.code) { + const to = relative(cwd, fromFileUrl(dep.code.specifier)).replaceAll( + "\\", + "/", + ); + const toPkg = to.split("/")[0]; + if (fromPkg !== toPkg) { + pkgDeps.get(fromPkg)!.add(toPkg); + } + } + if (dep.types) { + const to = relative(cwd, fromFileUrl(dep.types.specifier)).replaceAll( + "\\", + "/", + ); + const toPkg = to.split("/")[0]; + if (fromPkg !== toPkg) { + pkgDeps.get(fromPkg)!.add(toPkg); + } + } + } +} + +const orderedPackages: string[] = []; +const seen = new Set(); +function visit(pkg: string) { + if (seen.has(pkg)) return; + seen.add(pkg); + for (const dep of pkgDeps.get(pkg)!) { + visit(dep); + } + orderedPackages.push(pkg); +} +for (const pkg of packages) { + visit(pkg); +} + +// Now walk through all files, and replace relative imports between packages +// with absolute jsr imports like so: +// ``` +// // cli/parse_args.ts +// import { assert } from "../assert/assert.ts"; +// import * as path from "../path/mod.ts"; +// ``` +// becomes +// ``` +// // cli/parse_args.ts +// import { assert } from "@std/assert/assert"; +// import * as path from "@std/path"; +// ``` +// Also replace all absolute https://deno.land/std@$STD_VERSION/ imports with absolute jsr +// imports. +for await (const entry of walk(cwd)) { + if (!entry.isFile) continue; + if (entry.path.includes("/_tools")) continue; // ignore tools + if (entry.path.includes("/testdata/")) continue; // ignore testdata + + if (!entry.path.endsWith(".md") && !entry.path.endsWith(".ts")) continue; + const text = await Deno.readTextFile(entry.path); + const currentUrl = toFileUrl(entry.path); + const currentPkg = + relative(cwd, entry.path).replaceAll("\\", "/").split("/")[0]; + + // Find all relative imports. + const relativeImportRegex = /from\s+["']\.?\.\/([^"']+)["']/g; + const relativeImports = []; + for (const match of text.matchAll(relativeImportRegex)) { + relativeImports.push("../" + match[1]); + } + + // Find all absolute imports. + const absoluteImportRegex = + /https:\/\/deno\.land\/std@\$STD_VERSION\/([^"'\s]+)/g; + const absoluteImports = []; + for (const match of text.matchAll(absoluteImportRegex)) { + absoluteImports.push("https://deno.land/std@$STD_VERSION/" + match[1]); + } + + const replacedImports: [string, string][] = []; + + for (const specifier of relativeImports) { + const targetUrl = new URL(specifier, currentUrl); + const path = fromFileUrl(targetUrl); + const target = relative(cwd, path).replaceAll("\\", "/"); + const pkg = target.split("/")[0]; + if (pkg === currentPkg) { + let newSpecifier = relative(dirname(entry.path), target).replaceAll( + "\\", + "/", + ); + if (!newSpecifier.startsWith(".")) { + newSpecifier = "./" + newSpecifier; + } + replacedImports.push([specifier, newSpecifier]); + } else { + const newSpecifier = "@std/" + + target.replace(/(\.d)?\.ts$/, "").replace(/\/mod$/, ""); + replacedImports.push([specifier, newSpecifier]); + } + } + + for (const specifier of absoluteImports) { + const target = specifier.replace( + /^https:\/\/deno\.land\/std@\$STD_VERSION\//, + "", + ); + const newSpecifier = "@std/" + + target.replace(/(\.d)?\.ts$/, "").replace(/\/mod$/, ""); + replacedImports.push([specifier, newSpecifier]); + } + + // Replace all imports. + let newText = text; + for (const [oldSpecifier, newSpecifier] of replacedImports) { + newText = newText.replace(oldSpecifier, newSpecifier); + } + + // Write the file back. + await Deno.writeTextFile(entry.path, newText); +} + +// Generate `$package/deno.json` files. +for (const pkg of packages) { + const exportsList = exportsByPackage.get(pkg)!; + let exports; + if (exportsList.length === 1 && exportsList[0][0] === ".") { + exports = "./mod.ts"; + } else { + exports = Object.fromEntries(exportsList); + } + const denoJson = { + name: `@std/${pkg}`, + version: VERSION, + exports, + }; + await Deno.writeTextFile( + join(pkg, "deno.json"), + JSON.stringify(denoJson, null, 2) + "\n", + ); +} + +// Generate `deno.json` file. +const denoJson = JSON.parse(await Deno.readTextFile("deno.json")); +denoJson.workspaces = orderedPackages.map((pkg) => `./${pkg}`); +for (const pkg of packages) { + denoJson.imports[`@std/${pkg}`] = `jsr:@std/${pkg}@^${VERSION}`; + denoJson.imports[`@std/${pkg}/`] = `jsr:/@std/${pkg}@^${VERSION}/`; +} +await Deno.writeTextFile( + "deno.json", + JSON.stringify(denoJson, null, 2) + "\n", +); diff --git a/_tools/packages.ts b/_tools/packages.ts new file mode 100644 index 000000000000..1223bee9d05c --- /dev/null +++ b/_tools/packages.ts @@ -0,0 +1,63 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { walk } from "../fs/walk.ts"; +import { relative } from "../path/mod.ts"; + +export async function discoverPackages() { + const packages = []; + for await (const entry of Deno.readDir(".")) { + if ( + entry.isDirectory && !entry.name.startsWith(".") && + !entry.name.startsWith("_") && entry.name !== "coverage" + ) { + packages.push(entry.name); + } + } + packages.sort(); + + console.log("Discovered", packages.length, "packages."); + return packages; +} + +export async function discoverExportsByPackage(packages: string[]) { + // Collect all of the exports for each package. + const exportsByPackage = new Map(); + for (const pkg of packages) { + const exports = await discoverExports(pkg); + exportsByPackage.set(pkg, exports); + } + return exportsByPackage; +} + +async function discoverExports(pkg: string) { + const exports: [string, string][] = []; + const base = await Deno.realPath(pkg); + const files = walk(base, { + includeFiles: true, + includeDirs: false, + includeSymlinks: false, + }); + for await (const file of files) { + const path = "/" + relative(base, file.path).replaceAll("\\", "/"); + const name = path.replace(/(\.d)?\.ts$/, ""); + if (name === path && !name.endsWith(".json")) continue; // not a typescript + if (name.includes("/.") || name.includes("/_")) continue; // hidden/internal files + if ( + (name.endsWith("_test") || name.endsWith("/test")) && + !(name === "/test" && pkg === "front_matter") + ) continue; // test files + if (name.includes("/example/") || name.endsWith("_example")) continue; // example files + if (name.includes("/testdata/")) continue; // testdata files + if (name.endsWith("/deno.json")) continue; // deno.json files + + const key = "." + name.replace(/\/mod$/, ""); + exports.push([key, "." + path]); + } + exports.sort((a, b) => a[0].localeCompare(b[0])); + return exports; +} + +if (import.meta.main) { + const packages = await discoverPackages(); + console.log(packages); +}