.some(mash)

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly

.github/scripts/build.ts

@@ -0,0 +1,67 @@
+#!/usr/bin/env -S pkgx deno run --allow-read --allow-write
+
+import * as flags from "https://deno.land/std@0.206.0/flags/mod.ts";
+import { Path } from "https://deno.land/x/libpkgx@v0.16.0/mod.ts";
+import { Script } from "./index.ts";
+
+const args = flags.parse(Deno.args);
+const indir = (s => Path.abs(s) ?? Path.cwd().join(s))(args['input'])
+const outdir = (s => Path.abs(s) ?? Path.cwd().join(s))(args['output'])
+const index_json_path = args['index-json']
+
+if (!indir || !outdir || !index_json_path) {
+  console.error(`usage: build.ts --input <path> --output <path> --index-json <path>`);
+  Deno.exit(64);
+}
+
+const scripts = JSON.parse(Deno.readTextFileSync(index_json_path)).scripts as Script[]
+
+const categories: Record<string, Script[]> = {}
+const users: Record<string, Script[]> = {}
+
+for (const script of scripts) {
+  if (script.category) {
+    categories[script.category] ??= []
+    categories[script.category].push(script)
+  }
+  const user = script.fullname.split('/')[0]
+  users[user] ??= []
+  users[user].push(script)
+}
+
+// sort each entry in categories and users by the script birthtime
+for (const scripts of Object.values(categories)) {
+  scripts.sort((a, b) => new Date(b.birthtime).getTime() - new Date(a.birthtime).getTime());
+}
+for (const scripts of Object.values(users)) {
+  scripts.sort((a, b) => new Date(b.birthtime).getTime() - new Date(a.birthtime).getTime());
+}
+
+for (const category in categories) {
+  const d = outdir.join(category)
+  const scripts = categories[category].filter(({description}) => description)
+  d.mkdir('p').join('index.json').write({ json: { scripts }, force: true, space: 2 })
+}
+
+for (const user in users) {
+  const d = outdir.join('u', user)
+  const scripts = users[user].filter(({description}) => description)
+  d.mkdir('p').join('index.json').write({ json: { scripts }, force: true, space: 2 })
+}
+
+for (const script of scripts) {
+  console.error(script)
+  const [user, name] = script.fullname.split('/')
+  const { category } = script
+  const gh_slug = new URL(script.url).pathname.split('/').slice(1, 3).join('/')
+  const infile = indir.join(gh_slug, 'scripts', name)
+
+  infile.cp({ into: outdir.join('u', user).mkdir('p') })
+
+  const leaf = infile.basename().split('-').slice(1).join('-')
+  if (category && !outdir.join(category, leaf).exists()) { // not already snagged
+    infile.cp({ to: outdir.join(category, leaf) })
+  }
+}
+
+outdir.join('u/index.json').write({ json: { users }, force: true, space: 2})

.github/scripts/index.ts

@@ -0,0 +1,162 @@
+#!/usr/bin/env -S pkgx deno run --allow-run=bash --allow-read=.
+
+import { join, basename, dirname } from "https://deno.land/std@0.206.0/path/mod.ts";
+import { walk, exists } from "https://deno.land/std@0.206.0/fs/mod.ts";
+import * as flags from "https://deno.land/std@0.206.0/flags/mod.ts";
+
+if (import.meta.main) {
+  const args = flags.parse(Deno.args);
+  const inputdir = args['input']
+
+  if (!inputdir) {
+    console.error(`usage: index.ts --input <path>`);
+    Deno.exit(1);
+  }
+
+  Deno.chdir(inputdir);
+
+  const scripts: Script[] = []
+  for await (const slug of iterateGitRepos('.')) {
+    console.error(`iterating: ${slug}`);
+    scripts.push(...await get_metadata(slug));
+  }
+
+  scripts.sort((a, b) => b.birthtime.getTime() - a.birthtime.getTime());
+
+  const categories = (() => {
+    const categories: Record<string, number> = {}
+    for (const script of scripts) {
+      if (script.category && script.description) {
+        categories[script.category] ??= 0;
+        categories[script.category]++;
+      }
+    }
+    return Object.keys(categories)
+  })();
+
+
+  console.log(JSON.stringify({ scripts, categories }, null, 2));
+}
+
+////////////////////////////////////////////////////////////////////// lib
+async function extractMarkdownSection(filePath: string, sectionTitle: string): Promise<string | undefined> {
+  const data = await Deno.readTextFile(filePath);
+  const lines = data.split('\n');
+  let capturing = false;
+  let sectionContent = '';
+
+  for (const line of lines) {
+    if (line.startsWith('## ')) {
+      if (capturing) {
+        break;  // stop if we reach another ## section
+      } else if (normalize_title(line.slice(3)) == normalize_title(sectionTitle)) {
+        capturing = true;
+      } else if (line.slice(3).trim() == mash_title(sectionTitle)) {
+        capturing = true;
+      }
+    } else if (capturing) {
+      sectionContent += line + '\n';
+    }
+  }
+
+  return chuzzle(sectionContent);
+
+  function normalize_title(input: string) {
+    return input.toLowerCase().replace(/[^a-z0-9]/g, '').trim();
+  }
+
+  function mash_title(input: string) {
+    const [category, ...name] = input.trim().split('-')
+    return `\`mash ${category} ${name.join('-')}\``
+  }
+}
+
+export interface Script {
+  fullname: string
+  birthtime: Date
+  description?: string
+  avatar: string
+  url: string
+  category?: string
+  README?: string
+  cmd: string
+}
+
+async function* iterateGitRepos(basePath: string): AsyncIterableIterator<string> {
+  for await (const entry of walk(basePath, { maxDepth: 2 })) {
+    if (entry.isDirectory && await exists(join(entry.path, '.git'))) {
+      yield entry.path;
+    }
+  }
+}
+
+function chuzzle(ln: string): string | undefined {
+  const out = ln.trim()
+  return out || undefined;
+}
+
+async function get_metadata(slug: string) {
+
+  const cmdString = `git -C '${slug}' log --pretty=format:'%H %aI' --name-only --diff-filter=AR -- scripts`;
+
+  const process = Deno.run({
+    cmd: ["bash", "-c", cmdString],
+    stdout: "piped"
+  });
+
+  const output = new TextDecoder().decode(await process.output());
+  await process.status();
+  process.close();
+
+  const lines = chuzzle(output)?.split('\n') ?? [];
+  const rv: Script[] = []
+  let currentCommitDate: string | undefined;
+
+  for (let line of lines) {
+    line = line.trim()
+
+    if (line.includes(' ')) {  // Detect lines with commit hash and date
+      currentCommitDate = line.split(' ')[1];
+    } else if (line && currentCommitDate) {
+      const filename = join(slug, line)
+      if (!await exists(filename)) {
+        // the file used to exist but has been deleted
+        console.warn("skipping deleted: ", filename, line)
+        continue
+      }
+
+      console.error(line)
+
+      const repo_metadata = JSON.parse(await Deno.readTextFile(join(slug, 'metadata.json')))
+
+      const README = await extractMarkdownSection(join(slug, 'README.md'), basename(filename));
+      const birthtime = new Date(currentCommitDate!);
+      const avatar = repo_metadata.avatar
+      const fullname = join(dirname(slug), ...stem(filename))
+      const url = repo_metadata.url +'/scripts/' + basename(filename)
+      const category = (([x, y]) => x?.length > 0 && y ? x : undefined)(basename(filename).split("-"))
+      const description = README ? extract_description(README) : undefined
+      const cmd = category ? `mash ${category} ${basename(filename).split('-').slice(1).join('-')}` : `mash ${fullname}`
+
+      rv.push({ fullname, birthtime, description, avatar, url, category, README, cmd })
+    }
+  }
+
+  return rv;
+
+  function stem(filename: string): string[] {
+    const base = basename(filename)
+    const parts = base.split('.')
+    if (parts.length == 1) {
+      return parts.slice(0, 1)
+    } else {
+      return parts.slice(0, -1) // no extension, but allow eg. foo.bar.js to be foo.bar
+    }
+  }
+}
+
+function extract_description(input: string) {
+  const regex = /^(.*?)\n#|^.*$/ms;
+  const match = regex.exec(input);
+  return match?.[1]?.trim();
+}

.github/scripts/trawl.ts

@@ -0,0 +1,67 @@
+#!/usr/bin/env -S pkgx deno run --allow-run --allow-net --allow-env=GH_TOKEN --allow-write=.
+
+import * as flags from "https://deno.land/std@0.206.0/flags/mod.ts";
+
+const args = flags.parse(Deno.args);
+const outdir = args['out']
+
+const ghToken = Deno.env.get("GH_TOKEN");
+if (!ghToken) {
+  console.error("error: GitHub token is required. Set the GH_TOKEN environment variable.");
+  Deno.exit(1)
+}
+
+Deno.mkdirSync(outdir, { recursive: true });
+
+async function cloneAllForks(user: string, repo: string) {
+  let page = 1;
+  while (true) {
+    const response = await fetch(`https://api.github.com/repos/${user}/${repo}/forks?page=${page}`, {
+      headers: {
+        "Authorization": `token ${ghToken}`
+      }
+    });
+
+    if (!response.ok) {
+      throw new Error(`err: ${response.statusText}`);
+    }
+
+    const forks = await response.json();
+    if (forks.length === 0) {
+      break;  // No more forks
+    }
+
+    for (const fork of forks) {
+      await clone(fork)
+
+      Deno.writeTextFileSync(`${outdir}/${fork.full_name}/metadata.json`, JSON.stringify({
+        stars: fork.stargazers_count,
+        license: fork.license?.spdx_id,
+        avatar: fork.owner.avatar_url,
+        url: fork.html_url + '/blob/' + fork.default_branch
+      }, null, 2))
+    }
+
+    page++;
+  }
+}
+
+async function clone({clone_url, full_name, ...fork}: any) {
+  console.log(`Cloning ${clone_url}...`);
+  const proc = new Deno.Command("git", { args: ["-C", outdir, "clone", clone_url, full_name]}).spawn()
+  if (!(await proc.status).success) {
+    throw new Error(`err: ${await proc.status}`)
+  }
+}
+
+await cloneAllForks('pkgxdev', 'mash');
+
+// we have some general utility scripts here
+await clone({clone_url: 'https://github.com/pkgxdev/mash.git', full_name: 'pkgxdev/mash'});
+// deploy expects this and fails otherwise
+Deno.writeTextFileSync(`${outdir}/pkgxdev/mash/metadata.json`, `{
+  "stars": 0,
+  "license": "Apache-2.0",
+  "avatar": "https://avatars.githubusercontent.com/u/140643783?v=4",
+  "url": "https://github.com/pkgxdev/mash/blob/main"
+}`)

.github/workflows/ci.yml

@@ -0,0 +1,12 @@
+on:
+  pull_request:
+    paths: mash
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pkgxdev/setup@v2
+      - run: ./mash pkgxdev/demo
+      - run: ./mash pkgxdev/demo   # check cache route works too

.github/workflows/deploy.yml

@@ -0,0 +1,53 @@
+on:
+  push:
+    branches: main
+    paths:
+      - .github/workflows/deploy.yml
+      - .github/scripts/*
+  pull_request:
+    paths:
+      - .github/workflows/deploy.yml
+  schedule:
+    - cron: '23 * * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    if: github.repository == 'pkgxdev/mash'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pkgxdev/setup@v2
+
+      - run: .github/scripts/trawl.ts --out ./build
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - run: |
+          mkdir out
+          .github/scripts/index.ts --input ./build > ./out/index.json
+
+      - run: .github/scripts/build.ts --input ./build --output ./out --index-json ./out/index.json
+
+      - uses: actions/configure-pages@v4
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: out
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' }}
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    permissions:
+      pages: write      # to deploy to Pages
+      id-token: write   # to verify the deployment originates from an appropriate source
+    steps:
+      - uses: actions/deploy-pages@v4
+        id: deployment

.gitignore

@@ -0,0 +1,3 @@
+.DS_Store
+/out
+/build

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "deno.enable": true,
+  "deno.lint": true,
+  "deno.unstable": true
+}