chore: add home-cooked “continuous benchmarking”

.github/workflows/bench-main.yml

@@ -0,0 +1,38 @@
+name: Benchmark Main Branch
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'src/**/*.ts'
+
+jobs:
+  benchmark:
+    name: Run Benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22.x'
+          cache: pnpm
+
+      - name: Install dependencies
+        run: |
+          pnpm install -C scripts/benchmarks
+          pnpm install -C scripts/radashi-db
+
+      - uses: awalsh128/cache-apt-pkgs-action@v1.4.2
+        with:
+          packages: valgrind
+          version: 1 # The “cache version” for cache-busting
+
+      - name: Run benchmarks
+        env:
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
+        run: |
+          # https://pythonspeed.com/articles/consistent-benchmarking-in-ci/
+          valgrind --tool=cachegrind ./scripts/benchmarks/node_modules/.bin/tsx ./scripts/benchmarks/ci-bench-main.ts

.github/workflows/bench-pr.yml

@@ -0,0 +1,39 @@
+name: Benchmark Pull Request
+
+on:
+  pull_request:
+    branches: [main, next]
+    paths:
+      - 'src/**/*.ts'
+
+jobs:
+  benchmark:
+    name: Run Benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22.x'
+          cache: pnpm
+
+      - name: Install dependencies
+        run: |
+          pnpm install -C scripts/benchmarks
+          pnpm install -C scripts/radashi-db
+
+      - uses: awalsh128/cache-apt-pkgs-action@v1.4.2
+        with:
+          packages: valgrind
+          version: 1 # The “cache version” for cache-busting
+
+      - name: Run benchmarks
+        env:
+          RADASHI_BOT_TOKEN: ${{ secrets.RADASHI_BOT_TOKEN }}
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
+        run: |
+          # https://pythonspeed.com/articles/consistent-benchmarking-in-ci/
+          valgrind --tool=cachegrind ./scripts/benchmarks/node_modules/.bin/tsx ./scripts/benchmarks/ci-bench-pr.ts ${{ github.base_ref }} ${{ github.event.number }}

.github/workflows/seed-benchmarks.yml

@@ -0,0 +1,32 @@
+name: Seed Benchmarks
+
+on:
+  workflow_dispatch:
+
+jobs:
+  seed-benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22.x'
+          cache: pnpm
+
+      - name: Install dependencies
+        run: |
+          pnpm install -C scripts/benchmarks
+          pnpm install -C scripts/radashi-db
+
+      - uses: awalsh128/cache-apt-pkgs-action@v1.4.2
+        with:
+          packages: valgrind
+          version: 1 # The “cache version” for cache-busting
+
+      - name: Run benchmarks and seed database
+        env:
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
+        run: |
+          # https://pythonspeed.com/articles/consistent-benchmarking-in-ci/
+          valgrind --tool=cachegrind ./scripts/benchmarks/node_modules/.bin/tsx ./scripts/benchmarks/seed-benchmarks.ts

package.json

@@ -49,7 +49,6 @@
   },
   "devDependencies": {
     "@biomejs/biome": "^1.8.3",
-    "@codspeed/vitest-plugin": "^3.1.0",
     "@typescript-eslint/parser": "^7.16.1",
     "@vitest/coverage-v8": "2.0.3",
     "concurrently": "^8.2.2",
@@ -66,10 +65,5 @@
   "browserslist": [
     ">0.1% and not dead",
     "node >= 16"
-  ],
-  "pnpm": {
-    "patchedDependencies": {
-      "@codspeed/vitest-plugin@3.1.0": "patches/@codspeed__vitest-plugin@3.1.0.patch"
-    }
-  }
+  ]
 }

scripts/benchmarks/ci-bench-main.ts

@@ -0,0 +1,96 @@
+import { execa } from 'execa'
+import { existsSync } from 'node:fs'
+import { supabase } from 'radashi-db/supabase.js'
+import { createVitest } from 'vitest/node'
+import { getChangedFiles } from './src/get-changed.js'
+import { type Benchmark, reportToBenchmarkHandler } from './src/reporter.js'
+
+main()
+
+async function main() {
+  if (!process.env.SUPABASE_KEY) {
+    throw new Error('SUPABASE_KEY is not set')
+  }
+
+  // Get the last benched SHA
+  const metaResult = await supabase
+    .from('meta')
+    .select('value')
+    .eq('id', 'last_benched_sha')
+    .limit(1)
+    .single()
+
+  if (metaResult.error) {
+    console.error('Error fetching last benched SHA:', metaResult.error)
+    return
+  }
+
+  const lastBenchedSha = metaResult.data.value as string
+  const currentSha = await execa('git', ['rev-parse', 'HEAD']).then(
+    result => result.stdout,
+  )
+
+  if (lastBenchedSha === currentSha) {
+    console.log('No changes since last benched SHA')
+    return
+  }
+
+  const files = await getChangedFiles(lastBenchedSha, ['src/**/*.ts'])
+
+  const results: Benchmark[] = []
+
+  const vitest = await createVitest('benchmark', {
+    benchmark: {
+      reporters: [
+        reportToBenchmarkHandler(benchmark => {
+          results.push(benchmark)
+        }),
+      ],
+    },
+  })
+
+  for (const file of files) {
+    // Run benchmarks for modified or added source files in a function group
+    if (
+      (file.status === 'M' || file.status === 'A') &&
+      /^src\/.+?\//.test(file.name)
+    ) {
+      const benchFile = file.name
+        .replace('src', 'benchmarks')
+        .replace(/\.ts$/, '.bench.ts')
+
+      if (existsSync(benchFile)) {
+        console.log(`Running benchmarks in ./${benchFile}`)
+        await vitest.start([benchFile])
+      }
+    }
+  }
+
+  console.log('Results', results)
+
+  const { error: insertError } = await supabase.from('benchmarks').insert(
+    results.map(result => ({
+      ...result,
+      sha: currentSha,
+    })),
+  )
+
+  if (insertError) {
+    insertError.message =
+      'Error inserting benchmark results: ' + insertError.message
+    throw insertError
+  }
+
+  const { error: updateError } = await supabase
+    .from('meta')
+    .update({ value: currentSha })
+    .eq('id', 'last_benched_sha')
+
+  if (updateError) {
+    updateError.message =
+      'Error updating last benched SHA: ' + updateError.message
+    throw updateError
+  }
+
+  console.log('Completed', results.length, 'benchmarks')
+}