From f93d7761bd51dd7bed5d9e14a602775ee7d08331 Mon Sep 17 00:00:00 2001 From: James Gilles Date: Tue, 3 Oct 2023 14:42:38 -0400 Subject: [PATCH] Mess with github actions benchmark script --- .github/workflows/benchmarks.yml | 76 ++++++++++++++++++++++++++----- crates/bench/benches/special.rs | 6 +-- crates/bench/src/bin/summarize.rs | 2 +- 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 9f60e75654..c28fa31cb7 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -2,7 +2,7 @@ on: push: branches: - master - - kazimuth/benchwrangle + - kazimuth/bench-pretty workflow_dispatch: inputs: pr_number: @@ -40,6 +40,37 @@ jobs: PR_DATA=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr_number }} --jq '{ baseRefName: .base.ref, headRefName: .head.ref }') echo "PR_BASE_REF=$(echo $PR_DATA | jq -r '.baseRefName')" >> $GITHUB_ENV echo "PR_HEAD_REF=$(echo $PR_DATA | jq -r '.headRefName')" >> $GITHUB_ENV + + # https://stackoverflow.com/questions/58066966/commenting-a-pull-request-in-a-github-action + # https://github.com/boa-dev/criterion-compare-action/blob/main/main.js + - name: test comment + if: ${{ env.PR_BASE_REF }} + uses: actions/github-script@v6 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + let stuff = require('fs').readFileSync('crates/bench/clippy.toml', 'utf8'); + let body = `Test comment: clippy.toml contents: '${stuff}'`; + + try { + const { data: comment } = await github.issues.createComment({ + owner: context.issue.owner, + repo: context.issue.repo, + issue_number: context.issue.number, + body: body, + }); + core.info( + `Created comment id '${comment.id}' on issue '${contextObj.number}' in '${contextObj.repo}'.` + ); + core.setOutput("comment-id", comment.id); + } catch (err) { + core.warning(`Failed to comment: ${err}`); + core.info("Commenting is not possible from forks."); + core.info("Logging here instead."); + console.log(body); + } + + - name: Install stable toolchain uses: actions-rs/toolchain@v1 with: @@ -68,19 +99,42 @@ jobs: run: | rustup component add clippy - - name: Criterion compare base branch + - name: Master; run bench + if: github.ref == 'ref/head/master' + working-directory: crates/bench/ + run: | + echo "Running benchmarks with sqlite" + cargo bench -- --save-baseline master + cargo run --bin summarize pack master + + # https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts + - name: Master; upload packed bench results + if: github.ref == 'ref/head/master' + uses: actions/upload-artifact@v3 + with: + name: master_packed_bench_json-${{ github.sha }} + path: target/criterion/master.json + + - name: PR; download bench results for compare if: ${{ env.PR_BASE_REF }} - uses: clockworklabs/criterion-compare-action@main + uses: actions/download-artifact@v3 with: - cwd: "crates/bench" - branchName: ${{ env.PR_BASE_REF }} + name: master_packed_bench_json-${{ env.PR_BASE_REF }} + path: target/criterion/master.json - - name: Criterion compare previous commit - if: env.PR_BASE_REF == '' - uses: clockworklabs/criterion-compare-action@main - with: - cwd: "crates/bench" - branchName: "HEAD~1" + - name: PR; run bench + if: ${{ env.PR_BASE_REF }} + working-directory: crates/bench/ + run: | + echo "Running benchmarks without sqlite" + cargo bench -- --save-baseline branch '(special|stdb_module|stdb_raw)' + cargo run --bin summarize pack branch + + - name: PR; compare benchmarks + if: ${{ env.PR_BASE_REF }} + working-directory: crates/bench/ + run: | + cargo run --bin summarize markdown-report master.json branch.json --report-name report - name: Clean up if: always() diff --git a/crates/bench/benches/special.rs b/crates/bench/benches/special.rs index 4c692e284b..415f83f392 100644 --- a/crates/bench/benches/special.rs +++ b/crates/bench/benches/special.rs @@ -29,7 +29,7 @@ fn custom_module_benchmarks(c: &mut Criterion) { let args = ProductValue { elements: vec![AlgebraicValue::Builtin(BuiltinValue::String("0".repeat(65536)))], }; - c.bench_function("stdb_module/large_arguments/64KiB", |b| { + c.bench_function("special/stdb_module/large_arguments/64KiB", |b| { b.iter_batched( || args.clone(), |args| runtime.block_on(async { module.call_reducer_binary("fn_with_1_args", args).await.unwrap() }), @@ -41,7 +41,7 @@ fn custom_module_benchmarks(c: &mut Criterion) { let args = ProductValue { elements: vec![AlgebraicValue::Builtin(BuiltinValue::U32(n))], }; - c.bench_function(&format!("stdb_module/print_bulk/lines={n}"), |b| { + c.bench_function(&format!("special/stdb_module/print_bulk/lines={n}"), |b| { b.iter_batched( || args.clone(), |args| runtime.block_on(async { module.call_reducer_binary("print_many_things", args).await.unwrap() }), @@ -54,7 +54,7 @@ fn custom_module_benchmarks(c: &mut Criterion) { fn serialize_benchmarks(c: &mut Criterion) { let name = T::name_snake_case(); let count = 100; - let mut group = c.benchmark_group("serialize"); + let mut group = c.benchmark_group("special/serialize"); group.throughput(criterion::Throughput::Elements(count)); let data = create_sequential::(0xdeadbeef, count as u32, 100); diff --git a/crates/bench/src/bin/summarize.rs b/crates/bench/src/bin/summarize.rs index 5d84e103bc..9ff01cc2c4 100644 --- a/crates/bench/src/bin/summarize.rs +++ b/crates/bench/src/bin/summarize.rs @@ -45,7 +45,7 @@ enum Command { /// Otherwise, read from the loose criterion files in the filesystem. baseline_old: Option, - /// Report will be written to this file. If not specified, will be written to stdout. + /// Report will be written to `{target_dir}/criterion/{report_name}.md`. #[arg(long = "report-name", required = false)] report_name: Option, },