diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index af24320de3..b750c3b2fd 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -2,7 +2,7 @@ on:
push:
branches:
- master
- - kazimuth/benchwrangle
+
workflow_dispatch:
inputs:
pr_number:
@@ -10,6 +10,10 @@ on:
required: false
default: ''
+ # note: the "benchmarks please" comments aren't dispatched here,
+ # there's a script running on one of our internal servers that reads those and then
+ # dispatches to this workflow using the workflow_dispatch there.
+
name: Benchmarks
env:
@@ -20,6 +24,9 @@ jobs:
name: run benchmarks
runs-on: benchmarks-runner
steps:
+ - name: Enable CPU boost
+ run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost
+
- name: Checkout sources for a PR
if: ${{ github.event.inputs.ref }}
uses: actions/checkout@v3
@@ -37,9 +44,7 @@ jobs:
if: github.event.inputs.pr_number
run: |
echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV
- PR_DATA=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr_number }} --jq '{ baseRefName: .base.ref, headRefName: .head.ref }')
- echo "PR_BASE_REF=$(echo $PR_DATA | jq -r '.baseRefName')" >> $GITHUB_ENV
- echo "PR_HEAD_REF=$(echo $PR_DATA | jq -r '.headRefName')" >> $GITHUB_ENV
+
- name: Install stable toolchain
uses: actions-rs/toolchain@v1
with:
@@ -48,19 +53,6 @@ jobs:
target: wasm32-unknown-unknown
override: true
- - name: ⚡ Cache
- uses: actions/cache@v3
- with:
- path: |
- ~/.cargo/bin/
- ~/.cargo/registry/index/
- ~/.cargo/registry/cache/
- ~/.cargo/git/db/
- ~/.cargo/.crates.toml
- ~/.cargo/.crates2.json
- target/
- key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }}
-
- name: Build
working-directory: crates/bench/
run: |
@@ -70,19 +62,132 @@ jobs:
run: |
rustup component add clippy
- - name: Criterion compare base branch
- if: ${{ env.PR_BASE_REF }}
- uses: clockworklabs/criterion-compare-action@main
+ - name: Disable CPU boost
+ run: echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost
+
+ - name: Extract branch name
+ if: "! github.event.inputs.pr_number"
+ shell: bash
+ run: |
+ BRANCH_NAME=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
+ echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+ echo "NORMALIZED_BRANCH_NAME=${BRANCH_NAME//\//-}" >> $GITHUB_ENV
+
+ - name: Branch; run bench
+ if: "! github.event.inputs.pr_number"
+ run: |
+ echo "Running benchmarks with sqlite"
+ pushd crates/bench
+ cargo bench --bench generic --bench special -- --save-baseline $NORMALIZED_BRANCH_NAME
+ cargo run --bin summarize pack $NORMALIZED_BRANCH_NAME
+ popd
+ mkdir criterion-results
+ cp target/criterion/$NORMALIZED_BRANCH_NAME.json criterion-results/
+ cp target/criterion/$NORMALIZED_BRANCH_NAME.json criterion-results/$GITHUB_SHA.json
+
+ # TODO: can we optionally download if it only might fail?
+ #- name: PR; download bench results for compare
+ # if: github.event.inputs.pr_number
+ # uses: actions/github-script@v6
+ # with:
+ # github-token: ${{secrets.GITHUB_TOKEN}}
+ # script: |
+ # try {
+ # let artifact = github.rest.actions.getArtifact({
+ # owner: "clockwork",
+ # repo: "SpacetimeDB",
+ #
+ # })
+ # }
+
+ - name: PR; run bench
+ if: github.event.inputs.pr_number
+ run: |
+ echo "Running benchmarks without sqlite"
+ # have to pass explicit names, otherwise it will try to run the tests and fail for some reason...
+ pushd crates/bench
+ cargo bench --bench generic --bench special -- --save-baseline branch '(special|stdb_module|stdb_raw)'
+ cargo run --bin summarize pack branch
+ popd
+ mkdir criterion-results
+ cp target/criterion/branch.json criterion-results/pr-$PR_NUMBER.json
+
+ - name: PR; compare benchmarks
+ if: github.event.inputs.pr_number
+ working-directory: crates/bench/
+ run: |
+ if [ -e target/criterion/$NORMALIZED_BRANCH_NAME.json ]; then
+ cargo run --bin summarize markdown-report branch.json $NORMALIZED_BRANCH_NAME.json --report-name report
+ else
+ cargo run --bin summarize markdown-report branch.json --report-name report
+ fi
+
+ # this will work for both PR and master
+ - name: Upload criterion results to DO spaces
+ uses: shallwefootball/s3-upload-action@master
with:
- cwd: "crates/bench"
- branchName: ${{ env.PR_BASE_REF }}
+ aws_key_id: ${{ secrets.AWS_KEY_ID }}
+ aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+ aws_bucket: "spacetimedb-ci-benchmarks"
+ source_dir: criterion-results
+ endpoint: https://nyc3.digitaloceanspaces.com
+ destination_dir: benchmarks
+
+ - name: Fetch markdown summary PR
+ if: github.event.inputs.pr_number
+ run: |
+ curl -sS https://benchmarks.spacetimedb.com/compare/master/pr-$PR_NUMBER > report.md
+
+ - name: Fetch markdown summary PR
+ if: "! github.event.inputs.pr_number"
+ run: |
+ git fetch
+ old=$(git rev-parse HEAD~1)
+ curl -sS https://benchmarks.spacetimedb.com/compare/$old/$GITHUB_SHA > report.md
- - name: Criterion compare previous commit
- if: env.PR_BASE_REF == ''
- uses: clockworklabs/criterion-compare-action@main
+ # https://stackoverflow.com/questions/58066966/commenting-a-pull-request-in-a-github-action
+ # https://github.com/boa-dev/criterion-compare-action/blob/main/main.js
+ - name: test comment
+ uses: actions/github-script@v6
with:
- cwd: "crates/bench"
- branchName: "HEAD~1"
+ github-token: ${{secrets.GITHUB_TOKEN}}
+ script: |
+ let stuff = require('fs').readFileSync('report.md', 'utf8');
+ let body = `Benchmark results
\n\n${stuff}\n\n `;
+
+ try {
+ if (process.env.PR_NUMBER) {
+ let number = parseInt(process.env.PR_NUMBER);
+ core.info("context: issue number: "+number)
+ const { data: comment } = await github.rest.issues.createComment({
+ owner: "clockworklabs",
+ repo: "SpacetimeDB",
+ issue_number: number,
+ body: body,
+ });
+ core.info(
+ `Created comment id '${comment.id}' on issue '${number}' in 'clockworklabs/SpacetimeDB'.`
+ );
+ core.setOutput("comment-id", comment.id);
+ } else {
+ const { data: comment } = github.rest.repos.createCommitComment({
+ commit_sha: context.sha,
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ body: body
+ })
+ core.info(
+ `Created comment id '${comment.id}' on commit '${context.sha}' in 'clockworklabs/SpacetimeDB'.`
+ );
+ core.setOutput("comment-id", comment.id);
+ }
+ } catch (err) {
+ core.warning(`Failed to comment: ${err}`);
+ core.info("Commenting is not possible from forks.");
+ core.info("Logging here instead.");
+ console.log(body);
+ }
+
- name: Clean up
if: always()
diff --git a/Cargo.lock b/Cargo.lock
index e52cc8445b..964b697198 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4140,6 +4140,7 @@ dependencies = [
"lazy_static",
"log",
"rand 0.8.5",
+ "regex",
"rusqlite",
"serde",
"serde_json",
@@ -4151,6 +4152,7 @@ dependencies = [
"tempdir",
"tokio",
"tracing-subscriber",
+ "walkdir",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 4b739603f0..fff3265697 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -175,6 +175,7 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
url = "2.3.1"
urlencoding = "2.1.2"
uuid = { version = "1.2.1", features = ["v4"] }
+walkdir = "2.2.5"
wasmbin = "0.6"
# wasmer prior to 4.1.1 had a signal handling bug on macOS.
diff --git a/crates/bench/Cargo.toml b/crates/bench/Cargo.toml
index 27de5ded99..44add0d529 100644
--- a/crates/bench/Cargo.toml
+++ b/crates/bench/Cargo.toml
@@ -13,6 +13,9 @@ harness = false
name = "generic"
harness = false
+[[bin]]
+name = "summarize"
+
[lib]
bench = false
@@ -39,3 +42,5 @@ byte-unit.workspace = true
futures.workspace = true
tracing-subscriber.workspace = true
lazy_static.workspace = true
+walkdir.workspace = true
+regex.workspace = true
diff --git a/crates/bench/README.md b/crates/bench/README.md
index 506c266434..cf55ce7323 100644
--- a/crates/bench/README.md
+++ b/crates/bench/README.md
@@ -57,6 +57,27 @@ cargo bench -- 'mem/.*/unique'
```
Will run benchmarks involving unique primary keys against all databases, without writing to disc.
+## Pretty report
+To generate a nicely formatted markdown report, you can use the "summarize" binary.
+This is used on CI (see [`../../.github/workflows/benchmarks.yml`](../../.github/workflows/benchmarks.yml)).
+
+To generate a report without comparisons, use:
+```bash
+cargo bench --bench generic --bench special -- --save-baseline current
+cargo run --bin summarize markdown-report current
+```
+
+To compare to another branch, do:
+```bash
+git checkout master
+cargo bench --bench generic --bench special -- --save-baseline base
+git checkout high-octane-feature-branch
+cargo bench --bench generic --bench special -- --save-baseline current
+cargo run --bin summarize markdown-report current base
+```
+
+Of course, this will take about an hour, so it might be better to let the CI do it for you.
+
## Adding more
There are two ways to write benchmarks:
@@ -64,6 +85,8 @@ There are two ways to write benchmarks:
- Targeted, non-generic benchmarks (`benches/special.rs`)
- Generic over database backends (`benches/generic.rs`)
+See the following sections for how to write these.
+
### Targeted benchmarks
These are regular [Criterion.rs](https://github.com/bheisler/criterion.rs) benchmarks. Nothing fancy, do whatever you like with these. Put them in `benches/special.rs`.
@@ -95,12 +118,6 @@ There are also some scripts that rely on external tools to extract data from the
cargo install critcmp
```
-The simplest way to use critcmp is to save two baselines with Criterion's benchmark harness and then compare them. For example:
-```bash
-cargo bench -- --save-baseline before
-cargo bench -- --save-baseline change
-critcmp before change
-```
### OSX Only
diff --git a/crates/bench/benches/special.rs b/crates/bench/benches/special.rs
index 9b20865786..19baf8bec7 100644
--- a/crates/bench/benches/special.rs
+++ b/crates/bench/benches/special.rs
@@ -47,7 +47,7 @@ fn custom_module_benchmarks(c: &mut Criterion) {
fn serialize_benchmarks(c: &mut Criterion) {
let name = T::name_snake_case();
let count = 100;
- let mut group = c.benchmark_group("serialize");
+ let mut group = c.benchmark_group("special/serialize");
group.throughput(criterion::Throughput::Elements(count));
let data = create_sequential::(0xdeadbeef, count as u32, 100);
diff --git a/crates/bench/clippy.toml b/crates/bench/clippy.toml
new file mode 100644
index 0000000000..44bb81fab1
--- /dev/null
+++ b/crates/bench/clippy.toml
@@ -0,0 +1,2 @@
+# we use println in summarize.rs, don't complain about it
+disallowed-macros = []
diff --git a/crates/bench/src/bin/summarize.rs b/crates/bench/src/bin/summarize.rs
new file mode 100644
index 0000000000..aa1692a3e6
--- /dev/null
+++ b/crates/bench/src/bin/summarize.rs
@@ -0,0 +1,736 @@
+//! Script to summarize benchmark results in a pretty markdown table / json file / push to prometheus.
+
+use std::{
+ collections::HashSet,
+ fmt::Write as FmtWrite,
+ io::Write,
+ path::{Path, PathBuf},
+};
+
+use anyhow::Result;
+use clap::{Parser, Subcommand};
+use regex::{Captures, Regex};
+
+/// Helper script to pack / summarize Criterion benchmark results.
+#[derive(Parser)]
+struct Cli {
+ /// The path to the target directory where Criterion's benchmark data is stored.
+ /// Uses the location of the `summarize` executable by default.
+ #[arg(long = "target-dir")]
+ target_dir: Option,
+
+ #[command(subcommand)]
+ command: Command,
+}
+
+#[derive(Subcommand, Clone)]
+enum Command {
+ /// Pack a criterion baseline to a single json file.
+ /// This is used to store baselines in CI.
+ Pack {
+ /// The name of the baseline to pack.
+ #[arg(default_value = "base")]
+ baseline: String,
+ },
+ /// Use packed json files to generate a markdown report
+ /// suitable for posting in a github PR.
+ MarkdownReport {
+ /// The name of the new baseline to compare.
+ /// End with ".json" to load from a packed JSON file in `{target_dir}/criterion`.
+ /// Otherwise, read from the loose criterion files in the filesystem.
+ baseline_new: String,
+
+ /// The name of the old baseline to compare against.
+ /// End with ".json" to load from a packed JSON file in `{target_dir}/criterion`.
+ /// Otherwise, read from the loose criterion files in the filesystem.
+ baseline_old: Option,
+
+ /// Report will be written to `{target_dir}/criterion/{report_name}.md`.
+ #[arg(long = "report-name", required = false)]
+ report_name: Option,
+ },
+}
+
+fn main() {
+ let args = Cli::parse();
+
+ let target_dir = if let Some(target_dir) = args.target_dir {
+ let target_dir = std::path::PathBuf::from(target_dir);
+ assert!(
+ target_dir.exists(),
+ "target directory {} does not exist, set a different target directory with \
+ --target-dir or the CARGO_TARGET_DIR env var",
+ target_dir.display()
+ );
+ target_dir
+ } else {
+ let mut target_dir = std::env::current_exe().expect("no executable path?");
+ target_dir.pop();
+ target_dir.pop();
+ target_dir
+ };
+
+ let crit_dir = target_dir.clone().join("criterion");
+ assert!(
+ crit_dir.exists(),
+ "criterion directory {} inside target directory {} does not exist, \
+ set a different target directory with --target-dir or the CARGO_TARGET_DIR env var",
+ crit_dir.display(),
+ target_dir.display()
+ );
+
+ let benchmarks = data::Benchmarks::gather(&crit_dir).expect("failed to read benchmarks");
+
+ match args.command {
+ Command::Pack {
+ baseline: baseline_name,
+ } => {
+ assert!(
+ !baseline_name.ends_with(".json"),
+ "it's pointless to re-pack an already packed baseline..."
+ );
+ let baseline = benchmarks.by_baseline.get(&baseline_name).expect("baseline not found");
+
+ let path = packed_baseline_json_path(&crit_dir, &baseline_name);
+ let mut file = std::fs::File::create(&path).expect("failed to create file");
+ serde_json::to_writer_pretty(&mut file, baseline).expect("failed to write json");
+ println!("Wrote {}", path.display());
+ }
+ Command::MarkdownReport {
+ baseline_old: baseline_old_name,
+ baseline_new: baseline_new_name,
+ report_name,
+ } => {
+ let old = baseline_old_name
+ .map(|name| load_baseline(&benchmarks, &crit_dir, &name))
+ .transpose()
+ .expect("failed to load old baseline")
+ .unwrap_or_else(|| data::BaseBenchmarks {
+ name: "n/a".to_string(),
+ benchmarks: Default::default(),
+ });
+
+ let new = load_baseline(&benchmarks, &crit_dir, &baseline_new_name).expect("failed to load new baseline");
+
+ let report = generate_markdown_report(old, new).expect("failed to generate markdown report");
+
+ if let Some(report_name) = report_name {
+ let path = crit_dir.join(format!("{}.md", report_name));
+ let mut file = std::fs::File::create(&path).expect("failed to create file");
+ file.write_all(report.as_bytes()).expect("failed to write report");
+ println!("Wrote {}", path.display());
+ } else {
+ println!("{}", report);
+ }
+ }
+ }
+}
+
+/// `"{crit_dir}/"{name}.json"`
+fn packed_baseline_json_path(crit_dir: &Path, name: &str) -> PathBuf {
+ crit_dir.join(format!("{}.json", name))
+}
+
+/// If name ends with ".json", load from a packed json file. Otherwise, load from the benchmarks read from the filesystem.
+fn load_baseline(benchmarks: &data::Benchmarks, crit_dir: &Path, name: &str) -> Result {
+ if name.ends_with(".json") {
+ load_packed_baseline(crit_dir, name)
+ } else {
+ benchmarks
+ .by_baseline
+ .get(name)
+ .cloned()
+ .ok_or_else(|| anyhow::anyhow!("baseline {} not found", name))
+ }
+}
+
+fn load_packed_baseline(crit_dir: &Path, name: &str) -> Result {
+ assert!(name.ends_with(".json"));
+ let name = name.trim_end_matches(".json");
+ let path = packed_baseline_json_path(crit_dir, name);
+ let file = std::fs::File::open(path)?;
+ let baseline = serde_json::from_reader(file)?;
+ Ok(baseline)
+}
+
+fn generate_markdown_report(old: data::BaseBenchmarks, new: data::BaseBenchmarks) -> Result {
+ let mut result = String::new();
+
+ writeln!(&mut result, "# Benchmark Report")?;
+ writeln!(&mut result)?;
+
+ writeln!(
+ &mut result,
+ "Legend:
+
+- `load`: number of rows pre-loaded into the database
+- `count`: number of rows touched by the transaction
+- index types:
+ - `unique`: a single index on the `id` column
+ - `non_unique`: no indexes
+ - `multi_index`: non-unique index on every column
+- schemas:
+ - `person(id: u32, name: String, age: u64)`
+ - `location(id: u32, x: u64, y: u64)`
+
+All throughputs are single-threaded.
+
+ "
+ )?;
+
+ let remaining = old
+ .benchmarks
+ .keys()
+ .chain(new.benchmarks.keys())
+ .collect::>();
+ let mut remaining = remaining.into_iter().collect::>();
+ remaining.sort();
+
+ writeln!(&mut result, "## Empty transaction")?;
+ let table = extract_benchmarks_to_table(
+ r"(?x) (?P[^/]+) / (?P[^/]+) / empty",
+ &old,
+ &new,
+ &mut remaining,
+ )?;
+ writeln!(&mut result, "{table}")?;
+
+ writeln!(&mut result, "## Single-row insertions")?;
+ let table = extract_benchmarks_to_table(
+ r"(?x) (?P[^/]+) / (?P[^/]+) / insert_1 /
+ (?P[^/]+) / (?P[^/]+) /
+ load = (?P[^/]+)",
+ &old,
+ &new,
+ &mut remaining,
+ )?;
+ writeln!(&mut result, "{table}")?;
+
+ writeln!(&mut result, "## Multi-row insertions")?;
+ let table = extract_benchmarks_to_table(
+ r"(?x) (?P[^/]+) / (?P[^/]+) / insert_bulk /
+ (?P[^/]+) / (?P[^/]+) /
+ load = (?P[^/]+) / count = (?P[^/]+)",
+ &old,
+ &new,
+ &mut remaining,
+ )?;
+ writeln!(&mut result, "{table}")?;
+
+ writeln!(&mut result, "## Full table iterate")?;
+ let table = extract_benchmarks_to_table(
+ r"(?x) (?P[^/]+) / (?P[^/]+) / iterate /
+ (?P[^/]+) / (?P[^/]+) ",
+ &old,
+ &new,
+ &mut remaining,
+ )?;
+ writeln!(&mut result, "{table}")?;
+
+ writeln!(&mut result, "## Find unique key")?;
+ let table = extract_benchmarks_to_table(
+ r"(?x) (?P[^/]+) / (?P[^/]+) / find_unique /
+ (?P[^/]+) /
+ load = (?P[^/]+) ",
+ &old,
+ &new,
+ &mut remaining,
+ )?;
+ writeln!(&mut result, "{table}")?;
+
+ writeln!(&mut result, "## Filter")?;
+ let table = extract_benchmarks_to_table(
+ r"(?x) (?P[^/]+) / (?P[^/]+) / filter /
+ (?P[^/]+) / (?P[^/]+) /
+ load = (?P[^/]+) / count = (?P[^/]+)",
+ &old,
+ &new,
+ &mut remaining,
+ )?;
+ writeln!(&mut result, "{table}")?;
+
+ writeln!(&mut result, "## Serialize")?;
+ let table = extract_benchmarks_to_table(
+ r"(?x) serialize / (?P[^/]+) / (?P[^/]+) /
+ count = (?P[^/]+)",
+ &old,
+ &new,
+ &mut remaining,
+ )?;
+ writeln!(&mut result, "{table}")?;
+
+ writeln!(&mut result, "## Module: invoke with large arguments")?;
+ let table = extract_benchmarks_to_table(
+ r"(?x) stdb_module / large_arguments / (?P[^/]+)",
+ &old,
+ &new,
+ &mut remaining,
+ )?;
+ writeln!(&mut result, "{table}")?;
+
+ writeln!(&mut result, "## Module: print bulk")?;
+ let table = extract_benchmarks_to_table(
+ r"(?x) stdb_module / print_bulk / lines = (?P[^/]+)",
+ &old,
+ &new,
+ &mut remaining,
+ )?;
+ writeln!(&mut result, "{table}")?;
+
+ // catch-all for remaining benchmarks
+ writeln!(&mut result, "## Remaining benchmarks")?;
+ let table = extract_benchmarks_to_table(r"(?x) (?P.+)", &old, &new, &mut remaining)?;
+ writeln!(&mut result, "{table}")?;
+
+ assert_eq!(remaining.len(), 0);
+
+ Ok(result)
+}
+
+/// A given benchmark group fits a pattern such as
+/// `[db]/[disk]/insert_1/[schema]/[index_type]/load=[load]`
+///
+/// Pass a regex using named capture groups to extract all such benchmarks to a table.
+/// We use insignificant whitespace to make these easier to read.
+/// For example:
+///
+/// `r"(?x) (?P[^/]+) / (?P[^/]+) / insert_1 / (?P[^/]+) / (?P[^/]+) / load = (?P[^/]+)"`
+///
+/// Some strings are treated specially:
+/// - `disk -> 💿`, `mem -> 🧠`
+fn extract_benchmarks_to_table(
+ pattern: &str,
+ old: &data::BaseBenchmarks,
+ new: &data::BaseBenchmarks,
+ remaining: &mut Vec<&String>,
+) -> Result {
+ let regex = regex::Regex::new(pattern)?;
+
+ let mut capture_names: Vec<_> = regex.capture_names().map(|name| name.unwrap_or("")).collect();
+ capture_names.remove(0); // thi
+
+ let mut headers = capture_names
+ .clone()
+ .iter()
+ .map(|s| s.replace('_', " "))
+ .collect::>();
+ headers.push("new latency".to_string());
+ headers.push("old latency".to_string());
+ headers.push("new throughput".to_string());
+ headers.push("old throughput".to_string());
+
+ let mut rows = Vec::new();
+
+ let mut extracted = HashSet::new();
+
+ for (i, bench_name) in remaining.iter().enumerate() {
+ let captures = if let Some(captures) = regex.captures(bench_name) {
+ extracted.insert(i);
+ captures
+ } else {
+ continue;
+ };
+
+ let mut row = Vec::new();
+ for capture in &capture_names {
+ let cell = captures.name(capture).unwrap().as_str();
+
+ row.push(emojify(cell));
+ }
+
+ if let Some(new) = new.benchmarks.get(&**bench_name) {
+ row.push(time(new.nanoseconds(), new.stddev()))
+ } else {
+ row.push("-".to_string());
+ }
+
+ if let Some(old) = old.benchmarks.get(&**bench_name) {
+ row.push(time(old.nanoseconds(), old.stddev()))
+ } else {
+ row.push("-".to_string());
+ }
+
+ if let Some(new) = new.benchmarks.get(&**bench_name) {
+ if let Some(data::Throughput::Elements(throughput)) = new.throughput() {
+ row.push(throughput_per(throughput, "tx"))
+ } else {
+ row.push("-".to_string());
+ }
+ } else {
+ row.push("-".to_string());
+ }
+
+ if let Some(old) = old.benchmarks.get(&**bench_name) {
+ if let Some(data::Throughput::Elements(throughput)) = old.throughput() {
+ row.push(throughput_per(throughput, "tx"))
+ } else {
+ row.push("-".to_string());
+ }
+ } else {
+ row.push("-".to_string());
+ }
+
+ rows.push(row)
+ }
+
+ rows.sort();
+
+ *remaining = remaining
+ .iter()
+ .enumerate()
+ .filter_map(|(i, s)| if extracted.contains(&i) { None } else { Some(*s) })
+ .collect();
+
+ Ok(format_markdown_table(headers, rows))
+}
+
+fn format_markdown_table(headers: Vec, rows: Vec>) -> String {
+ for row in &rows {
+ assert_eq!(row.len(), headers.len(), "internal error: mismatched row lengths");
+ }
+
+ let mut result = "\n".to_string();
+
+ let mut max_widths = headers.iter().map(|s| s.len()).collect::>();
+ for row in &rows {
+ for (i, cell) in row.iter().enumerate() {
+ max_widths[i] = max_widths[i].max(cell.len());
+ }
+ }
+
+ result.push_str("| ");
+ for (i, header) in headers.iter().enumerate() {
+ result.push_str(&format!("{:width$} | ", header, width = max_widths[i]));
+ }
+ result.push('\n');
+
+ result.push('|');
+ for max_width in &max_widths {
+ result.push_str(&format!("-{:- String {
+ const MIN_MICRO: f64 = 2_000.0;
+ const MIN_MILLI: f64 = 2_000_000.0;
+ const MIN_SEC: f64 = 2_000_000_000.0;
+
+ let (div, label) = if nanos < MIN_MICRO {
+ (1.0, "ns")
+ } else if nanos < MIN_MILLI {
+ (1_000.0, "µs")
+ } else if nanos < MIN_SEC {
+ (1_000_000.0, "ms")
+ } else {
+ (1_000_000_000.0, "s")
+ };
+ format!("{:.1}±{:.2}{}", nanos / div, stddev / div, label)
+}
+
+fn throughput_per(per: f64, unit: &str) -> String {
+ const MIN_K: f64 = (2 * (1 << 10) as u64) as f64;
+ const MIN_M: f64 = (2 * (1 << 20) as u64) as f64;
+ const MIN_G: f64 = (2 * (1 << 30) as u64) as f64;
+
+ if per < MIN_K {
+ format!("{} {}/sec", per as u64, unit)
+ } else if per < MIN_M {
+ format!("{:.1} K{}/sec", (per / (1 << 10) as f64), unit)
+ } else if per < MIN_G {
+ format!("{:.1} M{}/sec", (per / (1 << 20) as f64), unit)
+ } else {
+ format!("{:.1} G{}/sec", (per / (1 << 30) as f64), unit)
+ }
+}
+
+lazy_static::lazy_static! {
+ static ref EMOJI: Regex = Regex::new(r"(on_disk|mem)").unwrap();
+}
+
+fn emojify(text: &str) -> String {
+ EMOJI
+ .replace_all(text, |cap: &Captures| match &cap[0] {
+ "disk" => "💿",
+ "mem" => "🧠",
+ _ => unimplemented!(),
+ })
+ .to_string()
+}
+
+/// Data types for deserializing stored Criterion benchmark results.
+///
+/// Unfortunately, there is no published library for this, so we use the schema
+/// from `critcmp` under the MIT license:
+/// https://github.com/BurntSushi/critcmp/blob/daaf0383c3981c98a6eaaef47142755e5bddb3c4/src/data.rs
+///
+/// TODO(jgilles): update this if we update our Criterion version past 0.4.
+#[allow(unused)]
+#[allow(clippy::all)]
+#[allow(rust_2018_idioms)]
+mod data {
+ /*
+ The MIT License (MIT)
+
+ Copyright (c) 2015 Andrew Gallant
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+ use std::collections::BTreeMap;
+ use std::fs::File;
+ use std::io;
+ use std::path::Path;
+
+ use serde::de::DeserializeOwned;
+ use serde::{Deserialize, Serialize};
+ use serde_json as json;
+ use walkdir::WalkDir;
+
+ // NOTE(jgilles): added this to make this compile
+ use anyhow::{anyhow as err, bail as fail, Result};
+
+ #[derive(Clone, Debug, Default)]
+ pub struct Benchmarks {
+ pub by_baseline: BTreeMap,
+ }
+
+ #[derive(Clone, Debug, Deserialize, Serialize)]
+ pub struct BaseBenchmarks {
+ pub name: String,
+ pub benchmarks: BTreeMap,
+ }
+
+ #[derive(Clone, Debug, Deserialize, Serialize)]
+ pub struct Benchmark {
+ pub baseline: String,
+ pub fullname: String,
+ #[serde(rename = "criterion_benchmark_v1")]
+ pub info: CBenchmark,
+ #[serde(rename = "criterion_estimates_v1")]
+ pub estimates: CEstimates,
+ }
+
+ #[derive(Clone, Debug, Deserialize, Serialize)]
+ pub struct CBenchmark {
+ pub group_id: String,
+ pub function_id: Option,
+ pub value_str: Option,
+ pub throughput: Option,
+ pub full_id: String,
+ pub directory_name: String,
+ }
+
+ #[derive(Clone, Debug, Deserialize, Serialize)]
+ #[serde(rename_all = "PascalCase")]
+ pub struct CThroughput {
+ pub bytes: Option,
+ pub elements: Option,
+ }
+
+ #[derive(Clone, Debug, Deserialize, Serialize)]
+ pub struct CEstimates {
+ pub mean: CStats,
+ pub median: CStats,
+ pub median_abs_dev: CStats,
+ pub slope: Option,
+ pub std_dev: CStats,
+ }
+
+ #[derive(Clone, Debug, Deserialize, Serialize)]
+ pub struct CStats {
+ pub confidence_interval: CConfidenceInterval,
+ pub point_estimate: f64,
+ pub standard_error: f64,
+ }
+
+ #[derive(Clone, Debug, Deserialize, Serialize)]
+ pub struct CConfidenceInterval {
+ pub confidence_level: f64,
+ pub lower_bound: f64,
+ pub upper_bound: f64,
+ }
+
+ impl Benchmarks {
+ pub fn gather>(criterion_dir: P) -> Result {
+ let mut benchmarks = Benchmarks::default();
+ for result in WalkDir::new(criterion_dir) {
+ let dent = result?;
+ let b = match Benchmark::from_path(dent.path())? {
+ None => continue,
+ Some(b) => b,
+ };
+ benchmarks
+ .by_baseline
+ .entry(b.baseline.clone())
+ .or_insert_with(|| BaseBenchmarks {
+ name: b.baseline.clone(),
+ benchmarks: BTreeMap::new(),
+ })
+ .benchmarks
+ .insert(b.benchmark_name().to_string(), b);
+ }
+ Ok(benchmarks)
+ }
+ }
+
+ impl Benchmark {
+ fn from_path>(path: P) -> Result