-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: Single-shot benchmarking + continuous benchmarking (#183)
Setups `iai-callgrind`-based benchmarks for single-shot benchmarking (by using `callgrind` internally). Cleans up the benchmark cases and adds some structure so they can be executed both with `iai-callgrind` and `criterion`, depending on whether we want to measure instruction count or time. They can be called separatedly, ```bash # Single-shot, requires some extra setup cargo bench --bench iai_benches # Time-based, takes longer to run cargo bench --bench criterion_benches ``` See DEVELOPMENT.md for instructions. --- The instruction count benchmarks are now uploaded to [bencher.dev](https://bencher.dev/perf/portgraph), so we get an historical comparison of the performance and a CI check can alert about regressions. I believe the "No thresholds found" error in this PR will go away once this gets run in `main`. The service choice was mainly between bencher.dev and codspeed.io . I choose the former since it supports single-shot benchmarks natively. See this issue in `ratatui` where the Bencher maintainer discusses some differences, [ratatui/ratatui#1092](https://www.github.com/ratatui/ratatui/issues/1092#issuecomment-2415565274).
- Loading branch information
Showing
17 changed files
with
781 additions
and
309 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| name: Archive Bencher.dev PR benchmarks | ||
| on: | ||
| pull_request: | ||
| types: | ||
| - closed | ||
|
|
||
| jobs: | ||
| archive_pr_branch: | ||
| name: Archive closed PR branch with Bencher | ||
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: bencherdev/bencher@main | ||
| - name: Archive closed PR branch with Bencher | ||
| run: | | ||
| bencher archive \ | ||
| --project portgraph \ | ||
| --token '${{ secrets.BENCHER_API_TOKEN }}' \ | ||
| --branch "$GITHUB_HEAD_REF" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,68 +1,106 @@ | ||
| use criterion::{black_box, criterion_group, AxisScale, BenchmarkId, Criterion, PlotConfiguration}; | ||
| use criterion::{criterion_group, Criterion}; | ||
| use itertools::Itertools; | ||
| use portgraph::{algorithms::TopoConvexChecker, PortView}; | ||
| use portgraph::{NodeIndex, PortGraph}; | ||
|
|
||
| use super::generators::make_two_track_dag; | ||
|
|
||
| fn bench_convex_construction(c: &mut Criterion) { | ||
| let mut g = c.benchmark_group("initialize convex checker object"); | ||
| g.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic)); | ||
|
|
||
| for size in [100, 1_000, 10_000] { | ||
| g.bench_with_input( | ||
| BenchmarkId::new("initalize_convexity", size), | ||
| &size, | ||
| |b, size| { | ||
| let graph = make_two_track_dag(*size); | ||
| b.iter(|| black_box(TopoConvexChecker::new(&graph))) | ||
| }, | ||
| ); | ||
| use crate::helpers::*; | ||
|
|
||
| // ----------------------------------------------------------------------------- | ||
| // Benchmark functions | ||
| // ----------------------------------------------------------------------------- | ||
|
|
||
| struct ConvexConstruction { | ||
| graph: PortGraph, | ||
| } | ||
| impl SizedBenchmark for ConvexConstruction { | ||
| fn name() -> &'static str { | ||
| "initialize_convexity" | ||
| } | ||
|
|
||
| fn setup(size: usize) -> Self { | ||
| let graph = make_two_track_dag(size); | ||
| Self { graph } | ||
| } | ||
|
|
||
| fn run(&self) -> impl Sized { | ||
| TopoConvexChecker::new(&self.graph) | ||
| } | ||
| g.finish(); | ||
| } | ||
|
|
||
| /// We benchmark the worst case scenario, where the "subgraph" is the | ||
| /// entire graph itself. | ||
| fn bench_convex_full(c: &mut Criterion) { | ||
| let mut g = c.benchmark_group("Runtime convexity check. Full graph."); | ||
| g.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic)); | ||
| struct ConvexFull { | ||
| checker: TopoConvexChecker<PortGraph>, | ||
| nodes: Vec<NodeIndex>, | ||
| } | ||
| impl SizedBenchmark for ConvexFull { | ||
| fn name() -> &'static str { | ||
| "check_convexity_full" | ||
| } | ||
|
|
||
| for size in [100, 1_000, 10_000] { | ||
| fn setup(size: usize) -> Self { | ||
| let graph = make_two_track_dag(size); | ||
| let checker = TopoConvexChecker::new(&graph); | ||
| g.bench_with_input( | ||
| BenchmarkId::new("check_convexity_full", size), | ||
| &size, | ||
| |b, _size| b.iter(|| black_box(checker.is_node_convex(graph.nodes_iter()))), | ||
| ); | ||
| let nodes = graph.nodes_iter().collect_vec(); | ||
| let checker = TopoConvexChecker::new(graph); | ||
| Self { checker, nodes } | ||
| } | ||
|
|
||
| fn run(&self) -> impl Sized { | ||
| self.checker.is_node_convex(self.nodes.iter().copied()) | ||
| } | ||
| g.finish(); | ||
| } | ||
|
|
||
| /// We benchmark the an scenario where the size of the "subgraph" is sub-linear on the size of the graph. | ||
| fn bench_convex_sparse(c: &mut Criterion) { | ||
| let mut g = c.benchmark_group("Runtime convexity check. Sparse subgraph on an n^2 size graph."); | ||
| g.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic)); | ||
|
|
||
| for size in [100usize, 1_000, 5_000] { | ||
| let graph_size = size.pow(2); | ||
| let graph = make_two_track_dag(graph_size); | ||
| let checker = TopoConvexChecker::new(&graph); | ||
| let nodes = graph.nodes_iter().step_by(graph_size / size).collect_vec(); | ||
| g.bench_with_input( | ||
| BenchmarkId::new("check_convexity_sparse", size), | ||
| &size, | ||
| |b, _size| b.iter(|| black_box(checker.is_node_convex(nodes.iter().copied()))), | ||
| ); | ||
| struct ConvexSparse { | ||
| checker: TopoConvexChecker<PortGraph>, | ||
| nodes: Vec<NodeIndex>, | ||
| } | ||
| impl SizedBenchmark for ConvexSparse { | ||
| fn name() -> &'static str { | ||
| "check_convexity_sparse" | ||
| } | ||
|
|
||
| fn setup(size: usize) -> Self { | ||
| let graph = make_two_track_dag(size); | ||
| let subgraph_size = (size as f64).sqrt().floor() as usize; | ||
| let nodes = graph | ||
| .nodes_iter() | ||
| .step_by(size / subgraph_size) | ||
| .collect_vec(); | ||
| let checker = TopoConvexChecker::new(graph); | ||
| Self { checker, nodes } | ||
| } | ||
|
|
||
| fn run(&self) -> impl Sized { | ||
| self.checker.is_node_convex(self.nodes.iter().copied()) | ||
| } | ||
| g.finish(); | ||
| } | ||
|
|
||
| // ----------------------------------------------------------------------------- | ||
| // iai_callgrind definitions | ||
| // ----------------------------------------------------------------------------- | ||
|
|
||
| sized_iai_benchmark!(callgrind_convex_construction, ConvexConstruction); | ||
| sized_iai_benchmark!(callgrind_convex_full, ConvexFull); | ||
| sized_iai_benchmark!(callgrind_convex_sparse, ConvexSparse); | ||
|
|
||
| iai_callgrind::library_benchmark_group!( | ||
| name = callgrind_group; | ||
| benchmarks = | ||
| callgrind_convex_construction, | ||
| callgrind_convex_full, | ||
| callgrind_convex_sparse, | ||
| ); | ||
|
|
||
| // ----------------------------------------------------------------------------- | ||
| // Criterion definitions | ||
| // ----------------------------------------------------------------------------- | ||
|
|
||
| criterion_group! { | ||
| name = benches; | ||
| name = criterion_group; | ||
| config = Criterion::default(); | ||
| targets = | ||
| bench_convex_full, | ||
| bench_convex_sparse, | ||
| bench_convex_construction | ||
| ConvexConstruction::criterion, | ||
| ConvexFull::criterion, | ||
| ConvexSparse::criterion, | ||
| } |
Oops, something went wrong.