From a0238b08f6e8609e9c5ec9ce91f49c9f74f1a560 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Mon, 27 Aug 2018 14:31:18 -0700 Subject: [PATCH] Introduce benchmarking using Criterion. --- Cargo.toml | 7 ++ README.md | 35 +++++++- benches/benches.rs | 15 ++++ benches/suites/mod.rs | 8 ++ benches/suites/progress.rs | 13 +++ benches/suites/progress_set.rs | 155 +++++++++++++++++++++++++++++++++ benches/suites/raft.rs | 68 +++++++++++++++ benches/suites/raw_node.rs | 22 +++++ src/lib.rs | 3 + src/raft.rs | 5 +- 10 files changed, 328 insertions(+), 3 deletions(-) create mode 100644 benches/benches.rs create mode 100644 benches/suites/mod.rs create mode 100644 benches/suites/progress.rs create mode 100644 benches/suites/progress_set.rs create mode 100644 benches/suites/raft.rs create mode 100644 benches/suites/raw_node.rs diff --git a/Cargo.toml b/Cargo.toml index 9c9fe7cbb..6388c279f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,13 @@ quick-error = "1.2.2" rand = "0.5.4" fxhash = "0.2.1" +[dev-dependencies] +criterion = ">0.2.4" + +[[bench]] +name = "benches" +harness = false + [badges] travis-ci = { repository = "pingcap/raft-rs" } diff --git a/README.md b/README.md index 7b2aac169..fdd594b55 100644 --- a/README.md +++ b/README.md @@ -38,20 +38,26 @@ Using `rustup` you can get started this way: ```bash rustup override set stable rustup toolchain install nightly +rustup component add clippy-review --nightly +rustup component add rustfmt-preview ``` In order to have your PR merged running the following must finish without error: ```bash -cargo +nightly test --features dev +cargo test --all && \ +cargo +nightly clippy --all && \ +cargo fmt --all -- --check ``` You may optionally want to install `cargo-watch` to allow for automated rebuilding while editing: ```bash -cargo watch -s "cargo check --features dev" +cargo watch -s "cargo check" ``` +### Modifying Protobufs + If proto file `eraftpb.proto` changed, run the command to regenerate `eraftpb.rs`: ```bash @@ -60,6 +66,31 @@ protoc proto/eraftpb.proto --rust_out=src You can check `Cargo.toml` to find which version of `protobuf-codegen` is required. +### Benchmarks + +We use [Criterion](https://github.com/japaric/criterion.rs) for benchmarking. + +> It's currently an ongoing effort to build an appropriate benchmarking suite. If you'd like to help out please let us know! [Interested?](https://github.com/pingcap/raft-rs/issues/109) + +You can run the benchmarks by installing `gnuplot` then running: + +```bash +cargo bench +``` + +You can check `target/criterion/report/index.html` for plots and charts relating to the benchmarks. + +You can check the performance between two branches: + +```bash +git checkout master +cargo bench --bench benches -- --save-baseline master +git checkout other +cargo bench --bench benches -- --baseline master +``` + +This will report relative increases or decreased for each benchmark. + ## Acknowledgments Thanks [etcd](https://github.com/coreos/etcd) for providing the amazing Go implementation! diff --git a/benches/benches.rs b/benches/benches.rs new file mode 100644 index 000000000..501fd1edf --- /dev/null +++ b/benches/benches.rs @@ -0,0 +1,15 @@ +#![allow(dead_code)] // Due to criterion we need this to avoid warnings. + +#[macro_use] +extern crate criterion; +extern crate raft; + +mod suites; +use suites::*; + +criterion_main!( + bench_raft, + bench_raw_node, + bench_progress, + bench_progress_set, +); diff --git a/benches/suites/mod.rs b/benches/suites/mod.rs new file mode 100644 index 000000000..f6e36f7ef --- /dev/null +++ b/benches/suites/mod.rs @@ -0,0 +1,8 @@ +mod raft; +pub use self::raft::*; +mod raw_node; +pub use self::raw_node::*; +mod progress; +pub use self::progress::*; +mod progress_set; +pub use self::progress_set::*; diff --git a/benches/suites/progress.rs b/benches/suites/progress.rs new file mode 100644 index 000000000..030ef506c --- /dev/null +++ b/benches/suites/progress.rs @@ -0,0 +1,13 @@ +use criterion::{Bencher, Criterion}; +use raft::Progress; + +criterion_group!(bench_progress, bench_progress_default); + +pub fn bench_progress_default(c: &mut Criterion) { + let bench = |b: &mut Bencher| { + // No setup. + b.iter(|| Progress::default()); + }; + + c.bench_function("Progress::default", bench); +} diff --git a/benches/suites/progress_set.rs b/benches/suites/progress_set.rs new file mode 100644 index 000000000..0b14e8ceb --- /dev/null +++ b/benches/suites/progress_set.rs @@ -0,0 +1,155 @@ +use criterion::{Bencher, Criterion}; +use raft::ProgressSet; + +criterion_group!( + bench_progress_set, + bench_progress_set_new, + bench_progress_set_insert_voter, + bench_progress_set_insert_learner, + bench_progress_set_promote_learner, + bench_progress_set_remove, + bench_progress_set_iter, + bench_progress_set_get, + bench_progress_set_nodes, +); + +fn quick_progress_set(voters: usize, learners: usize) -> ProgressSet { + let mut set = ProgressSet::new(voters, learners); + (0..voters).for_each(|id| { + set.insert_voter(id as u64, Default::default()).ok(); + }); + (voters..learners).for_each(|id| { + set.insert_learner(id as u64, Default::default()).ok(); + }); + set +} + +pub fn bench_progress_set_new(c: &mut Criterion) { + let bench = |voters, learners| { + move |b: &mut Bencher| { + // No setup. + b.iter(|| ProgressSet::new(voters, learners)); + } + }; + + c.bench_function("ProgressSet::new (0, 0)", bench(0, 0)); + c.bench_function("ProgressSet::new (3, 1)", bench(3, 1)); + c.bench_function("ProgressSet::new (5, 2)", bench(5, 2)); + c.bench_function("ProgressSet::new (7, 3)", bench(7, 3)); +} + +pub fn bench_progress_set_insert_voter(c: &mut Criterion) { + let bench = |voters, learners| { + move |b: &mut Bencher| { + let set = quick_progress_set(voters, learners); + b.iter(|| { + let mut set = set.clone(); + set.insert_voter(99, Default::default()).ok() + }); + } + }; + c.bench_function("ProgressSet::insert_voter (0, 0)", bench(0, 0)); + c.bench_function("ProgressSet::insert_voter (3, 1)", bench(3, 1)); + c.bench_function("ProgressSet::insert_voter (5, 2)", bench(5, 2)); + c.bench_function("ProgressSet::insert_voter (7, 3)", bench(7, 3)); +} + +pub fn bench_progress_set_insert_learner(c: &mut Criterion) { + let bench = |voters, learners| { + move |b: &mut Bencher| { + let set = quick_progress_set(voters, learners); + b.iter(|| { + let mut set = set.clone(); + set.insert_learner(99, Default::default()).ok() + }); + } + }; + c.bench_function("ProgressSet::insert_learner (0, 0)", bench(0, 0)); + c.bench_function("ProgressSet::insert_learner (3, 1)", bench(3, 1)); + c.bench_function("ProgressSet::insert_learner (5, 2)", bench(5, 2)); + c.bench_function("ProgressSet::insert_learner (7, 3)", bench(7, 3)); +} + +pub fn bench_progress_set_remove(c: &mut Criterion) { + let bench = |voters, learners| { + move |b: &mut Bencher| { + let set = quick_progress_set(voters, learners); + b.iter(|| { + let mut set = set.clone(); + set.remove(3) + }); + } + }; + c.bench_function("ProgressSet::remove (0, 0)", bench(0, 0)); + c.bench_function("ProgressSet::remove (3, 1)", bench(3, 1)); + c.bench_function("ProgressSet::remove (5, 2)", bench(5, 2)); + c.bench_function("ProgressSet::remove (7, 3)", bench(7, 3)); +} + +pub fn bench_progress_set_promote_learner(c: &mut Criterion) { + let bench = |voters, learners| { + move |b: &mut Bencher| { + let set = quick_progress_set(voters, learners); + b.iter(|| { + let mut set = set.clone(); + set.promote_learner(3) + }); + } + }; + c.bench_function("ProgressSet::promote (0, 0)", bench(0, 0)); + c.bench_function("ProgressSet::promote (3, 1)", bench(3, 1)); + c.bench_function("ProgressSet::promote (5, 2)", bench(5, 2)); + c.bench_function("ProgressSet::promote (7, 3)", bench(7, 3)); +} + +pub fn bench_progress_set_iter(c: &mut Criterion) { + let bench = |voters, learners| { + move |b: &mut Bencher| { + let set = quick_progress_set(voters, learners); + b.iter(|| { + let set = set.clone(); + let agg = set.iter().all(|_| true); + agg + }); + } + }; + c.bench_function("ProgressSet::iter (0, 0)", bench(0, 0)); + c.bench_function("ProgressSet::iter (3, 1)", bench(3, 1)); + c.bench_function("ProgressSet::iter (5, 2)", bench(5, 2)); + c.bench_function("ProgressSet::iter (7, 3)", bench(7, 3)); +} + +pub fn bench_progress_set_nodes(c: &mut Criterion) { + let bench = |voters, learners| { + move |b: &mut Bencher| { + let set = quick_progress_set(voters, learners); + b.iter(|| { + let set = set.clone(); + let agg = set.iter().all(|_| true); + agg + }); + } + }; + c.bench_function("ProgressSet::nodes (0, 0)", bench(0, 0)); + c.bench_function("ProgressSet::nodes (3, 1)", bench(3, 1)); + c.bench_function("ProgressSet::nodes (5, 2)", bench(5, 2)); + c.bench_function("ProgressSet::nodes (7, 3)", bench(7, 3)); +} + +pub fn bench_progress_set_get(c: &mut Criterion) { + let bench = |voters, learners| { + move |b: &mut Bencher| { + let set = quick_progress_set(voters, learners); + b.iter(|| { + let set = set.clone(); + { + set.get(1); + } + }); + } + }; + c.bench_function("ProgressSet::get (0, 0)", bench(0, 0)); + c.bench_function("ProgressSet::get (3, 1)", bench(3, 1)); + c.bench_function("ProgressSet::get (5, 2)", bench(5, 2)); + c.bench_function("ProgressSet::get (7, 3)", bench(7, 3)); +} diff --git a/benches/suites/raft.rs b/benches/suites/raft.rs new file mode 100644 index 000000000..55764edf0 --- /dev/null +++ b/benches/suites/raft.rs @@ -0,0 +1,68 @@ +use criterion::{Bencher, Criterion}; +use raft::{storage::MemStorage, Config, Raft}; + +criterion_group!(bench_raft, bench_raft_new, bench_raft_campaign,); + +fn quick_raft(voters: usize, learners: usize) -> Raft { + let id = 1; + let storage = MemStorage::default(); + let config = Config::new(id); + let mut raft = Raft::new(&config, storage); + (0..voters).for_each(|id| { + raft.add_node(id as u64); + }); + (voters..learners).for_each(|id| { + raft.add_learner(id as u64); + }); + raft +} + +pub fn bench_raft_new(c: &mut Criterion) { + let bench = |voters, learners| { + move |b: &mut Bencher| { + // No setup. + b.iter(|| quick_raft(voters, learners)); + } + }; + + c.bench_function("Raft::new (0, 0)", bench(0, 0)); + c.bench_function("Raft::new (3, 1)", bench(3, 1)); + c.bench_function("Raft::new (5, 2)", bench(5, 2)); + c.bench_function("Raft::new (7, 3)", bench(7, 3)); +} + +pub fn bench_raft_campaign(c: &mut Criterion) { + let bench = |voters, learners, variant| { + move |b: &mut Bencher| { + b.iter(|| { + // TODO: Make raft clone somehow. + let mut raft = quick_raft(voters, learners); + raft.campaign(variant) + }) + } + }; + + // We don't want to make `raft::raft` public at this point. + let pre_election = b"CampaignPreElection"; + let election = b"CampaignElection"; + let transfer = b"CampaignTransfer"; + + c.bench_function( + "Raft::campaign (3, 1, pre_election)", + bench(3, 1, &pre_election[..]), + ); + c.bench_function("Raft::campaign (3, 1, election)", bench(3, 1, &election[..])); + c.bench_function("Raft::campaign (3, 1, transfer)", bench(3, 1, &transfer[..])); + c.bench_function( + "Raft::campaign (5, 2, pre_election)", + bench(5, 2, &pre_election[..]), + ); + c.bench_function("Raft::campaign (5, 2, election)", bench(5, 2, &election[..])); + c.bench_function("Raft::campaign (5, 2, transfer)", bench(5, 2, &transfer[..])); + c.bench_function( + "Raft::campaign (7, 3, pre_election)", + bench(7, 3, &pre_election[..]), + ); + c.bench_function("Raft::campaign (7, 3, election)", bench(7, 3, &election[..])); + c.bench_function("Raft::campaign (7, 3, transfer)", bench(7, 3, &transfer[..])); +} diff --git a/benches/suites/raw_node.rs b/benches/suites/raw_node.rs new file mode 100644 index 000000000..bbc55d0f3 --- /dev/null +++ b/benches/suites/raw_node.rs @@ -0,0 +1,22 @@ +use criterion::{Bencher, Criterion}; +use raft::{storage::MemStorage, Config, RawNode}; + +criterion_group!(bench_raw_node, bench_raw_node_new,); + +fn quick_raw_node() -> RawNode { + let id = 1; + let peers = vec![]; + let storage = MemStorage::default(); + let config = Config::new(id); + let node = RawNode::new(&config, storage, peers).unwrap(); + node +} + +pub fn bench_raw_node_new(c: &mut Criterion) { + let bench = |b: &mut Bencher| { + // No setup. + b.iter(|| quick_raw_node()); + }; + + c.bench_function("RawNode::new", bench); +} diff --git a/src/lib.rs b/src/lib.rs index 1343d6325..d1a61f6c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -232,6 +232,9 @@ pub mod eraftpb; mod errors; mod log_unstable; mod progress; +#[cfg(test)] +pub mod raft; +#[cfg(not(test))] mod raft; mod raft_log; pub mod raw_node; diff --git a/src/raft.rs b/src/raft.rs index 4f4f62823..799d2d20b 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -780,7 +780,10 @@ impl Raft { .count() } - fn campaign(&mut self, campaign_type: &[u8]) { + /// Campaign to attempt to become a leader. + /// + /// If prevote is enabled, this is handled as well. + pub fn campaign(&mut self, campaign_type: &[u8]) { let (vote_msg, term) = if campaign_type == CAMPAIGN_PRE_ELECTION { self.become_pre_candidate(); // Pre-vote RPCs are sent for next term before we've incremented self.term.