diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ae6285..ba05b1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,25 @@ # Changelog +## [0.3.1](https://github.com/Blobfolio/brunch/releases/tag/v0.3.1) - 2022-08-23 + +### Changes + +* Coloration tweaks; +* Improved float handling; +* Suppress _Change_ summary column when there aren't any; +* Warn if the same `Bench` name is submitted twice; + +### Fixed + +* env `NO_BRUNCH_HISTORY` should only apply when `=1`; +* Normalize whitespace in `Bench` names to prevent display weirdness; + + + ## [0.3.0](https://github.com/Blobfolio/brunch/releases/tag/v0.3.0) - 2022-08-22 -This release includes a number of improvements to the `Brunch` API, but as a result, existing benchmarks will need a few (minor) changes when migrating from `0.2.x` to `0.3.x`. +This release includes a number of improvements to the `Brunch` API, but also some **breaking changes**. Existing benchmarks will require a few (minor) adjustments when migrating from `0.2.x` to `0.3.x`. First and foremost, `Bench::new` has been streamlined, and now takes the name as a single argument (rather than two). When migrating, just glue the two values back together, e.g. `"foo::bar", "baz(20)"` to `"foo::bar::baz(20)"`. diff --git a/CREDITS.md b/CREDITS.md index dd8748c..a6a0cf3 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -1,7 +1,7 @@ # Project Dependencies Package: brunch - Version: 0.3.0 - Generated: 2022-08-22 19:57:13 UTC + Version: 0.3.1 + Generated: 2022-08-23 19:16:04 UTC | Package | Version | Author(s) | License | | ---- | ---- | ---- | ---- | diff --git a/Cargo.toml b/Cargo.toml index ae384e5..3f085f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brunch" -version = "0.3.0" +version = "0.3.1" authors = ["Blobfolio, LLC. "] edition = "2021" rust-version = "1.61" diff --git a/README.md b/README.md index b1cc992..7b00989 100644 --- a/README.md +++ b/README.md @@ -4,18 +4,18 @@ [![crates.io](https://img.shields.io/crates/v/brunch.svg)](https://crates.io/crates/brunch) [![Build Status](https://github.com/Blobfolio/brunch/workflows/Build/badge.svg)](https://github.com/Blobfolio/brunch/actions) [![Dependency Status](https://deps.rs/repo/github/blobfolio/brunch/status.svg)](https://deps.rs/repo/github/blobfolio/brunch) - +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](https://github.com/Blobfolio/brunch) `Brunch` is a very simple Rust micro-benchmark runner inspired by [`easybench`](https://crates.io/crates/easybench). It has roughly a million times fewer dependencies than [`criterion`](https://crates.io/crates/criterion), does not require nightly, and maintains a "last run" state so can show relative changes benchmark-to-benchmark. -The formatting is also quite pretty. +(The formatting is also quite pretty.) As with all Rust benchmarking, there are a lot of caveats, and results might be artificially fast or slow. For best results: -* build optimized; -* collect lots of samples; -* repeat identical runs to get a feel for the natural variation; +* Build optimized; +* Collect lots of samples; +* Repeat identical runs to get a feel for the natural variation; `Brunch` cannot measure time below the level of a nanosecond, so if you're trying to benchmark methods that are _really_ fast, you may need to wrap them in a method that runs through several iterations at once. For example: @@ -77,7 +77,7 @@ harness = false The following optional environmental variables are supported: -* `NO_BRUNCH_HISTORY`: don't save or load run-to-run history data; +* `NO_BRUNCH_HISTORY=1`: don't save or load run-to-run history data; * `BRUNCH_DIR=/some/directory`: save run-to-run history data to this folder instead of `std::env::temp_dir`; diff --git a/src/bench.rs b/src/bench.rs index c2e248e..53927fa 100644 --- a/src/bench.rs +++ b/src/bench.rs @@ -77,7 +77,13 @@ impl Benches { /// // Repeat push as needed. /// benches.finish(); /// ``` - pub fn push(&mut self, b: Bench) { self.0.push(b); } + pub fn push(&mut self, mut b: Bench) { + if ! b.is_spacer() && self.has_name(&b.name) { + b.stats.replace(Err(BrunchError::DupeName)); + } + + self.0.push(b); + } /// # Finish. /// @@ -105,8 +111,14 @@ impl Benches { // Build the summaries. let mut history = History::default(); let mut summary = Table::default(); + let names: Vec> = self.0.iter() + .filter_map(|b| + if b.is_spacer() { None } + else { Some(b.name.chars().collect()) } + ) + .collect(); for b in &self.0 { - summary.push(b, &history); + summary.push(b, &names, &history); } // Update the history. @@ -129,6 +141,13 @@ impl Benches { } } +impl Benches { + /// # Has Name. + fn has_name(&self, name: &str) -> bool { + self.0.iter().any(|b| b.name == name) + } +} + #[derive(Debug)] @@ -175,8 +194,27 @@ impl Bench { let name = name.as_ref().trim(); assert!(! name.is_empty(), "Name is required."); + // Compact and normalize whitespace, but otherwise pass whatever the + // name is on through. + let mut ws = false; + let name = name.chars() + .filter_map(|c| + if c.is_whitespace() { + if ws { None } + else { + ws = true; + Some(' ') + } + } + else { + ws = false; + Some(c) + } + ) + .collect(); + Self { - name: name.to_owned(), + name, samples: DEFAULT_SAMPLES, timeout: DEFAULT_TIMEOUT, stats: None, @@ -426,10 +464,10 @@ impl Default for Table { fn default() -> Self { Self(vec![ TableRow::Normal( - "\x1b[1;38;5;13mMethod".to_owned(), + "\x1b[1;95mMethod".to_owned(), "Mean".to_owned(), - "Change".to_owned(), - "Samples\x1b[0m".to_owned() + "Samples".to_owned(), + "Change\x1b[0m".to_owned(), ), TableRow::Spacer, ]) @@ -440,13 +478,17 @@ impl fmt::Display for Table { #[allow(clippy::many_single_char_names)] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // Maximum column widths. - let (w1, w2, w3, w4) = self.lens(); + let (w1, w2, w3, mut w4) = self.lens(); + let changes = self.show_changes(); + let width = + if changes { w1 + w2 + w3 + w4 + 12 } + else { + w4 = 0; + w1 + w2 + w3 + 8 + }; // Pre-generate the full-width spacer content. - let spacer = format!( - "\x1b[38;5;5m{}\x1b[0m\n", - "-".repeat(w1 + w2 + w3 + w4 + 12) - ); + let spacer = format!("\x1b[35m{}\x1b[0m\n", "-".repeat(width)); // Pre-generate padding too. We'll slice this to size each time padding // is needed. @@ -456,13 +498,19 @@ impl fmt::Display for Table { for v in &self.0 { let (c1, c2, c3, c4) = v.lens(); match v { - TableRow::Normal(a, b, c, d) => writeln!( + TableRow::Normal(a, b, c, d) if changes => writeln!( f, "{}{} {}{} {}{} {}{}", a, &pad[..w1 - c1], &pad[..w2 - c2], b, &pad[..w3 - c3], c, &pad[..w4 - c4], d, )?, + TableRow::Normal(a, b, c, _) => writeln!( + f, "{}{} {}{} {}{}", + a, &pad[..w1 - c1], + &pad[..w2 - c2], b, + &pad[..w3 - c3], c, + )?, TableRow::Error(a, b) => writeln!( f, "{}{} \x1b[1;38;5;208m{}\x1b[0m", a, &pad[..w1 - c1], b, @@ -477,23 +525,24 @@ impl fmt::Display for Table { impl Table { /// # Add Row. - fn push(&mut self, src: &Bench, history: &History) { + fn push(&mut self, src: &Bench, names: &[Vec], history: &History) { if src.is_spacer() { self.0.push(TableRow::Spacer); } else { - let name = format_name(&src.name); + let name = format_name(src.name.chars().collect(), names); match src.stats.unwrap_or(Err(BrunchError::NoRun)) { Ok(s) => { - let time = util::format_time(s.mean); + let time = s.nice_mean(); let diff = history.get(&src.name) .and_then(|h| s.is_deviant(h)) .unwrap_or_else(|| NO_CHANGE.to_owned()); + let (valid, total) = s.samples(); let samples = format!( - "\x1b[2m{}\x1b[0;38;5;5m/\x1b[0;2m{}\x1b[0m", - NiceU64::from(s.valid), - NiceU64::from(s.total), + "\x1b[2m{}\x1b[0;35m/\x1b[0;2m{}\x1b[0m", + NiceU64::from(valid), + NiceU64::from(total), ); - self.0.push(TableRow::Normal(name, time, diff, samples)); + self.0.push(TableRow::Normal(name, time, samples, diff)); }, Err(e) => { self.0.push(TableRow::Error(name, e)); @@ -502,6 +551,16 @@ impl Table { } } + /// # Has Changes? + /// + /// Returns true if any of the Change columns have a value. + fn show_changes(&self) -> bool { + self.0.iter().skip(2).any(|v| + if let TableRow::Normal(_, _, _, c) = v { c != NO_CHANGE } + else { false } + ) + } + /// # Widths. fn lens(&self) -> (usize, usize, usize, usize) { self.0.iter() @@ -553,22 +612,39 @@ impl TableRow { #[allow(clippy::option_if_let_else)] /// # Format Name. /// -/// Style up a benchmark name. -fn format_name(name: &str) -> String { - // Last opening parenthesis? - if let Some(pos) = name.rfind('(') { - // Is there a namespace thing behind it? - if let Some(pos2) = name[..pos].rfind("::") { - format!("\x1b[2m{}::\x1b[0m{}", &name[..pos2], &name[pos2 + 2..]) - } - else { - format!("\x1b[2m{}\x1b[0m{}", &name[..pos], &name[pos..]) - } +/// Style up a benchmark name by dimming common portions, and highlighting +/// unique ones. +/// +/// This approach won't scale well, but the bench count for any given set +/// should be relatively low. +fn format_name(mut name: Vec, names: &[Vec]) -> String { + // Find the first unique char occurrence. + let pos: usize = names.iter() + .filter_map(|other| + if name.eq(other) { None } + else { + name.iter() + .zip(other.iter()) + .position(|(l, r)| l != r) + .or_else(|| Some(name.len().min(other.len()))) + } + ) + .max() + .unwrap_or_default(); + + if pos == 0 { + "\x1b[94m".chars() + .chain(name.into_iter()) + .chain("\x1b[0m".chars()) + .collect() } - // Last namespace thing? - else if let Some(pos) = name.rfind("::") { - format!("\x1b[2m{}::\x1b[0m{}", &name[..pos], &name[pos + 2..]) + else { + let b = name.split_off(pos); + "\x1b[34m".chars() + .chain(name.into_iter()) + .chain("\x1b[94m".chars()) + .chain(b.into_iter()) + .chain("\x1b[0m".chars()) + .collect() } - // Leave it boring. - else { ["\x1b[2m", name, "\x1b[0m"].concat() } } diff --git a/src/error.rs b/src/error.rs index 29fab22..b56bdab 100644 --- a/src/error.rs +++ b/src/error.rs @@ -12,6 +12,9 @@ use std::fmt; /// /// This enum serves as the custom error type for `Brunch`. pub enum BrunchError { + /// # Duplicate name. + DupeName, + /// # No benches were specified. NoBench, @@ -36,8 +39,9 @@ impl std::error::Error for BrunchError {} impl fmt::Display for BrunchError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::DupeName => f.write_str("Benchmark names must be unique."), Self::NoBench => f.write_str("At least one benchmark is required."), - Self::NoRun => f.write_str("Missing \x1b[1;38;5;14mBench::run\x1b[0m."), + Self::NoRun => f.write_str("Missing \x1b[1;96mBench::run\x1b[0m."), Self::Overflow => f.write_str("Unable to crunch the numbers."), Self::TooFast => f.write_str("Too fast to benchmark!"), Self::TooSmall(n) => write!( diff --git a/src/lib.rs b/src/lib.rs index cd65ef1..b1a84af 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,18 +5,18 @@ [![crates.io](https://img.shields.io/crates/v/brunch.svg)](https://crates.io/crates/brunch) [![Build Status](https://github.com/Blobfolio/brunch/workflows/Build/badge.svg)](https://github.com/Blobfolio/brunch/actions) [![Dependency Status](https://deps.rs/repo/github/blobfolio/brunch/status.svg)](https://deps.rs/repo/github/blobfolio/brunch) - +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](https://github.com/Blobfolio/brunch) `Brunch` is a very simple Rust micro-benchmark runner inspired by [`easybench`](https://crates.io/crates/easybench). It has roughly a million times fewer dependencies than [`criterion`](https://crates.io/crates/criterion), does not require nightly, and maintains a "last run" state so can show relative changes benchmark-to-benchmark. -The formatting is also quite pretty. +(The formatting is also quite pretty.) As with all Rust benchmarking, there are a lot of caveats, and results might be artificially fast or slow. For best results: -* build optimized; -* collect lots of samples; -* repeat identical runs to get a feel for the natural variation; +* Build optimized; +* Collect lots of samples; +* Repeat identical runs to get a feel for the natural variation; `Brunch` cannot measure time below the level of a nanosecond, so if you're trying to benchmark methods that are _really_ fast, you may need to wrap them in a method that runs through several iterations at once. For example: @@ -71,7 +71,7 @@ harness = false The following optional environmental variables are supported: -* `NO_BRUNCH_HISTORY`: don't save or load run-to-run history data; +* `NO_BRUNCH_HISTORY=1`: don't save or load run-to-run history data; * `BRUNCH_DIR=/some/directory`: save run-to-run history data to this folder instead of [`std::env::temp_dir`]; diff --git a/src/macros.rs b/src/macros.rs index 3264e99..552d872 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -38,7 +38,7 @@ macro_rules! benches { // Run the benches. let mut benches = $crate::Benches::default(); $( - ::std::eprint!("\x1b[1;38;5;4m•\x1b[0m"); + ::std::eprint!("\x1b[1;34m•\x1b[0m"); benches.push($benches); )+ ::std::eprintln!("\n"); diff --git a/src/stats.rs b/src/stats.rs index b8133f3..76c149b 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -5,8 +5,13 @@ use crate::{ BrunchError, MIN_SAMPLES, + util, }; -use dactyl::NicePercent; +use dactyl::{ + NicePercent, + NiceU32, +}; +use num_traits::FromPrimitive; use quantogram::Quantogram; use serde::{ de, @@ -88,16 +93,16 @@ impl History { /// # Runtime Stats! pub(crate) struct Stats { /// # Total Samples. - pub(crate) total: usize, + total: usize, /// # Valid Samples. - pub(crate) valid: usize, + valid: usize, /// # Standard Deviation. deviation: f64, /// # Mean Duration of Valid Samples. - pub(crate) mean: f64, + mean: f64, } impl<'de> Deserialize<'de> for Stats { @@ -238,31 +243,43 @@ impl TryFrom> for Stats { let mut q = Quantogram::new(); q.add_unweighted_samples(samples.iter()); - // Grab the deviation of the full set. (This is a better representation - // of variation than if we pulled it only from the valid set.) - let deviation = q.stddev().ok_or(BrunchError::Overflow)?; - - // Determine outlier range (+- 5%). - let q1 = q.fussy_quantile(0.05, 2.0).ok_or(BrunchError::Overflow)?; - let q3 = q.fussy_quantile(0.95, 2.0).ok_or(BrunchError::Overflow)?; - let iqr = q3 - q1; - - // Low and high boundaries. - let lo = iqr.mul_add(-1.5, q1); - let hi = iqr.mul_add(1.5, q3); - - samples.retain(|&s| lo <= s && s <= hi); - - let valid = samples.len(); - if valid < MIN_SAMPLES { return Err(BrunchError::TooWild); } - - // Find the new mean. - q = Quantogram::new(); - q.add_unweighted_samples(samples.iter()); - let mean = q.mean().ok_or(BrunchError::Overflow)?; + // Grab the deviation of the full set. + let mut deviation = q.stddev().ok_or(BrunchError::Overflow)?; + let mut valid = total; + let mean = + // No deviation means no outliers. + if util::float_eq(deviation, 0.0) { + q.mean().ok_or(BrunchError::Overflow)? + } + // Weed out the weirdos. + else { + // Determine outlier range (+- 5%). + let q1 = q.fussy_quantile(0.05, 2.0).ok_or(BrunchError::Overflow)?; + let q3 = q.fussy_quantile(0.95, 2.0).ok_or(BrunchError::Overflow)?; + let iqr = q3 - q1; + + // Low and high boundaries. + let lo = iqr.mul_add(-1.5, q1); + let hi = iqr.mul_add(1.5, q3); + + // Remove outliers. + samples.retain(|&s| util::float_le(lo, s) && util::float_le(s, hi)); + + valid = samples.len(); + if valid < MIN_SAMPLES { return Err(BrunchError::TooWild); } + + // Find the new mean. + q = Quantogram::new(); + q.add_unweighted_samples(samples.iter()); + let mean = q.mean().ok_or(BrunchError::Overflow)?; + deviation = q.stddev().ok_or(BrunchError::Overflow)?; + mean + }; // Done! - Ok(Self{ total, valid, deviation, mean }) + let out = Self { total, valid, deviation, mean }; + if out.is_valid() { Ok(out) } + else { Err(BrunchError::Overflow) } } } @@ -277,8 +294,8 @@ impl Stats { pub(crate) fn is_deviant(self, other: Self) -> Option { let dev = 2.0 * self.deviation; if - matches!(other.mean.total_cmp(&(self.mean - dev)), Ordering::Less) || - matches!(other.mean.total_cmp(&(self.mean + dev)), Ordering::Greater) + util::float_lt(other.mean, self.mean - dev) || + util::float_gt(other.mean, self.mean + dev) { let (color, sign, diff) = match self.mean.total_cmp(&other.mean) { Ordering::Less => (92, "-", other.mean - self.mean), @@ -297,14 +314,46 @@ impl Stats { None } + /// # Nice Mean. + /// + /// Return the mean rescaled to the most appropriate unit. + pub(crate) fn nice_mean(self) -> String { + let mut mean = self.mean; + let unit: &str = + if util::float_lt(mean, 0.000_001) { + mean *= 1_000_000_000.000; + "ns" + } + else if util::float_lt(mean, 0.001) { + mean *= 1_000_000.000; + "\u{3bc}s" + } + else if util::float_lt(mean, 1.0) { + mean *= 1_000.000; + "ms" + } + else { "s " }; + + // Convert the whole and fractional parts to integers. + let trunc = u32::from_f64(mean.trunc()).unwrap_or_default(); + let fract = u8::from_f64((mean.fract() * 100.0).trunc()).unwrap_or_default(); + + format!("\x1b[0;1m{}.{:02} {}\x1b[0m", NiceU32::from(trunc), fract, unit) + } + + /// # Samples. + /// + /// Return the valid/total samples. + pub(crate) const fn samples(self) -> (usize, usize) { (self.valid, self.total) } + /// # Is Valid? fn is_valid(self) -> bool { MIN_SAMPLES <= self.valid && self.valid <= self.total && self.deviation.is_finite() && - matches!(self.deviation.total_cmp(&0.0), Ordering::Equal | Ordering::Greater) && + util::float_ge(self.deviation, 0.0) && self.mean.is_finite() && - matches!(self.mean.total_cmp(&0.0), Ordering::Equal | Ordering::Greater) + util::float_ge(self.mean, 0.0) } } @@ -314,7 +363,7 @@ impl Stats { /// /// Return the file path history should be written to or read from. fn history_path() -> Option { - if std::env::var_os("NO_BRUNCH_HISTORY").is_some() { None } + if std::env::var("NO_BRUNCH_HISTORY").map_or(false, |s| s.trim() == "1") { None } else { let mut p = try_dir(std::env::var_os("BRUNCH_DIR")) .or_else(|| try_dir(Some(std::env::temp_dir())))?; @@ -363,14 +412,12 @@ mod tests { // Make sure we end up where we began. assert_eq!(stat.total, d.total, "Deserialization changed total."); assert_eq!(stat.valid, d.valid, "Deserialization changed valid."); - assert_eq!( - stat.deviation.total_cmp(&d.deviation), - Ordering::Equal, + assert!( + util::float_eq(stat.deviation, d.deviation), "Deserialization changed deviation." ); - assert_eq!( - stat.mean.total_cmp(&d.mean), - Ordering::Equal, + assert!( + util::float_eq(stat.mean, d.mean), "Deserialization changed mean." ); } diff --git a/src/util.rs b/src/util.rs index ef1091b..bb45c5f 100644 --- a/src/util.rs +++ b/src/util.rs @@ -2,8 +2,7 @@ # Brunch: Utility Functions */ -use dactyl::NiceU32; -use num_traits::FromPrimitive; +use std::cmp::Ordering; use unicode_width::UnicodeWidthChar; @@ -26,38 +25,29 @@ pub(crate) fn black_box(dummy: T) -> T { } } -/// # Format w/ Unit. -/// -/// Give us a nice comma-separated integer with two decimal places and an -/// appropriate unit (running from pico seconds to milliseconds). -pub(crate) fn format_time(mut time: f64) -> String { - let unit: &str = - if time < 0.000_001 { - time *= 1_000_000_000.000; - "ns" - } - else if time < 0.001 { - time *= 1_000_000.000; - "\u{3bc}s" - } - else if time < 1.0 { - time *= 1_000.000; - "ms" - } - else if time < 60.0 { - "s " - } - else { - time /= 60.0; - "m " - }; - - format!( - "\x1b[1m{}.{:02} {}\x1b[0m", - NiceU32::from(u32::from_f64(time.trunc()).unwrap_or_default()).as_str(), - u32::from_f64(f64::floor(time.fract() * 100.0)).unwrap_or_default(), - unit - ) +/// # Float < Float. +pub(crate) fn float_lt(a: f64, b: f64) -> bool { + matches!(a.total_cmp(&b), Ordering::Less) +} + +/// # Float <= Float. +pub(crate) fn float_le(a: f64, b: f64) -> bool { + matches!(a.total_cmp(&b), Ordering::Less | Ordering::Equal) +} + +/// # Float == Float. +pub(crate) fn float_eq(a: f64, b: f64) -> bool { + matches!(a.total_cmp(&b), Ordering::Equal) +} + +/// # Float >= Float. +pub(crate) fn float_ge(a: f64, b: f64) -> bool { + matches!(a.total_cmp(&b), Ordering::Equal | Ordering::Greater) +} + +/// # Float > Float. +pub(crate) fn float_gt(a: f64, b: f64) -> bool { + matches!(a.total_cmp(&b), Ordering::Greater) } /// # Width. @@ -84,3 +74,30 @@ pub(crate) fn width(src: &str) -> usize { } }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cmp_float() { + assert!(float_lt(5.0, 10.0)); + assert!(! float_lt(10.0, 10.0)); + assert!(! float_lt(11.0, 10.0)); + + assert!(float_le(5.0, 10.0)); + assert!(float_le(10.0, 10.0)); + assert!(! float_le(11.0, 10.0)); + + assert!(float_eq(5.0, 5.0)); + assert!(! float_eq(5.0, 5.00000001)); + + assert!(float_ge(15.0, 10.0)); + assert!(float_ge(10.0, 10.0)); + assert!(! float_ge(9.999, 10.0)); + + assert!(float_gt(15.0, 10.0)); + assert!(! float_gt(10.0, 10.0)); + assert!(! float_gt(9.999, 10.0)); + } +}