Fix iteration problem with criterion, pre-creation of database outsid…

…e bench for hyperfine

crates/bench/benches/db.rs

@@ -1,81 +1,110 @@
 //! Benchmarks for evaluating how we fare against sqlite
 
 use criterion::measurement::WallTime;
-use criterion::{
-    criterion_group, criterion_main, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput,
-};
+use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, Throughput};
 use spacetimedb_bench::prelude::*;
 use std::time::Duration;
 
+// IMPORTANT!: It needs this option to run the setup once per `.iter`!
+const SIZE: BatchSize = BatchSize::PerIteration;
+
 fn build_group<'a>(c: &'a mut Criterion, named: &str, run: Runs) -> BenchmarkGroup<'a, WallTime> {
     let mut group = c.benchmark_group(named);
-    // We need to restrict the amount of iterations and set the benchmark for "large" operations.
+
     group.throughput(Throughput::Elements(run as u64));
     group.sample_size(DB_POOL as usize);
-    group.sampling_mode(SamplingMode::Flat);
+    group.measurement_time(Duration::from_secs(5));
+
     group
 }
 
 fn bench_insert_tx_per_row(c: &mut Criterion) {
     let run = Runs::Tiny;
     let mut group = build_group(c, "insert_row", run);
 
+    // Factor out the db creation because is IO that generate noise
     group.bench_function(BenchmarkId::new(SQLITE, 1), |b| {
-        let mut db_instance = 0;
         b.iter_batched(
-            || {
-                let path = sqlite::create_db(db_instance).unwrap();
-                db_instance += 1;
-                path
-            },
-            |data| {
-                let mut conn = sqlite::open_conn(&data).unwrap();
+            || sqlite::create_db(0).unwrap(),
+            |path| {
+                let mut conn = sqlite::open_conn(&path).unwrap();
                 sqlite::insert_tx_per_row(&mut conn, run).unwrap();
             },
-            BatchSize::NumBatches(DB_POOL as u64),
+            SIZE,
+        );
+    });
+    group.bench_function(BenchmarkId::new(SPACETIME, 1), |b| {
+        b.iter_batched(
+            || spacetime::create_db(0).unwrap(),
+            |path| {
+                let (conn, _, table_id) = spacetime::open_conn(&path).unwrap();
+                spacetime::insert_tx_per_row(&conn, table_id, run).unwrap();
+            },
+            SIZE,
+        );
+    });
+
+    group.finish();
+}
+
+fn bench_insert_tx(c: &mut Criterion) {
+    let run = Runs::Small;
+    let mut group = build_group(c, "insert_bulk_rows", run);
+
+    // Factor out the db creation because is IO that generate noise
+    group.bench_function(BenchmarkId::new(SQLITE, 2), |b| {
+        b.iter_batched(
+            || sqlite::create_db(0).unwrap(),
+            |path| {
+                let mut conn = sqlite::open_conn(&path).unwrap();
+                sqlite::insert_tx(&mut conn, run).unwrap();
+            },
+            SIZE,
+        );
+    });
+    group.bench_function(BenchmarkId::new(SPACETIME, 2), |b| {
+        b.iter_batched(
+            || spacetime::create_db(0).unwrap(),
+            |path| {
+                let (conn, _, table_id) = spacetime::open_conn(&path).unwrap();
+                spacetime::insert_tx(&conn, table_id, run).unwrap();
+            },
+            SIZE,
+        );
+    });
+
+    group.finish();
+}
+
+fn bench_select_no_index(c: &mut Criterion) {
+    let run = Runs::Tiny;
+    let mut group = build_group(c, "select_index_no", run);
+
+    // Factor out the db creation because is IO that generate noise
+    group.bench_function(BenchmarkId::new(SQLITE, 3), |b| {
+        b.iter_batched(
+            || sqlite::create_db(0).unwrap(),
+            |path| {
+                let mut conn = sqlite::open_conn(&path).unwrap();
+                sqlite::select_no_index(&mut conn, run).unwrap();
+            },
+            SIZE,
+        );
+    });
+    group.bench_function(BenchmarkId::new(SPACETIME, 3), |b| {
+        b.iter_batched(
+            || spacetime::create_db(0).unwrap(),
+            |path| {
+                let (conn, _, table_id) = spacetime::open_conn(&path).unwrap();
+                spacetime::select_no_index(&conn, table_id, run).unwrap();
+            },
+            SIZE,
         );
     });
-    // group.bench_function(BenchmarkId::new(SPACETIME, 1), |b| {
-    //     let mut pool = Pool::new(false).unwrap();
-    //     b.iter_with_setup(|| spacetime::insert_tx_per_row(&mut pool, run).unwrap())
-    // });
 
     group.finish();
 }
-//
-// fn bench_insert_tx(c: &mut Criterion) {
-//     let run = Runs::Small;
-//     let mut group = build_group(c, "insert_bulk_rows", run);
-//
-//     group.bench_function(BenchmarkId::new(SQLITE, 2), |b| {
-//         let mut pool = Pool::new(true).unwrap();
-//         b.iter(|| sqlite::insert_tx(&mut pool, run))
-//     });
-//     group.bench_function(BenchmarkId::new(SPACETIME, 2), |b| {
-//         let mut pool = Pool::new(true).unwrap();
-//         b.iter(|| spacetime::insert_tx(&mut pool, run))
-//     });
-//
-//     group.finish();
-// }
-//
-// fn bench_select_no_index(c: &mut Criterion) {
-//     let run = Runs::Tiny;
-//     let mut group = build_group(c, "select_index_no", run);
-//
-//     group.bench_function(BenchmarkId::new(SQLITE, 3), |b| {
-//         let mut pool = Pool::new(true).unwrap();
-//         b.iter(|| sqlite::select_no_index(&mut pool, run).unwrap())
-//     });
-//     group.bench_function(BenchmarkId::new(SPACETIME, 3), |b| {
-//         let mut pool = Pool::new(true).unwrap();
-//         b.iter(|| spacetime::select_no_index(&mut pool, run).unwrap())
-//     });
-//
-//     group.finish();
-// }
 
-// Note: Reflex this same benchmarks in `main.rs`
-//, bench_insert_tx, bench_select_no_index
-criterion_group!(benches, bench_insert_tx_per_row);
+// Note: Mirror this benchmarks in `main.rs`
+criterion_group!(benches, bench_insert_tx_per_row, bench_insert_tx, bench_select_no_index);
 criterion_main!(benches);

crates/bench/flamegraph.sh

@@ -11,5 +11,11 @@ cd "$(dirname "$0")"
 # sqlite vs spacetime
 cargo build --release
 bench="../../target/release/spacetimedb-bench"
+# How many Dbs to create
+total_create=1
+
+$bench --db spacetime create-db $total_create
+$bench --db sqlite create-db $total_create
+
 cargo flamegraph --deterministic --notes "sqlite ${1}"     -o sqlite.svg    -- --db sqlite ${1}
 cargo flamegraph --deterministic --notes "spacetime ${1}"  -o spacetime.svg -- --db spacetime ${1}

crates/bench/hyperfine.sh

@@ -12,9 +12,12 @@ cd "$(dirname "$0")"
 cargo build --release
 bench="../../target/release/spacetimedb-bench"
 total_dbs=10
+total_warmup=5
+# How many Dbs to create, total_dbs + total_warmup
+total_create=15
 
-$bench --db spacetime create-db $total_dbs
-$bench --db sqlite create-db $total_dbs
+$bench --db spacetime create-db $total_create
+$bench --db sqlite create-db $total_create
 
 # Add --show-output to see errors...
-hyperfine --show-output  --parameter-scan db 0 $total_dbs --shell=none --export-json out.json --warmup 5 --runs $total_dbs "${bench} --db spacetime ${1} {db}" "${bench} --db sqlite ${1} {db}"
+hyperfine --shell=none --export-json out.json --warmup $total_warmup --runs $total_dbs "${bench} --db spacetime ${1}" "${bench} --db sqlite ${1}"

crates/bench/src/main.rs

@@ -1,4 +1,6 @@
 use clap::{Parser, Subcommand};
+use rusqlite::Connection;
+use spacetimedb::db::relational_db::RelationalDB;
 use spacetimedb_bench::prelude::*;
 
 /// Bencher for SpacetimeDB
@@ -16,24 +18,18 @@ struct Cli {
 enum Commands {
     /// Generate insert, each generate a transaction
     Insert {
-        /// Which DB to select ie: 1.db, 2.db, etc...
-        db: usize,
         /// How many rows
         #[arg(value_enum)]
         rows: Option<Runs>,
     },
     /// Generate insert in bulk enclosed in a single transaction
     InsertBulk {
-        /// Which DB to select ie: 1.db, 2.db, etc...
-        db: usize,
         /// How many rows
         #[arg(value_enum)]
         rows: Option<Runs>,
     },
     /// Run queries without a index
     SelectNoIndex {
-        /// Which DB to select ie: 1.db, 2.db, etc...
-        db: usize,
         /// How many rows
         #[arg(value_enum)]
         rows: Option<Runs>,
@@ -45,48 +41,75 @@ enum Commands {
     },
 }
 
-macro_rules! bench_fn {
-    ($cli:ident, $db:ident, $fun:ident, $run:expr) => {{
-        let run = $run;
-        let db_instance = $db;
-
-        match $cli.db {
-            DbEngine::Sqlite => {
-                let path = sqlite::db_path_instance(db_instance);
-                let mut conn = sqlite::open_conn(&path)?;
-                sqlite::$fun(&mut conn, run)
-            }
-            DbEngine::Spacetime => {
-                let path = spacetime::db_path(db_instance);
-                let (conn, _tmp_dir, table_id) = spacetime::open_conn(&path)?;
-                spacetime::$fun(&conn, table_id, run)
-            }
+fn bench_fn<Space, Sqlite>(cli: Cli, space: Space, sqlite: Sqlite, run: Runs) -> ResultBench<()>
+where
+    Space: Fn(&RelationalDB, u32, Runs) -> ResultBench<()>,
+    Sqlite: Fn(&mut Connection, Runs) -> ResultBench<()>,
+{
+    match cli.db {
+        DbEngine::Sqlite => {
+            let db_instance = sqlite::get_counter()?;
+            let path = sqlite::db_path_instance(db_instance);
+            let mut conn = sqlite::open_conn(&path)?;
+            sqlite(&mut conn, run)?;
+            sqlite::set_counter(db_instance + 1)?;
+            Ok(())
+        }
+        DbEngine::Spacetime => {
+            let db_instance = spacetime::get_counter()?;
+            let path = spacetime::db_path(db_instance);
+            let (conn, _tmp_dir, table_id) = spacetime::open_conn(&path)?;
+            space(&conn, table_id, run)?;
+            spacetime::set_counter(db_instance + 1)?;
+            Ok(())
         }
-    }};
+    }
 }
 
+/// The workflow for running the bench without interference of creation of the database on disk
+/// that is expensive and generate noise specially in the case of spacetime that create many folder/files is:
+///
+/// - Run `--db ENGINE create-db $total_create` for pre-create the dbs like `0.db, 1.db...$total_create`
+/// - Execute the bench with `--db spacetime NAME`
+///
+/// For picking which db to use, this hack is implemented:
+///
+/// - Save with `set_counter(0)` a file with the start of the "iteration"
+/// - Load with `get_counter()` the file with the current `db id`
+/// - Execute the bench
+/// - Save the `db id` of the next iteration with `set_counter(db_id + 1)`
+///
+/// NOTE: This is workaround for `hyperfine` where is not possible to know which run is the one this command is invoked
+/// see https://github.com/sharkdp/hyperfine/issues/667
 fn main() -> ResultBench<()> {
+    // Note: Mirror this benchmarks in `benches/db.rs`
     let cli = Cli::parse();
-
     match cli.command {
-        Commands::Insert { db, rows } => {
-            bench_fn!(cli, db, insert_tx_per_row, rows.unwrap_or(Runs::Tiny))
-        }
-        Commands::InsertBulk { db, rows } => {
-            // bench_fn!(cli, insert_tx, rows.unwrap_or(Runs::Small), true)
-            Ok(())
-        }
-        Commands::SelectNoIndex { db, rows } => {
-            // bench_fn!(cli, select_no_index, rows.unwrap_or(Runs::Tiny), true)
-            Ok(())
-        }
+        Commands::Insert { rows } => bench_fn(
+            cli,
+            spacetime::insert_tx_per_row,
+            sqlite::insert_tx_per_row,
+            rows.unwrap_or(Runs::Tiny),
+        ),
+        Commands::InsertBulk { rows } => bench_fn(
+            cli,
+            spacetime::insert_tx,
+            sqlite::insert_tx,
+            rows.unwrap_or(Runs::Small),
+        ),
+        Commands::SelectNoIndex { rows } => bench_fn(
+            cli,
+            spacetime::select_no_index,
+            sqlite::select_no_index,
+            rows.unwrap_or(Runs::Tiny),
+        ),
         Commands::CreateDb { total_dbs } => match cli.db {
             DbEngine::Sqlite => {
-                sqlite::create_db(total_dbs)?;
+                sqlite::create_dbs(total_dbs)?;
                 Ok(())
             }
             DbEngine::Spacetime => {
-                spacetime::create_db(total_dbs)?;
+                spacetime::create_dbs(total_dbs)?;
                 Ok(())
             }
         },