Skip to content

Commit

Permalink
Put out benchmark fires (#404)
Browse files Browse the repository at this point in the history 
* Reverts benchmarks portion of "Bench for SQL scan / where (#370)"
This partially reverts commit 05956d6.

* Get benchmarks runner to remember what it has installed, hopefully

* Fix build error
  • Loading branch information
@kazimuth
kazimuth authored Oct 12, 2023
1 parent 68079ff commit cd88c21
Show file tree
Hide file tree
Showing 7 changed files with 40 additions and 352 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/benchmarks.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ jobs:
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
~/.cargo/.crates.toml
~/.cargo/.crates2.json
target/
key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }}

Expand Down
191 changes: 16 additions & 175 deletions crates/bench/benches/generic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ use criterion::{
measurement::{Measurement, WallTime},
Bencher, BenchmarkGroup, Criterion,
};
use spacetimedb::db::datastore::traits::TableSchema;
use spacetimedb_bench::{
database::BenchDatabase,
schemas::{create_sequential, BenchTable, IndexStrategy, Location, Person, RandomTable, BENCH_PKEY_INDEX},
Expand Down Expand Up @@ -70,16 +69,13 @@ fn table_suite<DB: BenchDatabase, T: BenchTable + RandomTable>(g: &mut Group, db
for (index_strategy, table_id, table_params) in &tables {
if *index_strategy == IndexStrategy::Unique {
iterate::<DB, T>(g, table_params, db, table_id, 100)?;
sql_select::<DB, T>(g, table_params, db, table_id, 100)?;

if table_params.contains("person") {
// perform "find" benchmarks
sql_find::<DB, T>(g, db, table_id, index_strategy, BENCH_PKEY_INDEX, 1000, 100)?;
find::<DB, T>(g, db, table_id, index_strategy, BENCH_PKEY_INDEX, 1000, 100)?;
}
} else {
// perform "filter" benchmarks
sql_where::<DB, T>(g, db, table_id, index_strategy, 1, 1000, 100)?;
filter::<DB, T>(g, db, table_id, index_strategy, 1, 1000, 100)?;
}
}
Expand Down Expand Up @@ -126,7 +122,7 @@ fn bench_harness<

#[inline(never)]
fn empty<DB: BenchDatabase>(g: &mut Group, db: &mut DB) -> ResultBench<()> {
let id = "empty".to_string();
let id = format!("empty");
g.bench_function(&id, |b| {
bench_harness(
b,
Expand Down Expand Up @@ -163,11 +159,11 @@ fn insert_1<DB: BenchDatabase, T: BenchTable + RandomTable>(
let mut data = data.clone();
db.clear_table(table_id)?;
let row = data.pop().unwrap();
db.insert_bulk(table_id, data)?;
db.insert_bulk(&table_id, data)?;
Ok(row)
},
|db, row| {
db.insert(table_id, row)?;
db.insert(&table_id, row)?;
Ok(())
},
)
Expand Down Expand Up @@ -200,46 +196,12 @@ fn insert_bulk<DB: BenchDatabase, T: BenchTable + RandomTable>(
db.clear_table(table_id)?;
let to_insert = data.split_off(load as usize);
if !data.is_empty() {
db.insert_bulk(table_id, data)?;
db.insert_bulk(&table_id, data)?;
}
Ok(to_insert)
},
|db, to_insert| {
db.insert_bulk(table_id, to_insert)?;
Ok(())
},
)
});
db.clear_table(table_id)?;
Ok(())
}

#[inline(never)]
fn sql_select<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
table_params: &str,
db: &mut DB,
table_id: &DB::TableId,
count: u32,
) -> ResultBench<()> {
let id = format!("sql_select/{table_params}/count={count}");
let data = create_sequential::<T>(0xdeadbeef, count, 1000);

db.insert_bulk(table_id, data)?;

// Each iteration performs a single transaction,
// though it iterates across many rows.
g.throughput(criterion::Throughput::Elements(1));

let table = db.get_table::<T>(table_id)?;

g.bench_function(&id, |b| {
bench_harness(
b,
db,
|_| Ok(()),
|db, _| {
db.sql_select(&table)?;
db.insert_bulk(&table_id, to_insert)?;
Ok(())
},
)
Expand Down Expand Up @@ -282,17 +244,15 @@ fn iterate<DB: BenchDatabase, T: BenchTable + RandomTable>(

/// Implements both "filter" and "find" benchmarks.
#[inline(never)]
#[allow(clippy::too_many_arguments)]
fn _filter_setup<DB: BenchDatabase, T: BenchTable + RandomTable>(
fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
bench_name: &str,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_index: u32,
load: u32,
buckets: u32,
) -> ResultBench<(String, TableSchema, Vec<T>)> {
) -> ResultBench<()> {
let filter_column_type = match T::product_type().elements[column_index as usize].algebraic_type {
AlgebraicType::String => "string",
AlgebraicType::U32 => "u32",
Expand All @@ -305,33 +265,15 @@ fn _filter_setup<DB: BenchDatabase, T: BenchTable + RandomTable>(
IndexStrategy::NonUnique => "non_indexed",
_ => unimplemented!(),
};
let id = format!("{bench_name}/{filter_column_type}/{indexed}/load={load}/count={mean_result_count}");
let id = format!("filter/{filter_column_type}/{indexed}/load={load}/count={mean_result_count}");

let data = create_sequential::<T>(0xdeadbeef, load, buckets as u64);

db.insert_bulk(table_id, data.clone())?;
db.insert_bulk(&table_id, data.clone())?;

// Each iteration performs a single transaction.
g.throughput(criterion::Throughput::Elements(1));

let table = db.get_table::<T>(table_id)?;

Ok((id, table, data))
}

#[inline(never)]
fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_index: u32,
load: u32,
buckets: u32,
) -> ResultBench<()> {
let (id, table, data) =
_filter_setup::<DB, T>(g, db, "filter", table_id, index_strategy, column_index, load, buckets)?;

// We loop through all buckets found in the sample data.
// This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
// Note that all databases have EXACTLY the same sample data.
Expand All @@ -348,53 +290,7 @@ fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
Ok(value)
},
|db, value| {
db.sql_where::<T>(&table, column_index, value)?;
Ok(())
},
)
});
db.clear_table(table_id)?;
Ok(())
}

#[inline(never)]
fn sql_where<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_index: u32,
load: u32,
buckets: u32,
) -> ResultBench<()> {
let (id, table, data) = _filter_setup::<DB, T>(
g,
db,
"sql_where",
table_id,
index_strategy,
column_index,
load,
buckets,
)?;

// We loop through all buckets found in the sample data.
// This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
// Note that all databases have EXACTLY the same sample data.
let mut i = 0;

g.bench_function(&id, |b| {
bench_harness(
b,
db,
|_| {
// pick something to look for
let value = data[i].clone().into_product_value().elements[column_index as usize].clone();
i = (i + 1) % load as usize;
Ok(value)
},
|db, value| {
db.sql_where::<T>(&table, column_index, value)?;
db.filter::<T>(&table_id, column_index, value)?;
Ok(())
},
)
Expand All @@ -405,84 +301,29 @@ fn sql_where<DB: BenchDatabase, T: BenchTable + RandomTable>(

/// Implements both "filter" and "find" benchmarks.
#[inline(never)]
fn _find_setup<DB: BenchDatabase, T: BenchTable + RandomTable>(
fn find<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
bench_name: &str,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_id: u32,
load: u32,
buckets: u32,
) -> ResultBench<(String, TableSchema, Vec<T>)> {
) -> ResultBench<()> {
assert_eq!(
*index_strategy,
IndexStrategy::Unique,
"find benchmarks require unique key"
);
let id = format!("{bench_name}/u32/load={load}");
let id = format!("find_unique/u32/load={load}");

let data = create_sequential::<T>(0xdeadbeef, load, buckets as u64);

db.insert_bulk(table_id, data.clone())?;
db.insert_bulk(&table_id, data.clone())?;

// Each iteration performs a single transaction.
g.throughput(criterion::Throughput::Elements(1));

let table = db.get_table::<T>(table_id)?;

Ok((id, table, data))
}

#[inline(never)]
fn find<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_id: u32,
load: u32,
buckets: u32,
) -> ResultBench<()> {
let (id, table, data) = _find_setup::<DB, T>(g, db, "find_unique", table_id, index_strategy, load, buckets)?;

// We loop through all buckets found in the sample data.
// This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
// Note that all benchmarks use exactly the same sample data.
let mut i = 0;

g.bench_function(&id, |b| {
bench_harness(
b,
db,
|_| {
let value = data[i].clone().into_product_value().elements[column_id as usize].clone();
i = (i + 1) % load as usize;
Ok(value)
},
|db, value| {
db.filter::<T>(&table, column_id, value)?;
Ok(())
},
)
});
db.clear_table(table_id)?;
Ok(())
}

/// Implements both "filter" and "find" benchmarks.
#[inline(never)]
fn sql_find<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_id: u32,
load: u32,
buckets: u32,
) -> ResultBench<()> {
let (id, table, data) =
_find_setup::<DB, T>(g, db, "sql_where_find_unique", table_id, index_strategy, load, buckets)?;

// We loop through all buckets found in the sample data.
// This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
// Note that all benchmarks use exactly the same sample data.
Expand All @@ -498,7 +339,7 @@ fn sql_find<DB: BenchDatabase, T: BenchTable + RandomTable>(
Ok(value)
},
|db, value| {
db.sql_where::<T>(&table, column_id, value)?;
db.filter::<T>(&table_id, column_id, value)?;
Ok(())
},
)
Expand Down
19 changes: 1 addition & 18 deletions crates/bench/src/database.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use spacetimedb::db::datastore::traits::TableSchema;
use spacetimedb_lib::AlgebraicValue;

use crate::schemas::{BenchTable, IndexStrategy};
Expand All @@ -20,9 +19,6 @@ pub trait BenchDatabase: Sized {

fn create_table<T: BenchTable>(&mut self, table_style: IndexStrategy) -> ResultBench<Self::TableId>;

/// Return table metadata so we can remove this from the hot path
fn get_table<T: BenchTable>(&mut self, table_id: &Self::TableId) -> ResultBench<TableSchema>;

/// Should not drop the table, only delete all the rows.
fn clear_table(&mut self, table_id: &Self::TableId) -> ResultBench<()>;

Expand All @@ -45,20 +41,7 @@ pub trait BenchDatabase: Sized {
/// Filter the table on the specified column index for the specified value.
fn filter<T: BenchTable>(
&mut self,
table: &TableSchema,
column_index: u32,
value: AlgebraicValue,
) -> ResultBench<()>;

/// Perform a `SELECT * FROM table`
/// Note: this can be non-generic because none of the implementations use the relevant generic argument.
fn sql_select(&mut self, table: &TableSchema) -> ResultBench<()>;

/// Perform a `SELECT * FROM table WHERE column = value`
/// Note: this can be non-generic because none of the implementations use the relevant generic argument.
fn sql_where<T: BenchTable>(
&mut self,
table: &TableSchema,
table_id: &Self::TableId,
column_index: u32,
value: AlgebraicValue,
) -> ResultBench<()>;
Expand Down
28 changes: 0 additions & 28 deletions crates/bench/src/lib.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
use crate::schemas::BenchTable;
use spacetimedb::db::datastore::traits::{ColumnSchema, TableSchema};
use spacetimedb_lib::auth::{StAccess, StTableType};

pub mod database;
pub mod schemas;
pub mod spacetime_module;
Expand All @@ -10,30 +6,6 @@ pub mod sqlite;

pub type ResultBench<T> = Result<T, anyhow::Error>;

pub(crate) fn create_schema<T: BenchTable>(table_name: &str) -> TableSchema {
let columns = T::product_type()
.elements
.into_iter()
.enumerate()
.map(|(pos, col)| ColumnSchema {
table_id: 0,
col_id: pos as u32,
col_name: col.name.unwrap(),
col_type: col.algebraic_type,
is_autoinc: false,
});

TableSchema {
table_id: 0,
table_name: table_name.to_string(),
indexes: vec![],
columns: columns.collect(),
constraints: vec![],
table_type: StTableType::System,
table_access: StAccess::Public,
}
}

#[cfg(test)]
mod tests {
use crate::{
Expand Down
Loading

0 comments on commit cd88c21

Please sign in to comment.