Skip to content

Commit

Permalink
feat: Sequential insertion in indexed trees (#10111)
Browse files Browse the repository at this point in the history 
- Adds sequential (no subtree) insertion in indexed trees
- If witnesses are requested, the response includes 2*N witnesses, N for
the low leaves and N for the insertions
   - If no witness is requested, it directly does sparse_batch_update
   - Updating an item multiple times in the same call is allowed
- Uses sequential insertion with witnesses to avoid doing N batch
updates of 1 item for the base rollup
- Uses sequential insertion without witnesses for syncing the public
data tree, to avoid doing 1 batch insertion per TX in the block
  • Loading branch information
@sirasistant
sirasistant authored Nov 28, 2024
1 parent 6fd5fc1 commit bfd9fa6
Show file tree
Hide file tree
Showing 19 changed files with 1,002 additions and 220 deletions.
189 changes: 163 additions & 26 deletions barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,24 +27,79 @@ const size_t MAX_BATCH_SIZE = 64;
template <typename TreeType> void add_values(TreeType& tree, const std::vector<NullifierLeafValue>& values)
{
Signal signal(1);
typename TreeType::AddCompletionCallback completion = [&](const auto&) -> void { signal.signal_level(0); };
bool success = true;
std::string error_message;
typename TreeType::AddCompletionCallback completion = [&](const auto& result) -> void {
success = result.success;
error_message = result.message;
signal.signal_level(0);
};

tree.add_or_update_values(values, completion);
signal.wait_for_level(0);
if (!success) {
throw std::runtime_error(format("Failed to add values: ", error_message));
}
}

template <typename TreeType> void add_values_with_witness(TreeType& tree, const std::vector<NullifierLeafValue>& values)
{
bool success = true;
std::string error_message;
Signal signal(1);
typename TreeType::AddCompletionCallbackWithWitness completion = [&](const auto&) -> void {
typename TreeType::AddCompletionCallbackWithWitness completion = [&](const auto& result) -> void {
success = result.success;
error_message = result.message;
signal.signal_level(0);
};

tree.add_or_update_values(values, completion);
signal.wait_for_level(0);
if (!success) {
throw std::runtime_error(format("Failed to add values with witness: ", error_message));
}
}

template <typename TreeType> void add_values_sequentially(TreeType& tree, const std::vector<NullifierLeafValue>& values)
{
bool success = true;
std::string error_message;
Signal signal(1);
typename TreeType::AddCompletionCallback completion = [&](const auto& result) -> void {
success = result.success;
error_message = result.message;
signal.signal_level(0);
};

tree.add_or_update_values_sequentially(values, completion);
signal.wait_for_level(0);
if (!success) {
throw std::runtime_error(format("Failed to add values sequentially: ", error_message));
}
}

template <typename TreeType>
void add_values_sequentially_with_witness(TreeType& tree, const std::vector<NullifierLeafValue>& values)
{
bool success = true;
std::string error_message;
Signal signal(1);
typename TreeType::AddSequentiallyCompletionCallbackWithWitness completion = [&](const auto& result) -> void {
success = result.success;
error_message = result.message;
signal.signal_level(0);
};

tree.add_or_update_values_sequentially(values, completion);
signal.wait_for_level(0);
if (!success) {
throw std::runtime_error(format("Failed to add values sequentially with witness: ", error_message));
}
}

template <typename TreeType> void multi_thread_indexed_tree_bench(State& state) noexcept
enum InsertionStrategy { SEQUENTIAL, BATCH };

template <typename TreeType, InsertionStrategy strategy> void multi_thread_indexed_tree_bench(State& state) noexcept
{
const size_t batch_size = size_t(state.range(0));
const size_t depth = TREE_DEPTH;
Expand All @@ -61,10 +116,14 @@ template <typename TreeType> void multi_thread_indexed_tree_bench(State& state)

const size_t initial_size = 1024 * 16;
std::vector<NullifierLeafValue> initial_batch(initial_size);
for (size_t i = 0; i < batch_size; ++i) {
for (size_t i = 0; i < initial_size; ++i) {
initial_batch[i] = fr(random_engine.get_random_uint256());
}
add_values(tree, initial_batch);
if (strategy == SEQUENTIAL) {
add_values_sequentially(tree, initial_batch);
} else {
add_values(tree, initial_batch);
}

for (auto _ : state) {
state.PauseTiming();
Expand All @@ -73,11 +132,15 @@ template <typename TreeType> void multi_thread_indexed_tree_bench(State& state)
values[i] = fr(random_engine.get_random_uint256());
}
state.ResumeTiming();
add_values(tree, values);
if (strategy == SEQUENTIAL) {
add_values_sequentially(tree, values);
} else {
add_values(tree, values);
}
}
}

template <typename TreeType> void single_thread_indexed_tree_bench(State& state) noexcept
template <typename TreeType, InsertionStrategy strategy> void single_thread_indexed_tree_bench(State& state) noexcept
{
const size_t batch_size = size_t(state.range(0));
const size_t depth = TREE_DEPTH;
Expand All @@ -94,10 +157,14 @@ template <typename TreeType> void single_thread_indexed_tree_bench(State& state)

const size_t initial_size = 1024 * 16;
std::vector<NullifierLeafValue> initial_batch(initial_size);
for (size_t i = 0; i < batch_size; ++i) {
for (size_t i = 0; i < initial_size; ++i) {
initial_batch[i] = fr(random_engine.get_random_uint256());
}
add_values(tree, initial_batch);
if (strategy == SEQUENTIAL) {
add_values_sequentially(tree, initial_batch);
} else {
add_values(tree, initial_batch);
}

for (auto _ : state) {
state.PauseTiming();
Expand All @@ -106,11 +173,16 @@ template <typename TreeType> void single_thread_indexed_tree_bench(State& state)
values[i] = fr(random_engine.get_random_uint256());
}
state.ResumeTiming();
add_values(tree, values);
if (strategy == SEQUENTIAL) {
add_values_sequentially(tree, values);
} else {
add_values(tree, values);
}
}
}

template <typename TreeType> void multi_thread_indexed_tree_with_witness_bench(State& state) noexcept
template <typename TreeType, InsertionStrategy strategy>
void multi_thread_indexed_tree_with_witness_bench(State& state) noexcept
{
const size_t batch_size = size_t(state.range(0));
const size_t depth = TREE_DEPTH;
Expand All @@ -127,10 +199,14 @@ template <typename TreeType> void multi_thread_indexed_tree_with_witness_bench(S

const size_t initial_size = 1024 * 16;
std::vector<NullifierLeafValue> initial_batch(initial_size);
for (size_t i = 0; i < batch_size; ++i) {
for (size_t i = 0; i < initial_size; ++i) {
initial_batch[i] = fr(random_engine.get_random_uint256());
}
add_values(tree, initial_batch);
if (strategy == SEQUENTIAL) {
add_values_sequentially(tree, initial_batch);
} else {
add_values(tree, initial_batch);
}

for (auto _ : state) {
state.PauseTiming();
Expand All @@ -139,11 +215,16 @@ template <typename TreeType> void multi_thread_indexed_tree_with_witness_bench(S
values[i] = fr(random_engine.get_random_uint256());
}
state.ResumeTiming();
add_values_with_witness(tree, values);
if (strategy == SEQUENTIAL) {
add_values_sequentially_with_witness(tree, values);
} else {
add_values_with_witness(tree, values);
}
}
}

template <typename TreeType> void single_thread_indexed_tree_with_witness_bench(State& state) noexcept
template <typename TreeType, InsertionStrategy strategy>
void single_thread_indexed_tree_with_witness_bench(State& state) noexcept
{
const size_t batch_size = size_t(state.range(0));
const size_t depth = TREE_DEPTH;
Expand All @@ -160,10 +241,14 @@ template <typename TreeType> void single_thread_indexed_tree_with_witness_bench(

const size_t initial_size = 1024 * 16;
std::vector<NullifierLeafValue> initial_batch(initial_size);
for (size_t i = 0; i < batch_size; ++i) {
for (size_t i = 0; i < initial_size; ++i) {
initial_batch[i] = fr(random_engine.get_random_uint256());
}
add_values(tree, initial_batch);
if (strategy == SEQUENTIAL) {
add_values_sequentially(tree, initial_batch);
} else {
add_values(tree, initial_batch);
}

for (auto _ : state) {
state.PauseTiming();
Expand All @@ -172,53 +257,105 @@ template <typename TreeType> void single_thread_indexed_tree_with_witness_bench(
values[i] = fr(random_engine.get_random_uint256());
}
state.ResumeTiming();
add_values_with_witness(tree, values);
if (strategy == SEQUENTIAL) {
add_values_sequentially_with_witness(tree, values);
} else {
add_values_with_witness(tree, values);
}
}
}

BENCHMARK(single_thread_indexed_tree_with_witness_bench<Poseidon2>)
BENCHMARK(single_thread_indexed_tree_with_witness_bench<Poseidon2, BATCH>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(2, MAX_BATCH_SIZE)
->Iterations(1000);

BENCHMARK(single_thread_indexed_tree_with_witness_bench<Poseidon2>)
BENCHMARK(single_thread_indexed_tree_with_witness_bench<Poseidon2, BATCH>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(512, 8192)
->Iterations(10);

BENCHMARK(multi_thread_indexed_tree_with_witness_bench<Poseidon2>)
BENCHMARK(single_thread_indexed_tree_with_witness_bench<Poseidon2, SEQUENTIAL>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(2, MAX_BATCH_SIZE)
->Iterations(1000);

BENCHMARK(multi_thread_indexed_tree_with_witness_bench<Poseidon2>)
BENCHMARK(single_thread_indexed_tree_with_witness_bench<Poseidon2, SEQUENTIAL>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(512, 8192)
->Iterations(10);

BENCHMARK(single_thread_indexed_tree_bench<Poseidon2>)
BENCHMARK(multi_thread_indexed_tree_with_witness_bench<Poseidon2, BATCH>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(2, MAX_BATCH_SIZE)
->Iterations(1000);

BENCHMARK(single_thread_indexed_tree_bench<Poseidon2>)
BENCHMARK(multi_thread_indexed_tree_with_witness_bench<Poseidon2, BATCH>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(512, 8192)
->Iterations(10);

BENCHMARK(multi_thread_indexed_tree_bench<Poseidon2>)
BENCHMARK(multi_thread_indexed_tree_with_witness_bench<Poseidon2, SEQUENTIAL>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(2, MAX_BATCH_SIZE)
->Iterations(1000);

BENCHMARK(multi_thread_indexed_tree_with_witness_bench<Poseidon2, SEQUENTIAL>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(512, 8192)
->Iterations(10);

BENCHMARK(single_thread_indexed_tree_bench<Poseidon2, BATCH>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(2, MAX_BATCH_SIZE)
->Iterations(1000);

BENCHMARK(single_thread_indexed_tree_bench<Poseidon2, BATCH>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(512, 8192)
->Iterations(10);

BENCHMARK(single_thread_indexed_tree_bench<Poseidon2, SEQUENTIAL>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(2, MAX_BATCH_SIZE)
->Iterations(1000);

BENCHMARK(single_thread_indexed_tree_bench<Poseidon2, SEQUENTIAL>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(512, 8192)
->Iterations(10);

BENCHMARK(multi_thread_indexed_tree_bench<Poseidon2, BATCH>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(2, MAX_BATCH_SIZE)
->Iterations(1000);

BENCHMARK(multi_thread_indexed_tree_bench<Poseidon2, BATCH>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(512, 8192)
->Iterations(100);

BENCHMARK(multi_thread_indexed_tree_bench<Poseidon2, SEQUENTIAL>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(2, MAX_BATCH_SIZE)
->Iterations(1000);

BENCHMARK(multi_thread_indexed_tree_bench<Poseidon2>)
BENCHMARK(multi_thread_indexed_tree_bench<Poseidon2, SEQUENTIAL>)
->Unit(benchmark::kMillisecond)
->RangeMultiplier(2)
->Range(512, 8192)
Expand Down
10 changes: 10 additions & 0 deletions ...c/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,16 @@ std::optional<fr> ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_lea

if (!child.has_value()) {
// std::cout << "No child" << std::endl;
// We still need to update the cache with the sibling. The fact that under us there is an empty subtree
// doesn't mean that same is happening with our sibling.
if (updateNodesByIndexCache) {
child_index_at_level = is_right ? (child_index_at_level * 2) + 1 : (child_index_at_level * 2);
std::optional<fr> sibling = is_right ? nodePayload.left : nodePayload.right;
index_t sibling_index_at_level = is_right ? child_index_at_level - 1 : child_index_at_level + 1;
if (sibling.has_value()) {
store_->put_cached_node_by_index(i + 1, sibling_index_at_level, sibling.value(), false);
}
}
return std::nullopt;
}
// std::cout << "Found child " << child.value() << std::endl;
Expand Down
Loading

0 comments on commit bfd9fa6

Please sign in to comment.