diff --git a/barretenberg/cpp/scripts/merkle_tree_tests.sh b/barretenberg/cpp/scripts/merkle_tree_tests.sh index f53e2750d5a..9e5e0f0b3c9 100755 --- a/barretenberg/cpp/scripts/merkle_tree_tests.sh +++ b/barretenberg/cpp/scripts/merkle_tree_tests.sh @@ -5,7 +5,7 @@ set -e # run commands relative to parent directory cd $(dirname $0)/.. -DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.* +DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBTreeStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.* TEST=${1:-$DEFAULT_TESTS} PRESET=${PRESET:-clang16} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index 7dc41b0abc1..d645b460708 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -39,16 +39,17 @@ template class ContentAddressedAppendOn using StoreType = Store; // Asynchronous methods accept these callback function types as arguments - using AppendCompletionCallback = std::function&)>; - using MetaDataCallback = std::function&)>; - using HashPathCallback = std::function&)>; - using FindLeafCallback = std::function&)>; - using GetLeafCallback = std::function&)>; + using AppendCompletionCallback = std::function&)>; + using MetaDataCallback = std::function&)>; + using HashPathCallback = std::function&)>; + using FindLeafCallback = std::function&)>; + using GetLeafCallback = std::function&)>; using CommitCallback = std::function&)>; - using RollbackCallback = std::function; + using RollbackCallback = std::function; using RemoveHistoricBlockCallback = std::function&)>; using UnwindBlockCallback = std::function&)>; - using FinaliseBlockCallback = std::function; + using FinaliseBlockCallback = std::function; + using GetBlockForIndexCallback = std::function&)>; // Only construct from provided store and thread pool, no copies or moves ContentAddressedAppendOnlyTree(std::unique_ptr store, @@ -90,7 +91,7 @@ template class ContentAddressedAppendOn * @param includeUncommitted Whether to include uncommitted changes */ void get_sibling_path(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, const HashPathCallback& on_completion, bool includeUncommitted) const; @@ -131,7 +132,7 @@ template class ContentAddressedAppendOn * @param includeUncommitted Whether to include uncommitted changes * @param on_completion Callback to be called on completion */ - void get_meta_data(const index_t& blockNumber, + void get_meta_data(const block_number_t& blockNumber, bool includeUncommitted, const MetaDataCallback& on_completion) const; @@ -151,7 +152,7 @@ template class ContentAddressedAppendOn * @param on_completion Callback to be called on completion */ void get_leaf(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const GetLeafCallback& completion) const; @@ -164,7 +165,7 @@ template class ContentAddressedAppendOn * @brief Returns the index of the provided leaf in the tree */ void find_leaf_index(const fr& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const; @@ -181,10 +182,23 @@ template class ContentAddressedAppendOn */ void find_leaf_index_from(const fr& leaf, const index_t& start_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const; + /** + * @brief Returns the block numbers that correspond to the given indices values + */ + void find_block_numbers(const std::vector& indices, const GetBlockForIndexCallback& on_completion) const; + + /** + * @brief Returns the block numbers that correspond to the given indices values, from the perspective of a + * historical block number + */ + void find_block_numbers(const std::vector& indices, + const block_number_t& blockNumber, + const GetBlockForIndexCallback& on_completion) const; + /** * @brief Commit the tree to the backing store */ @@ -200,11 +214,11 @@ template class ContentAddressedAppendOn */ uint32_t depth() const { return depth_; } - void remove_historic_block(const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion); + void remove_historic_block(const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion); - void unwind_block(const index_t& blockNumber, const UnwindBlockCallback& on_completion); + void unwind_block(const block_number_t& blockNumber, const UnwindBlockCallback& on_completion); - void finalise_block(const index_t& blockNumber, const FinaliseBlockCallback& on_completion); + void finalise_block(const block_number_t& blockNumber, const FinaliseBlockCallback& on_completion); protected: using ReadTransaction = typename Store::ReadTransaction; @@ -326,7 +340,7 @@ void ContentAddressedAppendOnlyTree::get_meta_data(bool in } template -void ContentAddressedAppendOnlyTree::get_meta_data(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::get_meta_data(const block_number_t& blockNumber, bool includeUncommitted, const MetaDataCallback& on_completion) const { @@ -361,7 +375,7 @@ void ContentAddressedAppendOnlyTree::get_sibling_path(cons template void ContentAddressedAppendOnlyTree::get_sibling_path(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, const HashPathCallback& on_completion, bool includeUncommitted) const { @@ -393,6 +407,62 @@ void ContentAddressedAppendOnlyTree::get_sibling_path(cons workers_->enqueue(job); } +template +void ContentAddressedAppendOnlyTree::find_block_numbers( + const std::vector& indices, const GetBlockForIndexCallback& on_completion) const +{ + auto job = [=, this]() { + execute_and_report( + [=, this](TypedResponse& response) { + response.inner.blockNumbers.reserve(indices.size()); + TreeMeta meta; + ReadTransactionPtr tx = store_->create_read_transaction(); + store_->get_meta(meta, *tx, true); + index_t maxIndex = meta.committedSize; + for (index_t index : indices) { + bool outOfRange = index >= maxIndex; + std::optional block = + outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx); + response.inner.blockNumbers.emplace_back(block); + } + }, + on_completion); + }; + workers_->enqueue(job); +} + +template +void ContentAddressedAppendOnlyTree::find_block_numbers( + const std::vector& indices, + const block_number_t& blockNumber, + const GetBlockForIndexCallback& on_completion) const +{ + auto job = [=, this]() { + execute_and_report( + [=, this](TypedResponse& response) { + response.inner.blockNumbers.reserve(indices.size()); + TreeMeta meta; + BlockPayload blockPayload; + ReadTransactionPtr tx = store_->create_read_transaction(); + store_->get_meta(meta, *tx, true); + if (!store_->get_block_data(blockNumber, blockPayload, *tx)) { + throw std::runtime_error(format("Unable to find block numbers for indices for block ", + blockNumber, + ", failed to get block data.")); + } + index_t maxIndex = std::min(meta.committedSize, blockPayload.size); + for (index_t index : indices) { + bool outOfRange = index >= maxIndex; + std::optional block = + outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx); + response.inner.blockNumbers.emplace_back(block); + } + }, + on_completion); + }; + workers_->enqueue(job); +} + template void ContentAddressedAppendOnlyTree::get_subtree_sibling_path( uint32_t subtree_depth, const HashPathCallback& on_completion, bool includeUncommitted) const @@ -473,7 +543,7 @@ std::optional ContentAddressedAppendOnlyTree::find_lea NodePayload nodePayload; bool success = store_->get_node_by_hash(hash, nodePayload, tx, requestContext.includeUncommitted); if (!success) { - // std::cout << "No root" << std::endl; + // std::cout << "No root " << hash << std::endl; return std::nullopt; } // std::cout << "Found root at depth " << i << " : " << hash << std::endl; @@ -601,7 +671,7 @@ void ContentAddressedAppendOnlyTree::get_leaf(const index_ template void ContentAddressedAppendOnlyTree::get_leaf(const index_t& leaf_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const GetLeafCallback& on_completion) const { @@ -657,7 +727,7 @@ void ContentAddressedAppendOnlyTree::find_leaf_index(const template void ContentAddressedAppendOnlyTree::find_leaf_index(const fr& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const { @@ -696,7 +766,7 @@ template void ContentAddressedAppendOnlyTree::find_leaf_index_from( const fr& leaf, const index_t& start_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const { @@ -788,7 +858,7 @@ void ContentAddressedAppendOnlyTree::rollback(const Rollba template void ContentAddressedAppendOnlyTree::remove_historic_block( - const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion) + const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion) { auto job = [=, this]() { execute_and_report( @@ -804,7 +874,7 @@ void ContentAddressedAppendOnlyTree::remove_historic_block } template -void ContentAddressedAppendOnlyTree::unwind_block(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::unwind_block(const block_number_t& blockNumber, const UnwindBlockCallback& on_completion) { auto job = [=, this]() { @@ -821,7 +891,7 @@ void ContentAddressedAppendOnlyTree::unwind_block(const in } template -void ContentAddressedAppendOnlyTree::finalise_block(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::finalise_block(const block_number_t& blockNumber, const FinaliseBlockCallback& on_completion) { auto job = [=, this]() { @@ -981,7 +1051,7 @@ void ContentAddressedAppendOnlyTree::add_batch_internal( new_root = new_hash; meta.root = new_hash; meta.size = new_size; - // std::cout << "New size: " << meta.size << std::endl; + // std::cout << "New size: " << meta.size << ", root " << meta.root << std::endl; store_->put_meta(meta); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp index 5ac60d919b7..83f72c9ca1f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp @@ -123,7 +123,7 @@ void check_sibling_path(TreeType& tree, void check_historic_sibling_path(TreeType& tree, index_t index, fr_sibling_path expected_sibling_path, - index_t blockNumber, + block_number_t blockNumber, bool expected_success = true) { Signal signal; @@ -160,7 +160,7 @@ void rollback_tree(TreeType& tree) signal.wait_for_level(); } -void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void remove_historic_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -171,7 +171,7 @@ void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expe signal.wait_for_level(); } -void unwind_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void unwind_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -206,7 +206,7 @@ void add_values(TreeType& tree, const std::vector& values) signal.wait_for_level(); } -void finalise_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void finalise_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const Response& response) -> void { @@ -312,7 +312,7 @@ void check_leaf( } void check_historic_leaf(TreeType& tree, - const index_t& blockNumber, + const block_number_t& blockNumber, const fr& leaf, index_t leaf_index, bool expected_success, @@ -350,6 +350,31 @@ void check_sibling_path(fr expected_root, fr node, index_t index, fr_sibling_pat EXPECT_EQ(hash, expected_root); } +void get_blocks_for_indices(TreeType& tree, + const std::vector& indices, + std::vector>& blockNumbers) +{ + Signal signal; + tree.find_block_numbers(indices, [&](const TypedResponse& response) { + blockNumbers = response.inner.blockNumbers; + signal.signal_level(); + }); + signal.wait_for_level(); +} + +void get_blocks_for_indices(TreeType& tree, + const block_number_t& blockNumber, + const std::vector& indices, + std::vector>& blockNumbers) +{ + Signal signal; + tree.find_block_numbers(indices, blockNumber, [&](const TypedResponse& response) { + blockNumbers = response.inner.blockNumbers; + signal.signal_level(); + }); + signal.wait_for_level(); +} + TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create) { constexpr size_t depth = 10; @@ -1284,7 +1309,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_remove_historic_block_da for (uint32_t i = 0; i < historicPathsZeroIndex.size(); i++) { // retrieving historic data should fail if the block is outside of the window - const index_t blockNumber = i + 1; + const block_number_t blockNumber = i + 1; const bool expectedSuccess = expectedBlockHeight <= windowSize || blockNumber > (expectedBlockHeight - windowSize); check_historic_sibling_path(tree, 0, historicPathsZeroIndex[i], blockNumber, expectedSuccess); @@ -1399,7 +1424,7 @@ void test_unwind(std::string directory, const uint32_t blocksToRemove = numBlocksToUnwind; for (uint32_t i = 0; i < blocksToRemove; i++) { - const index_t blockNumber = numBlocks - i; + const block_number_t blockNumber = numBlocks - i; check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], true); // attempting to unwind a block that is not the tip should fail @@ -1513,6 +1538,101 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_sync_and_unwind_large_bl } } +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_retrieve_block_numbers_by_index) +{ + std::string name = random_string(); + constexpr uint32_t depth = 10; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + + const size_t block_size = 32; + + for (size_t i = 0; i < 5; i++) { + std::vector values = create_values(block_size); + add_values(tree, values); + commit_tree(tree); + } + std::vector indices{ 12, 33, 63, 64, 65, 80, 96, 159, 160 }; + std::vector> blockNumbers; + + // All but the last block number should be valid when looking at latest + get_blocks_for_indices(tree, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + index_t maxIndex = 5 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // Now get blocks for indices from the perspective of block 2 + get_blocks_for_indices(tree, 2, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = 2 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + unwind_block(tree, 5); + unwind_block(tree, 4); + + get_blocks_for_indices(tree, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + maxIndex = 3 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // fork from block 1 + std::unique_ptr forkStore = std::make_unique(name, depth, 1, db); + TreeType treeFork(std::move(forkStore), pool); + + // Now, using the fork, get block indices but find it's limited to those of block 1 + get_blocks_for_indices(treeFork, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // Now, using the fork, get block indics from the perspective of block 2, but find it's limited to those of block 1 + get_blocks_for_indices(treeFork, 2, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } +} + TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_advance_finalised_blocks) { std::string name = random_string(); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp index 197f784d59f..45cc232e8e1 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp @@ -47,14 +47,13 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree>&)>; + using AddCompletionCallbackWithWitness = std::function>&)>; using AddSequentiallyCompletionCallbackWithWitness = - std::function>&)>; - using AddCompletionCallback = std::function&)>; + std::function>&)>; + using AddCompletionCallback = std::function&)>; - using LeafCallback = std::function>&)>; - using FindLowLeafCallback = std::function&)>; + using LeafCallback = std::function>&)>; + using FindLowLeafCallback = std::function&)>; ContentAddressedIndexedTree(std::unique_ptr store, std::shared_ptr workers, @@ -151,7 +150,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; @@ -169,7 +168,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; @@ -178,7 +177,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::get_leaf(const index_t& template void ContentAddressedIndexedTree::get_leaf(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const LeafCallback& completion) const { @@ -458,7 +457,7 @@ void ContentAddressedIndexedTree::find_leaf_index( template void ContentAddressedIndexedTree::find_leaf_index( const LeafValueType& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const { @@ -496,7 +495,7 @@ void ContentAddressedIndexedTree::find_leaf_index_from( template void ContentAddressedIndexedTree::find_leaf_index_from( const LeafValueType& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, const index_t& start_index, bool includeUncommitted, const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const @@ -559,7 +558,7 @@ void ContentAddressedIndexedTree::find_low_leaf(const fr& template void ContentAddressedIndexedTree::find_low_leaf(const fr& leaf_key, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLowLeafCallback& on_completion) const { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp index 5177b370833..8a7443ef05c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp @@ -104,8 +104,11 @@ template void check_root(TypeOfTree& tree, fr expected_roo } template -fr_sibling_path get_historic_sibling_path( - TypeOfTree& tree, index_t blockNumber, index_t index, bool includeUncommitted = true, bool expected_success = true) +fr_sibling_path get_historic_sibling_path(TypeOfTree& tree, + block_number_t blockNumber, + index_t index, + bool includeUncommitted = true, + bool expected_success = true) { fr_sibling_path h; Signal signal; @@ -177,7 +180,7 @@ GetLowIndexedLeafResponse get_low_leaf(TypeOfTree& tree, const LeafValueType& le template GetLowIndexedLeafResponse get_historic_low_leaf(TypeOfTree& tree, - index_t blockNumber, + block_number_t blockNumber, const LeafValueType& leaf, bool includeUncommitted = true) { @@ -236,7 +239,7 @@ void check_find_leaf_index_from(TypeOfTree& tree, template void check_historic_find_leaf_index(TypeOfTree& tree, const LeafValueType& leaf, - index_t blockNumber, + block_number_t blockNumber, index_t expected_index, bool expected_success, bool includeUncommitted = true) @@ -257,7 +260,7 @@ void check_historic_find_leaf_index(TypeOfTree& tree, template void check_historic_find_leaf_index_from(TypeOfTree& tree, const LeafValueType& leaf, - index_t blockNumber, + block_number_t blockNumber, index_t start_index, index_t expected_index, bool expected_success, @@ -280,7 +283,7 @@ template void check_historic_leaf(TypeOfTree& tree, const LeafValueType& leaf, index_t expected_index, - index_t blockNumber, + block_number_t blockNumber, bool expected_success, bool includeUncommitted = true) { @@ -300,7 +303,7 @@ void check_historic_leaf(TypeOfTree& tree, template void check_historic_sibling_path(TypeOfTree& tree, index_t index, - index_t blockNumber, + block_number_t blockNumber, const fr_sibling_path& expected_sibling_path, bool includeUncommitted = true, bool expected_success = true) @@ -413,7 +416,7 @@ void block_sync_values_sequential(TypeOfTree& tree, } template -void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void remove_historic_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -425,7 +428,7 @@ void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool ex } template -void finalise_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void finalise_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const Response& response) -> void { @@ -437,7 +440,7 @@ void finalise_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_ } template -void unwind_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void unwind_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -2596,7 +2599,7 @@ void test_nullifier_tree_unwind(std::string directory, const uint32_t blocksToRemove = numBlocksToUnwind; for (uint32_t i = 0; i < blocksToRemove; i++) { - const index_t blockNumber = numBlocks - i; + const block_number_t blockNumber = numBlocks - i; check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], true); unwind_block(tree, blockNumber); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp index cf2a55c1285..9bbea8ea42e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp @@ -46,6 +46,8 @@ class LMDBTransaction { template bool get_value_or_previous(T& key, K& data, const LMDBDatabase& db) const; + template bool get_value_or_greater(T& key, K& data, const LMDBDatabase& db) const; + template bool get_value(T& key, std::vector& data, const LMDBDatabase& db) const; template bool get_value(T& key, index_t& data, const LMDBDatabase& db) const; @@ -88,6 +90,12 @@ bool LMDBTransaction::get_value_or_previous(T& key, K& data, const LMDBDatabase& return lmdb_queries::get_value_or_previous(key, data, db, *this); } +template +bool LMDBTransaction::get_value_or_greater(T& key, K& data, const LMDBDatabase& db) const +{ + return lmdb_queries::get_value_or_greater(key, data, db, *this); +} + template bool LMDBTransaction::get_value_or_previous(T& key, K& data, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp index 3f4f07aa829..09f87cae606 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp @@ -55,29 +55,35 @@ LMDBTreeStore::LMDBTreeStore(std::string directory, std::string name, uint64_t m { LMDBDatabaseCreationTransaction tx(_environment); - _blockDatabase = std::make_unique( - _environment, tx, _name + std::string("blocks"), false, false, block_key_cmp); + _blockDatabase = + std::make_unique(_environment, tx, _name + BLOCKS_DB, false, false, block_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _nodeDatabase = - std::make_unique(_environment, tx, _name + std::string("nodes"), false, false, fr_key_cmp); + _nodeDatabase = std::make_unique(_environment, tx, _name + NODES_DB, false, false, fr_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _leafKeyToIndexDatabase = std::make_unique( - _environment, tx, _name + std::string("leaf indices"), false, false, fr_key_cmp); + _leafKeyToIndexDatabase = + std::make_unique(_environment, tx, _name + LEAF_INDICES_DB, false, false, fr_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _leafHashToPreImageDatabase = std::make_unique( - _environment, tx, _name + std::string("leaf pre-images"), false, false, fr_key_cmp); + _leafHashToPreImageDatabase = + std::make_unique(_environment, tx, _name + LEAF_PREIMAGES_DB, false, false, fr_key_cmp); + tx.commit(); + } + + { + LMDBDatabaseCreationTransaction tx(_environment); + _indexToBlockDatabase = + std::make_unique(_environment, tx, _name + BLOCK_INDICES_DB, false, false, index_key_cmp); tx.commit(); } } @@ -107,9 +113,11 @@ void LMDBTreeStore::get_stats(TreeDBStats& stats, ReadTransaction& tx) stats.leafIndicesDBStats = DBStats(LEAF_INDICES_DB, stat); call_lmdb_func(mdb_stat, tx.underlying(), _nodeDatabase->underlying(), &stat); stats.nodesDBStats = DBStats(NODES_DB, stat); + call_lmdb_func(mdb_stat, tx.underlying(), _indexToBlockDatabase->underlying(), &stat); + stats.blockIndicesDBStats = DBStats(BLOCK_INDICES_DB, stat); } -void LMDBTreeStore::write_block_data(uint64_t blockNumber, +void LMDBTreeStore::write_block_data(const block_number_t& blockNumber, const BlockPayload& blockData, LMDBTreeStore::WriteTransaction& tx) { @@ -120,13 +128,15 @@ void LMDBTreeStore::write_block_data(uint64_t blockNumber, tx.put_value(key, encoded, *_blockDatabase); } -void LMDBTreeStore::delete_block_data(uint64_t blockNumber, LMDBTreeStore::WriteTransaction& tx) +void LMDBTreeStore::delete_block_data(const block_number_t& blockNumber, LMDBTreeStore::WriteTransaction& tx) { BlockMetaKeyType key(blockNumber); tx.delete_value(key, *_blockDatabase); } -bool LMDBTreeStore::read_block_data(uint64_t blockNumber, BlockPayload& blockData, LMDBTreeStore::ReadTransaction& tx) +bool LMDBTreeStore::read_block_data(const block_number_t& blockNumber, + BlockPayload& blockData, + LMDBTreeStore::ReadTransaction& tx) { BlockMetaKeyType key(blockNumber); std::vector data; @@ -137,6 +147,87 @@ bool LMDBTreeStore::read_block_data(uint64_t blockNumber, BlockPayload& blockDat return success; } +void LMDBTreeStore::write_block_index_data(const block_number_t& blockNumber, + const index_t& sizeAtBlock, + WriteTransaction& tx) +{ + // There can be multiple block numbers aganst the same index (zero size blocks) + LeafIndexKeyType key(sizeAtBlock); + std::vector data; + // Read the block index payload + bool success = tx.get_value(key, data, *_indexToBlockDatabase); + BlockIndexPayload payload; + if (success) { + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + } + + // Double check it's not already present (it shouldn't be) + // We then add the block number and sort + // Sorting shouldn't be necessary as we add blocks in ascending order, but we will make sure + // Sorting here and when we unwind blocks means that looking up the block number for an index becomes O(1) + // These lookups are much more frequent than adds or deletes so we take the hit here + if (!payload.contains(blockNumber)) { + payload.blockNumbers.emplace_back(blockNumber); + payload.sort(); + } + + // Write the new payload back down + msgpack::sbuffer buffer; + msgpack::pack(buffer, payload); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + tx.put_value(key, encoded, *_indexToBlockDatabase); +} + +bool LMDBTreeStore::find_block_for_index(const index_t& index, block_number_t& blockNumber, ReadTransaction& tx) +{ + LeafIndexKeyType key(index + 1); + std::vector data; + // Retrieve the payload + bool success = tx.get_value_or_greater(key, data, *_indexToBlockDatabase); + if (!success) { + return false; + } + BlockIndexPayload payload; + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + if (payload.blockNumbers.empty()) { + return false; + } + // The block numbers are sorted so we simply return the lowest + blockNumber = payload.blockNumbers[0]; + return true; +} + +void LMDBTreeStore::delete_block_index(const index_t& sizeAtBlock, + const block_number_t& blockNumber, + WriteTransaction& tx) +{ + // To delete a block number form an index we retieve all the block numbers from that index + // Then we find and remove the block number in question + // Then we write back down + LeafIndexKeyType key(sizeAtBlock); + std::vector data; + // Retrieve the data + bool success = tx.get_value(key, data, *_indexToBlockDatabase); + if (!success) { + return; + } + BlockIndexPayload payload; + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + + payload.delete_block(blockNumber); + + // if it's now empty, delete it + if (payload.blockNumbers.empty()) { + tx.delete_value(key, *_indexToBlockDatabase); + return; + } + // not empty write it back + msgpack::sbuffer buffer; + msgpack::pack(buffer, payload); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + tx.put_value(key, encoded, *_indexToBlockDatabase); +} + void LMDBTreeStore::write_meta_data(const TreeMeta& metaData, LMDBTreeStore::WriteTransaction& tx) { msgpack::sbuffer buffer; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp index ab4dfb7316c..a28ce245967 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp @@ -25,7 +25,7 @@ namespace bb::crypto::merkle_tree { struct BlockPayload { index_t size; - index_t blockNumber; + block_number_t blockNumber; fr root; MSGPACK_FIELDS(size, blockNumber, root) @@ -55,6 +55,49 @@ struct NodePayload { return left == other.left && right == other.right && ref == other.ref; } }; + +struct BlockIndexPayload { + std::vector blockNumbers; + + MSGPACK_FIELDS(blockNumbers) + + bool operator==(const BlockIndexPayload& other) const { return blockNumbers == other.blockNumbers; } + + void sort() { std::sort(blockNumbers.begin(), blockNumbers.end()); } + + bool contains(const block_number_t& blockNumber) + { + auto it = std::lower_bound(blockNumbers.begin(), blockNumbers.end(), blockNumber); + if (it == blockNumbers.end()) { + // The block was not found, we can return + return false; + } + return *it == blockNumber; + } + + void delete_block(const block_number_t& blockNumber) + { + if (blockNumbers.empty()) { + return; + } + // shuffle the block number down, removing the one we want to remove and then pop the end item + auto it = std::lower_bound(blockNumbers.begin(), blockNumbers.end(), blockNumber); + if (it == blockNumbers.end()) { + // The block was not found, we can return + return; + } + // It could be a block higher than the one we are looking for + if (*it != blockNumber) { + return; + } + // we have found our block, shuffle blocks after this one down + auto readIt = it + 1; + while (readIt != blockNumbers.end()) { + *it++ = *readIt++; + } + blockNumbers.pop_back(); + } +}; /** * Creates an abstraction against a collection of LMDB databases within a single environment used to store merkle tree * data @@ -78,11 +121,18 @@ class LMDBTreeStore { void get_stats(TreeDBStats& stats, ReadTransaction& tx); - void write_block_data(uint64_t blockNumber, const BlockPayload& blockData, WriteTransaction& tx); + void write_block_data(const block_number_t& blockNumber, const BlockPayload& blockData, WriteTransaction& tx); + + bool read_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx); + + void delete_block_data(const block_number_t& blockNumber, WriteTransaction& tx); + + void write_block_index_data(const block_number_t& blockNumber, const index_t& sizeAtBlock, WriteTransaction& tx); - bool read_block_data(uint64_t blockNumber, BlockPayload& blockData, ReadTransaction& tx); + // index here is 0 based + bool find_block_for_index(const index_t& index, block_number_t& blockNumber, ReadTransaction& tx); - void delete_block_data(uint64_t blockNumber, WriteTransaction& tx); + void delete_block_index(const index_t& sizeAtBlock, const block_number_t& blockNumber, WriteTransaction& tx); void write_meta_data(const TreeMeta& metaData, WriteTransaction& tx); @@ -136,6 +186,7 @@ class LMDBTreeStore { LMDBDatabase::Ptr _nodeDatabase; LMDBDatabase::Ptr _leafKeyToIndexDatabase; LMDBDatabase::Ptr _leafHashToPreImageDatabase; + LMDBDatabase::Ptr _indexToBlockDatabase; template bool get_node_data(const fr& nodeHash, NodePayload& nodeData, TxType& tx); }; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp index cb4d342b09a..e9b85aa5bbc 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -14,6 +15,7 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/numeric/random/engine.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" @@ -255,3 +257,254 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_leaves_by_hash) EXPECT_FALSE(success); } } + +TEST_F(LMDBTreeStoreTest, can_write_and_retrieve_block_numbers_by_index) +{ + struct BlockAndIndex { + block_number_t blockNumber; + // this block contains leaves up to index (0 based) + index_t index; + }; + + std::vector blocks{ BlockAndIndex{ .blockNumber = 1, .index = 25 }, + BlockAndIndex{ .blockNumber = 2, .index = 60 }, + BlockAndIndex{ .blockNumber = 3, .index = 82 }, + BlockAndIndex{ .blockNumber = 4, .index = 114 }, + BlockAndIndex{ .blockNumber = 5, .index = 130 } }; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + // write all of the blocks. we will write them in reverse order + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (int i = int(blocks.size()) - 1; i >= 0; i--) { + // the arg is block size so add 1 + const BlockAndIndex& block = blocks[size_t(i)]; + store.write_block_index_data(block.blockNumber, block.index + 1, *transaction); + } + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(83, readBack, *transaction)); + EXPECT_EQ(readBack, 4); + + EXPECT_TRUE(store.find_block_for_index(130, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete the last block + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[4].index + 1, blocks[4].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(83, readBack, *transaction)); + EXPECT_EQ(readBack, 4); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete 2 more blocks + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_FALSE(store.find_block_for_index(82, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(83, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete non-exisatent indices to check it does nothing + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + store.delete_block_index(21, 1, *transaction); + store.delete_block_index(150, 6, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_FALSE(store.find_block_for_index(82, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(83, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } +} + +TEST_F(LMDBTreeStoreTest, can_write_and_retrieve_block_numbers_with_duplicate_indices) +{ + struct BlockAndIndex { + block_number_t blockNumber; + index_t index; + }; + + std::vector blocks{ BlockAndIndex{ .blockNumber = 1, .index = 25 }, + BlockAndIndex{ .blockNumber = 2, .index = 60 }, + BlockAndIndex{ .blockNumber = 3, .index = 60 }, + BlockAndIndex{ .blockNumber = 4, .index = 60 }, + BlockAndIndex{ .blockNumber = 5, .index = 130 } }; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + // write all of the blocks. we will write them in reverse order + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (int i = int(blocks.size()) - 1; i >= 0; i--) { + // the arg is block size so add 1 + const BlockAndIndex& block = blocks[size_t(i)]; + store.write_block_index_data(block.blockNumber, block.index + 1, *transaction); + } + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete block 2 at index 60 + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[1].index + 1, blocks[1].blockNumber, *transaction); + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the new lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // try and delete blocks that don't exist at index 60 + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[1].index + 1, 2, *transaction); + store.delete_block_index(blocks[1].index + 1, 5, *transaction); + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the new lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete 2 more blocks + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + } +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp index aa97a2d2518..c26768fa8ec 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp @@ -177,6 +177,45 @@ bool get_value_or_previous(TKey& key, return success; } +template +bool get_value_or_greater(TKey& key, std::vector& data, const LMDBDatabase& db, const TxType& tx) +{ + bool success = false; + std::vector keyBuffer = serialise_key(key); + uint32_t keySize = static_cast(keyBuffer.size()); + MDB_cursor* cursor = nullptr; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, tx.underlying(), db.underlying(), &cursor); + + try { + MDB_val dbKey; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); + + MDB_val dbVal; + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + + if (code == 0) { + // found a key >= our key. if it is not the same size, it must be out of range for what we are looking + // for, this means no more data available + if (keySize == dbKey.mv_size) { + // key is the same size, so this contains the data we are looking for + copy_to_vector(dbVal, data); + success = true; + } + } else if (code == MDB_NOTFOUND) { + // no key greater than or equal, nothing to extract + } else { + throw_error("get_value_or_greater::mdb_cursor_get", code); + } + } catch (std::exception& e) { + call_lmdb_func(mdb_cursor_close, cursor); + throw; + } + call_lmdb_func(mdb_cursor_close, cursor); + return success; +} + template void get_all_values_greater_or_equal_key(const TKey& key, std::vector>& data, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp index 013f1cc8478..abaec64a3c7 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp @@ -147,7 +147,7 @@ template class ContentAddressedCachedTreeStore { /** * @brief Reads the tree meta data, including uncommitted data if requested */ - bool get_block_data(const index_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const; + bool get_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const; /** * @brief Finds the index of the given leaf value in the tree if available. Includes uncommitted data if requested. @@ -198,13 +198,15 @@ template class ContentAddressedCachedTreeStore { fr get_current_root(ReadTransaction& tx, bool includeUncommitted) const; - void remove_historical_block(const index_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); + void remove_historical_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); - void unwind_block(const index_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); + void unwind_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); std::optional get_fork_block() const; - void advance_finalised_block(const index_t& blockNumber); + void advance_finalised_block(const block_number_t& blockNumber); + + std::optional find_block_for_index(const index_t& index, ReadTransaction& tx) const; private: std::string name_; @@ -233,7 +235,7 @@ template class ContentAddressedCachedTreeStore { void initialise(); - void initialise_from_block(const index_t& blockNumber); + void initialise_from_block(const block_number_t& blockNumber); bool read_persisted_meta(TreeMeta& m, ReadTransaction& tx) const; @@ -258,6 +260,10 @@ template class ContentAddressedCachedTreeStore { void extract_db_stats(TreeDBStats& stats); + void persist_block_for_index(const block_number_t& blockNumber, const index_t& index, WriteTransaction& tx); + + void delete_block_for_index(const block_number_t& blockNumber, const index_t& index, WriteTransaction& tx); + index_t constrain_tree_size(const RequestContext& requestContext, ReadTransaction& tx) const; WriteTransactionPtr create_write_transaction() const { return dataStore_->create_write_transaction(); } @@ -304,6 +310,31 @@ index_t ContentAddressedCachedTreeStore::constrain_tree_size(cons return sizeLimit; } +template +std::optional ContentAddressedCachedTreeStore::find_block_for_index( + const index_t& index, ReadTransaction& tx) const +{ + block_number_t blockNumber = 0; + bool success = dataStore_->find_block_for_index(index, blockNumber, tx); + return success ? std::make_optional(blockNumber) : std::nullopt; +} + +template +void ContentAddressedCachedTreeStore::persist_block_for_index(const block_number_t& blockNumber, + const index_t& index, + WriteTransaction& tx) +{ + dataStore_->write_block_index_data(blockNumber, index, tx); +} + +template +void ContentAddressedCachedTreeStore::delete_block_for_index(const block_number_t& blockNumber, + const index_t& index, + WriteTransaction& tx) +{ + dataStore_->delete_block_index(index, blockNumber, tx); +} + template std::pair ContentAddressedCachedTreeStore::find_low_value( const fr& new_leaf_key, const RequestContext& requestContext, ReadTransaction& tx) const @@ -553,7 +584,7 @@ void ContentAddressedCachedTreeStore::get_meta(TreeMeta& m, } template -bool ContentAddressedCachedTreeStore::get_block_data(const index_t& blockNumber, +bool ContentAddressedCachedTreeStore::get_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const { @@ -649,6 +680,7 @@ void ContentAddressedCachedTreeStore::commit(TreeMeta& finalMeta, .blockNumber = uncommittedMeta.unfinalisedBlockHeight, .root = uncommittedMeta.root }; dataStore_->write_block_data(uncommittedMeta.unfinalisedBlockHeight, block, *tx); + dataStore_->write_block_index_data(block.blockNumber, block.size, *tx); } uncommittedMeta.committedSize = uncommittedMeta.size; @@ -767,7 +799,7 @@ void ContentAddressedCachedTreeStore::persist_meta(TreeMeta& m, W } template -void ContentAddressedCachedTreeStore::advance_finalised_block(const index_t& blockNumber) +void ContentAddressedCachedTreeStore::advance_finalised_block(const block_number_t& blockNumber) { TreeMeta committedMeta; TreeMeta uncommittedMeta; @@ -827,7 +859,7 @@ void ContentAddressedCachedTreeStore::advance_finalised_block(con } template -void ContentAddressedCachedTreeStore::unwind_block(const index_t& blockNumber, +void ContentAddressedCachedTreeStore::unwind_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats) { @@ -894,6 +926,7 @@ void ContentAddressedCachedTreeStore::unwind_block(const index_t& remove_node(std::optional(blockData.root), 0, maxIndex, *writeTx); // remove the block from the block data table dataStore_->delete_block_data(blockNumber, *writeTx); + dataStore_->delete_block_index(blockData.size, blockData.blockNumber, *writeTx); uncommittedMeta.unfinalisedBlockHeight = previousBlockData.blockNumber; uncommittedMeta.size = previousBlockData.size; uncommittedMeta.committedSize = previousBlockData.size; @@ -916,7 +949,7 @@ void ContentAddressedCachedTreeStore::unwind_block(const index_t& } template -void ContentAddressedCachedTreeStore::remove_historical_block(const index_t& blockNumber, +void ContentAddressedCachedTreeStore::remove_historical_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats) { @@ -1107,7 +1140,7 @@ template void ContentAddressedCachedTreeStore -void ContentAddressedCachedTreeStore::initialise_from_block(const index_t& blockNumber) +void ContentAddressedCachedTreeStore::initialise_from_block(const block_number_t& blockNumber) { // Read the persisted meta data, if the name or depth of the tree is not consistent with what was provided during // construction then we throw diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp index 164a6b254cf..6b77a6a5ebd 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace bb::crypto::merkle_tree { @@ -16,9 +17,9 @@ struct TreeMeta { bb::fr root; index_t initialSize; bb::fr initialRoot; - uint64_t oldestHistoricBlock; - uint64_t unfinalisedBlockHeight; - uint64_t finalisedBlockHeight; + block_number_t oldestHistoricBlock; + block_number_t unfinalisedBlockHeight; + block_number_t finalisedBlockHeight; MSGPACK_FIELDS(name, depth, @@ -31,6 +32,34 @@ struct TreeMeta { unfinalisedBlockHeight, finalisedBlockHeight) + TreeMeta(std::string n, + uint32_t d, + const index_t& s, + const index_t& c, + const bb::fr& r, + const index_t& is, + const bb::fr& ir, + const block_number_t& o, + const block_number_t& u, + const block_number_t& f) + : name(std::move(n)) + , depth(d) + , size(s) + , committedSize(c) + , root(r) + , initialSize(is) + , initialRoot(ir) + , oldestHistoricBlock(o) + , unfinalisedBlockHeight(u) + , finalisedBlockHeight(f) + {} + TreeMeta() = default; + ~TreeMeta() = default; + TreeMeta(const TreeMeta& other) = default; + TreeMeta(TreeMeta&& other) noexcept { *this = std::move(other); } + TreeMeta& operator=(const TreeMeta& other) = default; + TreeMeta& operator=(TreeMeta&& other) noexcept = default; + bool operator==(const TreeMeta& other) const { return name == other.name && depth == other.depth && size == other.size && @@ -50,12 +79,4 @@ inline std::ostream& operator<<(std::ostream& os, const TreeMeta& meta) return os; } -struct LeavesMeta { - index_t size; - - MSGPACK_FIELDS(size) - - bool operator==(const LeavesMeta& other) const { return size == other.size; } -}; - } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 5fdfd19b85b..d525acb8672 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -12,19 +12,41 @@ #include #include #include +#include namespace bb::crypto::merkle_tree { struct TreeMetaResponse { TreeMeta meta; + + TreeMetaResponse() = default; + ~TreeMetaResponse() = default; + TreeMetaResponse(const TreeMetaResponse& other) = default; + TreeMetaResponse(TreeMetaResponse&& other) noexcept = default; + TreeMetaResponse& operator=(const TreeMetaResponse& other) = default; + TreeMetaResponse& operator=(TreeMetaResponse&& other) noexcept = default; }; struct AddDataResponse { index_t size; fr root; + + AddDataResponse() = default; + ~AddDataResponse() = default; + AddDataResponse(const AddDataResponse& other) = default; + AddDataResponse(AddDataResponse&& other) noexcept = default; + AddDataResponse& operator=(const AddDataResponse& other) = default; + AddDataResponse& operator=(AddDataResponse&& other) noexcept = default; }; struct GetSiblingPathResponse { fr_sibling_path path; + + GetSiblingPathResponse() = default; + ~GetSiblingPathResponse() = default; + GetSiblingPathResponse(const GetSiblingPathResponse& other) = default; + GetSiblingPathResponse(GetSiblingPathResponse&& other) noexcept = default; + GetSiblingPathResponse& operator=(const GetSiblingPathResponse& other) = default; + GetSiblingPathResponse& operator=(GetSiblingPathResponse&& other) noexcept = default; }; template struct LeafUpdateWitnessData { @@ -32,6 +54,18 @@ template struct LeafUpdateWitnessData { index_t index; fr_sibling_path path; + LeafUpdateWitnessData(const IndexedLeaf& l, const index_t& i, fr_sibling_path p) + : leaf(l) + , index(i) + , path(std::move(p)) + {} + LeafUpdateWitnessData() = default; + ~LeafUpdateWitnessData() = default; + LeafUpdateWitnessData(const LeafUpdateWitnessData& other) = default; + LeafUpdateWitnessData(LeafUpdateWitnessData&& other) noexcept = default; + LeafUpdateWitnessData& operator=(const LeafUpdateWitnessData& other) = default; + LeafUpdateWitnessData& operator=(LeafUpdateWitnessData&& other) noexcept = default; + MSGPACK_FIELDS(leaf, index, path); }; @@ -40,20 +74,59 @@ template struct AddIndexedDataResponse { fr_sibling_path subtree_path; std::shared_ptr>> sorted_leaves; std::shared_ptr>> low_leaf_witness_data; + + AddIndexedDataResponse() = default; + ~AddIndexedDataResponse() = default; + AddIndexedDataResponse(const AddIndexedDataResponse& other) = default; + AddIndexedDataResponse(AddIndexedDataResponse&& other) noexcept = default; + AddIndexedDataResponse& operator=(const AddIndexedDataResponse& other) = default; + AddIndexedDataResponse& operator=(AddIndexedDataResponse&& other) noexcept = default; }; template struct AddIndexedDataSequentiallyResponse { AddDataResponse add_data_result; std::shared_ptr>> low_leaf_witness_data; std::shared_ptr>> insertion_witness_data; + + AddIndexedDataSequentiallyResponse() = default; + ~AddIndexedDataSequentiallyResponse() = default; + AddIndexedDataSequentiallyResponse(const AddIndexedDataSequentiallyResponse& other) = default; + AddIndexedDataSequentiallyResponse(AddIndexedDataSequentiallyResponse&& other) noexcept = default; + AddIndexedDataSequentiallyResponse& operator=(const AddIndexedDataSequentiallyResponse& other) = default; + AddIndexedDataSequentiallyResponse& operator=(AddIndexedDataSequentiallyResponse&& other) noexcept = default; +}; + +struct BlockForIndexResponse { + std::vector> blockNumbers; + + BlockForIndexResponse() = default; + ~BlockForIndexResponse() = default; + BlockForIndexResponse(const BlockForIndexResponse& other) = default; + BlockForIndexResponse(BlockForIndexResponse&& other) noexcept = default; + BlockForIndexResponse& operator=(const BlockForIndexResponse& other) = default; + BlockForIndexResponse& operator=(BlockForIndexResponse&& other) noexcept = default; }; struct FindLeafIndexResponse { index_t leaf_index; + + FindLeafIndexResponse() = default; + ~FindLeafIndexResponse() = default; + FindLeafIndexResponse(const FindLeafIndexResponse& other) = default; + FindLeafIndexResponse(FindLeafIndexResponse&& other) noexcept = default; + FindLeafIndexResponse& operator=(const FindLeafIndexResponse& other) = default; + FindLeafIndexResponse& operator=(FindLeafIndexResponse&& other) noexcept = default; }; struct GetLeafResponse { std::optional leaf; + + GetLeafResponse() = default; + ~GetLeafResponse() = default; + GetLeafResponse(const GetLeafResponse& other) = default; + GetLeafResponse(GetLeafResponse&& other) noexcept = default; + GetLeafResponse& operator=(const GetLeafResponse& other) = default; + GetLeafResponse& operator=(GetLeafResponse&& other) noexcept = default; }; template struct GetIndexedLeafResponse { @@ -66,6 +139,17 @@ struct GetLowIndexedLeafResponse { MSGPACK_FIELDS(is_already_present, index); + GetLowIndexedLeafResponse(bool p, const index_t& i) + : is_already_present(p) + , index(i) + {} + GetLowIndexedLeafResponse() = default; + ~GetLowIndexedLeafResponse() = default; + GetLowIndexedLeafResponse(const GetLowIndexedLeafResponse& other) = default; + GetLowIndexedLeafResponse(GetLowIndexedLeafResponse&& other) noexcept = default; + GetLowIndexedLeafResponse& operator=(const GetLowIndexedLeafResponse& other) = default; + GetLowIndexedLeafResponse& operator=(GetLowIndexedLeafResponse&& other) noexcept = default; + bool operator==(const GetLowIndexedLeafResponse& other) const { return is_already_present == other.is_already_present && index == other.index; @@ -75,27 +159,66 @@ struct GetLowIndexedLeafResponse { struct CommitResponse { TreeMeta meta; TreeDBStats stats; + + CommitResponse() = default; + ~CommitResponse() = default; + CommitResponse(const CommitResponse& other) = default; + CommitResponse(CommitResponse&& other) noexcept = default; + CommitResponse& operator=(const CommitResponse& other) = default; + CommitResponse& operator=(CommitResponse&& other) noexcept = default; }; struct UnwindResponse { TreeMeta meta; TreeDBStats stats; + + UnwindResponse() = default; + ~UnwindResponse() = default; + UnwindResponse(const UnwindResponse& other) = default; + UnwindResponse(UnwindResponse&& other) noexcept = default; + UnwindResponse& operator=(const UnwindResponse& other) = default; + UnwindResponse& operator=(UnwindResponse&& other) noexcept = default; }; struct RemoveHistoricResponse { TreeMeta meta; TreeDBStats stats; + + RemoveHistoricResponse() = default; + ~RemoveHistoricResponse() = default; + RemoveHistoricResponse(const RemoveHistoricResponse& other) = default; + RemoveHistoricResponse(RemoveHistoricResponse&& other) noexcept = default; + RemoveHistoricResponse& operator=(const RemoveHistoricResponse& other) = default; + RemoveHistoricResponse& operator=(RemoveHistoricResponse&& other) noexcept = default; }; template struct TypedResponse { ResponseType inner; bool success{ true }; std::string message; + + TypedResponse() = default; + ~TypedResponse() = default; + TypedResponse(const TypedResponse& other) = default; + TypedResponse(TypedResponse&& other) noexcept = default; + TypedResponse& operator=(const TypedResponse& other) = default; + TypedResponse& operator=(TypedResponse&& other) noexcept = default; }; struct Response { bool success; std::string message; + + Response(bool s, std::string m) + : success(s) + , message(std::move(m)) + {} + Response() = default; + ~Response() = default; + Response(const Response& other) = default; + Response(Response&& other) noexcept = default; + Response& operator=(const Response& other) = default; + Response& operator=(Response&& other) noexcept = default; }; template @@ -116,8 +239,7 @@ void execute_and_report(const std::function&)>& } } -inline void execute_and_report(const std::function& f, - const std::function& on_completion) +inline void execute_and_report(const std::function& f, const std::function& on_completion) { Response response{ true, "" }; try { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp index 1aa333d061b..354ba949c20 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp @@ -10,7 +10,10 @@ namespace bb::crypto::merkle_tree { -void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, index_t blockNumber, fr root, bool expectedSuccess) +void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, + block_number_t blockNumber, + fr root, + bool expectedSuccess) { BlockPayload blockData; LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); @@ -25,7 +28,7 @@ void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, index_t block } void inline check_block_and_root_data( - LMDBTreeStore::SharedPtr db, index_t blockNumber, fr root, bool expectedSuccess, bool expectedRootSuccess) + LMDBTreeStore::SharedPtr db, block_number_t blockNumber, fr root, bool expectedSuccess, bool expectedRootSuccess) { BlockPayload blockData; LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); @@ -40,7 +43,7 @@ void inline check_block_and_root_data( } void inline check_block_and_size_data(LMDBTreeStore::SharedPtr db, - index_t blockNumber, + block_number_t blockNumber, index_t expectedSize, bool expectedSuccess) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp index 8d9c5de4933..c8ce520fb9c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -6,10 +6,11 @@ #include namespace bb::crypto::merkle_tree { using index_t = uint64_t; +using block_number_t = uint64_t; struct RequestContext { bool includeUncommitted; - std::optional blockNumber; + std::optional blockNumber; bb::fr root; }; @@ -17,6 +18,7 @@ const std::string BLOCKS_DB = "blocks"; const std::string NODES_DB = "nodes"; const std::string LEAF_PREIMAGES_DB = "leaf preimages"; const std::string LEAF_INDICES_DB = "leaf indices"; +const std::string BLOCK_INDICES_DB = "block indices"; struct DBStats { std::string name; @@ -71,6 +73,7 @@ struct TreeDBStats { DBStats nodesDBStats; DBStats leafPreimagesDBStats; DBStats leafIndicesDBStats; + DBStats blockIndicesDBStats; TreeDBStats() = default; TreeDBStats(uint64_t mapSize) @@ -80,24 +83,27 @@ struct TreeDBStats { const DBStats& blockStats, const DBStats& nodesStats, const DBStats& leafPreimagesDBStats, - const DBStats& leafIndicesStats) + const DBStats& leafIndicesStats, + const DBStats& blockIndicesStats) : mapSize(mapSize) , blocksDBStats(blockStats) , nodesDBStats(nodesStats) , leafPreimagesDBStats(leafPreimagesDBStats) , leafIndicesDBStats(leafIndicesStats) + , blockIndicesDBStats(blockIndicesStats) {} TreeDBStats(const TreeDBStats& other) = default; TreeDBStats(TreeDBStats&& other) noexcept { *this = std::move(other); } ~TreeDBStats() = default; - MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafIndicesDBStats) + MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafIndicesDBStats, blockIndicesDBStats) bool operator==(const TreeDBStats& other) const { return mapSize == other.mapSize && blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats && - leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats; + leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats && + blockIndicesDBStats == other.blockIndicesDBStats; } TreeDBStats& operator=(TreeDBStats&& other) noexcept @@ -108,6 +114,7 @@ struct TreeDBStats { nodesDBStats = std::move(other.nodesDBStats); leafPreimagesDBStats = std::move(other.leafPreimagesDBStats); leafIndicesDBStats = std::move(other.leafIndicesDBStats); + blockIndicesDBStats = std::move(other.blockIndicesDBStats); } return *this; } @@ -118,7 +125,7 @@ struct TreeDBStats { { os << "Map Size: " << stats.mapSize << " Blocks DB " << stats.blocksDBStats << ", Nodes DB " << stats.nodesDBStats << ", Leaf Pre-images DB " << stats.leafPreimagesDBStats << ", Leaf Indices DB " - << stats.leafIndicesDBStats; + << stats.leafIndicesDBStats << ", Block Indices DB " << stats.blockIndicesDBStats; return os; } }; diff --git a/barretenberg/cpp/src/barretenberg/world_state/types.hpp b/barretenberg/cpp/src/barretenberg/world_state/types.hpp index 34e74a0631c..f5fbbcbf257 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/types.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/types.hpp @@ -32,7 +32,7 @@ using StateReference = std::unordered_map; struct WorldStateRevision { index_t forkId{ 0 }; - index_t blockNumber{ 0 }; + block_number_t blockNumber{ 0 }; bool includeUncommitted{ false }; MSGPACK_FIELDS(forkId, blockNumber, includeUncommitted) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 9cb3b36bf6c..7e74fe44896 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -142,7 +142,7 @@ Fork::SharedPtr WorldState::retrieve_fork(const uint64_t& forkId) const } uint64_t WorldState::create_fork(const std::optional& blockNumber) { - index_t blockNumberForFork = 0; + block_number_t blockNumberForFork = 0; if (!blockNumber.has_value()) { // we are forking at latest WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; @@ -159,7 +159,7 @@ uint64_t WorldState::create_fork(const std::optional& blockNumber) return forkId; } -void WorldState::remove_forks_for_block(const index_t& blockNumber) +void WorldState::remove_forks_for_block(const block_number_t& blockNumber) { // capture the shared pointers outside of the lock scope so we are not under the lock when the objects are destroyed std::vector forks; @@ -191,7 +191,7 @@ void WorldState::delete_fork(const uint64_t& forkId) } } -Fork::SharedPtr WorldState::create_new_fork(const index_t& blockNumber) +Fork::SharedPtr WorldState::create_new_fork(const block_number_t& blockNumber) { Fork::SharedPtr fork = std::make_shared(); fork->_blockNumber = blockNumber; @@ -241,10 +241,10 @@ TreeMetaResponse WorldState::get_tree_info(const WorldStateRevision& revision, M return std::visit( [=](auto&& wrapper) { Signal signal(1); - TreeMetaResponse response; + TypedResponse local; - auto callback = [&](const TypedResponse& meta) { - response = meta.inner; + auto callback = [&](TypedResponse& meta) { + local = std::move(meta); signal.signal_level(0); }; @@ -255,12 +255,15 @@ TreeMetaResponse WorldState::get_tree_info(const WorldStateRevision& revision, M } signal.wait_for_level(0); - return response; + if (!local.success) { + throw std::runtime_error(local.message); + } + return local.inner; }, fork->_trees.at(tree_id)); } -void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const +void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const { Fork::SharedPtr fork = retrieve_fork(revision.forkId); @@ -271,13 +274,14 @@ void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::arra Signal signal(static_cast(tree_ids.size())); std::mutex mutex; + std::unordered_map> local; for (auto id : tree_ids) { const auto& tree = fork->_trees.at(id); - auto callback = [&signal, &responses, &mutex, id](const TypedResponse& meta) { + auto callback = [&signal, &local, &mutex, id](TypedResponse& meta) { { std::lock_guard lock(mutex); - responses[id] = meta.inner.meta; + local[id] = std::move(meta); } signal.signal_decrement(); }; @@ -293,6 +297,14 @@ void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::arra } signal.wait_for_level(0); + + for (auto tree_id : tree_ids) { + auto& m = local[tree_id]; + if (!m.success) { + throw std::runtime_error(m.message); + } + responses[tree_id] = std::move(m.inner.meta); + } } StateReference WorldState::get_state_reference(const WorldStateRevision& revision) const @@ -324,19 +336,15 @@ StateReference WorldState::get_state_reference(const WorldStateRevision& revisio Signal signal(static_cast(tree_ids.size())); StateReference state_reference; + std::unordered_map> local; std::mutex state_ref_mutex; for (auto id : tree_ids) { const auto& tree = fork->_trees.at(id); - auto callback = [&signal, &state_reference, &state_ref_mutex, initial_state, id]( - const TypedResponse& meta) { + auto callback = [&signal, &local, &state_ref_mutex, id](TypedResponse& meta) { { std::lock_guard lock(state_ref_mutex); - if (initial_state) { - state_reference.insert({ id, { meta.inner.meta.initialRoot, meta.inner.meta.initialSize } }); - } else { - state_reference.insert({ id, { meta.inner.meta.root, meta.inner.meta.size } }); - } + local[id] = std::move(meta); } signal.signal_decrement(); }; @@ -352,6 +360,19 @@ StateReference WorldState::get_state_reference(const WorldStateRevision& revisio } signal.wait_for_level(0); + + for (auto tree_id : tree_ids) { + auto& m = local[tree_id]; + if (!m.success) { + throw std::runtime_error(m.message); + } + if (initial_state) { + state_reference[tree_id] = std::make_pair(m.inner.meta.initialRoot, m.inner.meta.initialSize); + continue; + } + state_reference[tree_id] = std::make_pair(m.inner.meta.root, m.inner.meta.size); + } + return state_reference; } @@ -364,10 +385,10 @@ fr_sibling_path WorldState::get_sibling_path(const WorldStateRevision& revision, return std::visit( [leaf_index, revision](auto&& wrapper) { Signal signal(1); - fr_sibling_path path; + TypedResponse local; - auto callback = [&signal, &path](const TypedResponse& response) { - path = response.inner.path; + auto callback = [&signal, &local](TypedResponse& response) { + local = std::move(response); signal.signal_level(0); }; @@ -378,7 +399,42 @@ fr_sibling_path WorldState::get_sibling_path(const WorldStateRevision& revision, } signal.wait_for_level(0); - return path; + if (!local.success) { + throw std::runtime_error(local.message); + } + return local.inner.path; + }, + fork->_trees.at(tree_id)); +} + +void WorldState::get_block_numbers_for_leaf_indices(const WorldStateRevision& revision, + MerkleTreeId tree_id, + const std::vector& leafIndices, + std::vector>& blockNumbers) const +{ + Fork::SharedPtr fork = retrieve_fork(revision.forkId); + + std::visit( + [&leafIndices, revision, &blockNumbers](auto&& wrapper) { + Signal signal(1); + TypedResponse local; + + auto callback = [&signal, &local](TypedResponse& response) { + local = std::move(response); + signal.signal_level(); + }; + + if (revision.blockNumber) { + wrapper.tree->find_block_numbers(leafIndices, revision.blockNumber, callback); + } else { + wrapper.tree->find_block_numbers(leafIndices, callback); + } + signal.wait_for_level(0); + + if (!local.success) { + throw std::runtime_error(local.message); + } + blockNumbers = std::move(local.inner.blockNumbers); }, fork->_trees.at(tree_id)); } @@ -534,7 +590,15 @@ WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_re { auto& wrapper = std::get>(fork->_trees.at(MerkleTreeId::PUBLIC_DATA_TREE)); - PublicDataTree::AddCompletionCallback completion = [&](const auto&) -> void { signal.signal_decrement(); }; + PublicDataTree::AddCompletionCallback completion = [&](const auto& resp) -> void { + // take the first error + bool expected = true; + if (!resp.success && success.compare_exchange_strong(expected, false)) { + err_message = resp.message; + } + + signal.signal_decrement(); + }; wrapper.tree->add_or_update_values_sequentially(public_writes, completion); } @@ -566,9 +630,9 @@ GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevisi { Fork::SharedPtr fork = retrieve_fork(revision.forkId); Signal signal; - GetLowIndexedLeafResponse low_leaf_info; - auto callback = [&signal, &low_leaf_info](const TypedResponse& response) { - low_leaf_info = response.inner; + TypedResponse low_leaf_info; + auto callback = [&signal, &low_leaf_info](TypedResponse& response) { + low_leaf_info = std::move(response); signal.signal_level(); }; @@ -591,7 +655,11 @@ GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevisi } signal.wait_for_level(); - return low_leaf_info; + + if (!low_leaf_info.success) { + throw std::runtime_error(low_leaf_info.message); + } + return low_leaf_info.inner; } WorldStateStatusSummary WorldState::set_finalised_blocks(const index_t& toBlockNumber) @@ -599,11 +667,13 @@ WorldStateStatusSummary WorldState::set_finalised_blocks(const index_t& toBlockN WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; TreeMetaResponse archive_state = get_tree_info(revision, MerkleTreeId::ARCHIVE); if (toBlockNumber <= archive_state.meta.finalisedBlockHeight) { - throw std::runtime_error("Unable to finalise block, already finalised"); - } - if (!set_finalised_block(toBlockNumber)) { - throw std::runtime_error("Failed to set finalised block"); + throw std::runtime_error(format("Unable to finalise blocks to block number ", + toBlockNumber, + ", current finalised block: ", + archive_state.meta.finalisedBlockHeight)); } + // This will throw if it fails + set_finalised_block(toBlockNumber); WorldStateStatusSummary status; get_status_summary(status); return status; @@ -616,10 +686,10 @@ WorldStateStatusFull WorldState::unwind_blocks(const index_t& toBlockNumber) throw std::runtime_error("Unable to unwind block, block not found"); } WorldStateStatusFull status; - for (index_t blockNumber = archive_state.meta.unfinalisedBlockHeight; blockNumber > toBlockNumber; blockNumber--) { - if (!unwind_block(blockNumber, status)) { - throw std::runtime_error("Failed to unwind block"); - } + for (block_number_t blockNumber = archive_state.meta.unfinalisedBlockHeight; blockNumber > toBlockNumber; + blockNumber--) { + // This will throw if it fails + unwind_block(blockNumber, status); } populate_status_summary(status); return status; @@ -635,35 +705,43 @@ WorldStateStatusFull WorldState::remove_historical_blocks(const index_t& toBlock archive_state.meta.oldestHistoricBlock)); } WorldStateStatusFull status; - for (index_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; blockNumber++) { - if (!remove_historical_block(blockNumber, status)) { - throw std::runtime_error(format( - "Failed to remove historical block ", blockNumber, " when removing blocks up to ", toBlockNumber)); - } + for (block_number_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; + blockNumber++) { + // This will throw if it fails + remove_historical_block(blockNumber, status); } populate_status_summary(status); return status; } -bool WorldState::set_finalised_block(const index_t& blockNumber) +bool WorldState::set_finalised_block(const block_number_t& blockNumber) { - std::atomic_bool success = true; Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); Signal signal(static_cast(fork->_trees.size())); + std::array local; + std::mutex mtx; for (auto& [id, tree] : fork->_trees) { std::visit( - [&signal, &success, blockNumber](auto&& wrapper) { - wrapper.tree->finalise_block(blockNumber, [&signal, &success](const Response& resp) { - success = success && resp.success; + [&signal, &local, blockNumber, id, &mtx](auto&& wrapper) { + wrapper.tree->finalise_block(blockNumber, [&signal, &local, &mtx, id](Response& resp) { + { + std::lock_guard lock(mtx); + local[id] = std::move(resp); + } signal.signal_decrement(); }); }, tree); } signal.wait_for_level(); - return success; + for (auto& m : local) { + if (!m.success) { + throw std::runtime_error(m.message); + } + } + return true; } -bool WorldState::unwind_block(const index_t& blockNumber, WorldStateStatusFull& status) +bool WorldState::unwind_block(const block_number_t& blockNumber, WorldStateStatusFull& status) { std::atomic_bool success = true; std::string message; @@ -723,10 +801,13 @@ bool WorldState::unwind_block(const index_t& blockNumber, WorldStateStatusFull& blockNumber); } signal.wait_for_level(); + if (!success) { + throw std::runtime_error(message); + } remove_forks_for_block(blockNumber); - return success; + return true; } -bool WorldState::remove_historical_block(const index_t& blockNumber, WorldStateStatusFull& status) +bool WorldState::remove_historical_block(const block_number_t& blockNumber, WorldStateStatusFull& status) { std::atomic_bool success = true; std::string message; @@ -786,8 +867,11 @@ bool WorldState::remove_historical_block(const index_t& blockNumber, WorldStateS blockNumber); } signal.wait_for_level(); + if (!success) { + throw std::runtime_error(message); + } remove_forks_for_block(blockNumber); - return success; + return true; } bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_ref, uint32_t generator_point) @@ -829,7 +913,12 @@ bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_r bool WorldState::is_archive_tip(const WorldStateRevision& revision, const bb::fr& block_header_hash) const { - std::optional leaf_index = find_leaf_index(revision, MerkleTreeId::ARCHIVE, block_header_hash); + std::optional leaf_index = std::nullopt; + + try { + leaf_index = find_leaf_index(revision, MerkleTreeId::ARCHIVE, block_header_hash); + } catch (std::runtime_error&) { + } if (!leaf_index.has_value()) { return false; @@ -887,7 +976,7 @@ void WorldState::validate_trees_are_equally_synched() bool WorldState::determine_if_synched(std::array& metaResponses) { - index_t blockNumber = metaResponses[0].unfinalisedBlockHeight; + block_number_t blockNumber = metaResponses[0].unfinalisedBlockHeight; for (size_t i = 1; i < metaResponses.size(); i++) { if (blockNumber != metaResponses[i].unfinalisedBlockHeight) { return false; diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index 5afa4a010ef..c66412aae77 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -107,6 +107,11 @@ class WorldState { MerkleTreeId tree_id, index_t leaf_index) const; + void get_block_numbers_for_leaf_indices(const WorldStateRevision& revision, + MerkleTreeId tree_id, + const std::vector& leafIndices, + std::vector>& blockNumbers) const; + /** * @brief Get the leaf preimage object * @@ -259,12 +264,12 @@ class WorldState { uint64_t maxReaders); Fork::SharedPtr retrieve_fork(const uint64_t& forkId) const; - Fork::SharedPtr create_new_fork(const index_t& blockNumber); - void remove_forks_for_block(const index_t& blockNumber); + Fork::SharedPtr create_new_fork(const block_number_t& blockNumber); + void remove_forks_for_block(const block_number_t& blockNumber); - bool unwind_block(const index_t& blockNumber, WorldStateStatusFull& status); - bool remove_historical_block(const index_t& blockNumber, WorldStateStatusFull& status); - bool set_finalised_block(const index_t& blockNumber); + bool unwind_block(const block_number_t& blockNumber, WorldStateStatusFull& status); + bool remove_historical_block(const block_number_t& blockNumber, WorldStateStatusFull& status); + bool set_finalised_block(const block_number_t& blockNumber); void get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const; @@ -304,7 +309,7 @@ class WorldState { std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber); + const block_number_t& blockNumber); template void remove_historic_block_for_tree(TreeDBStats& dbStats, @@ -313,7 +318,7 @@ class WorldState { std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber); + const block_number_t& blockNumber); }; template @@ -342,7 +347,7 @@ void WorldState::unwind_tree(TreeDBStats& dbStats, std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber) + const block_number_t& blockNumber) { tree.unwind_block(blockNumber, [&](TypedResponse& response) { bool expected = true; @@ -362,7 +367,7 @@ void WorldState::remove_historic_block_for_tree(TreeDBStats& dbStats, std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber) + const block_number_t& blockNumber) { tree.remove_historic_block(blockNumber, [&](TypedResponse& response) { bool expected = true; @@ -384,15 +389,13 @@ std::optional> WorldState::get_indexed_leaf( using Tree = ContentAddressedIndexedTree; Fork::SharedPtr fork = retrieve_fork(rev.forkId); + TypedResponse> local; if (auto* const wrapper = std::get_if>(&fork->_trees.at(id))) { - std::optional> value; - Signal signal; - auto callback = [&](const TypedResponse>& response) { - if (response.inner.indexed_leaf.has_value()) { - value = response.inner.indexed_leaf; - } + Signal signal; + auto callback = [&](TypedResponse>& response) { + local = std::move(response); signal.signal_level(0); }; @@ -403,7 +406,11 @@ std::optional> WorldState::get_indexed_leaf( } signal.wait_for_level(); - return value; + if (!local.success) { + throw std::runtime_error("Failed to find indexed leaf: " + local.message); + } + + return local.inner.indexed_leaf; } throw std::runtime_error("Invalid tree type"); @@ -419,12 +426,17 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, Fork::SharedPtr fork = retrieve_fork(revision.forkId); std::optional leaf; + bool success = true; + std::string error_msg; Signal signal; if constexpr (std::is_same_v) { const auto& wrapper = std::get>(fork->_trees.at(tree_id)); - auto callback = [&signal, &leaf](const TypedResponse& resp) { - if (resp.inner.leaf.has_value()) { - leaf = resp.inner.leaf.value(); + auto callback = [&signal, &leaf, &success, &error_msg](const TypedResponse& response) { + if (!response.success || !response.inner.leaf.has_value()) { + success = false; + error_msg = response.message; + } else { + leaf = response.inner.leaf; } signal.signal_level(); }; @@ -439,12 +451,16 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, using Tree = ContentAddressedIndexedTree; auto& wrapper = std::get>(fork->_trees.at(tree_id)); - auto callback = [&signal, &leaf](const TypedResponse>& resp) { - if (resp.inner.indexed_leaf.has_value()) { - leaf = resp.inner.indexed_leaf.value().value; - } - signal.signal_level(); - }; + auto callback = + [&signal, &leaf, &success, &error_msg](const TypedResponse>& response) { + if (!response.success || !response.inner.indexed_leaf.has_value()) { + success = false; + error_msg = response.message; + } else { + leaf = response.inner.indexed_leaf.value().value; + } + signal.signal_level(); + }; if (revision.blockNumber) { wrapper.tree->get_leaf(leaf_index, revision.blockNumber, revision.includeUncommitted, callback); @@ -454,6 +470,7 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, } signal.wait_for_level(); + return leaf; } @@ -464,15 +481,13 @@ std::optional WorldState::find_leaf_index(const WorldStateRevision& rev index_t start_index) const { using namespace crypto::merkle_tree; - std::optional index; Fork::SharedPtr fork = retrieve_fork(rev.forkId); + TypedResponse local; Signal signal; - auto callback = [&](const TypedResponse& response) { - if (response.success) { - index = response.inner.leaf_index; - } + auto callback = [&](TypedResponse& response) { + local = std::move(response); signal.signal_level(0); }; if constexpr (std::is_same_v) { @@ -496,7 +511,12 @@ std::optional WorldState::find_leaf_index(const WorldStateRevision& rev } signal.wait_for_level(0); - return index; + + if (!local.success) { + return std::nullopt; + } + + return local.inner.leaf_index; } template void WorldState::append_leaves(MerkleTreeId id, const std::vector& leaves, Fork::Id fork_id) @@ -537,7 +557,7 @@ template void WorldState::append_leaves(MerkleTreeId id, const std: signal.wait_for_level(0); if (!success) { - throw std::runtime_error("Failed to append leaves: " + error_msg); + throw std::runtime_error(error_msg); } } @@ -576,7 +596,7 @@ BatchInsertionResult WorldState::batch_insert_indexed_leaves(MerkleTreeId id, signal.wait_for_level(); if (!success) { - throw std::runtime_error("Failed to batch insert indexed leaves: " + error_msg); + throw std::runtime_error(error_msg); } return result; @@ -615,7 +635,7 @@ SequentialInsertionResult WorldState::insert_indexed_leaves(MerkleTreeId id, signal.wait_for_level(); if (!success) { - throw std::runtime_error("Failed to sequentially insert indexed leaves: " + error_msg); + throw std::runtime_error(error_msg); } return result; diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp index 024ffbf4ac0..a5ced2921ad 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -1,6 +1,7 @@ #include "barretenberg/world_state/world_state.hpp" #include "barretenberg/crypto/merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_read_transaction.hpp" #include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" @@ -11,6 +12,7 @@ #include #include #include +#include #include #include @@ -79,6 +81,10 @@ void assert_leaf_index( const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, const Leaf& value, index_t expected_index) { std::optional index = ws.find_leaf_index(revision, tree_id, value); + EXPECT_TRUE(index.has_value()); + if (!index.has_value()) { + return; + } EXPECT_EQ(index.value(), expected_index); } @@ -363,6 +369,7 @@ TEST_F(WorldStateTest, NullifierTree) auto test_leaf = ws.get_indexed_leaf(WorldStateRevision::committed(), tree_id, 128); // at this point 142 should be the biggest leaf so it wraps back to 0 + EXPECT_TRUE(test_leaf.has_value()); EXPECT_EQ(test_leaf.value(), IndexedLeaf(test_nullifier, 0, 0)); auto predecessor_of_142_again = @@ -524,6 +531,19 @@ TEST_F(WorldStateTest, SyncExternalBlockFromEmpty) for (const auto& [tree_id, snapshot] : block_state_ref) { EXPECT_EQ(state_ref.at(tree_id), snapshot); } + + std::vector> blockNumbers; + ws.get_block_numbers_for_leaf_indices( + WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, { 0 }, blockNumbers); + EXPECT_EQ(blockNumbers.size(), 1); + EXPECT_EQ(blockNumbers[0], 1); + + EXPECT_THROW(ws.get_block_numbers_for_leaf_indices( + WorldStateRevision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 2, .includeUncommitted = false }, + MerkleTreeId::NOTE_HASH_TREE, + { 0 }, + blockNumbers), + std::runtime_error); } TEST_F(WorldStateTest, SyncBlockFromDirtyState) @@ -789,3 +809,43 @@ TEST_F(WorldStateTest, BuildsABlockInAFork) EXPECT_EQ(fork_state_ref, ws.get_state_reference(WorldStateRevision::committed())); } + +TEST_F(WorldStateTest, GetBlockForIndex) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + // bb::fr block_hash(1); + StateReference block_state_ref = { + { MerkleTreeId::NULLIFIER_TREE, + { fr("0x187a19972150cd1e76d8201d720da7682fcf4d93ec6a3c7b0d84bbefde5bd927"), 129 } }, + { MerkleTreeId::NOTE_HASH_TREE, + { fr("0x2467e5f90736b4ea977e7d21cfb3714181e16b7d6cd867768b59e2ea90fa3eaf"), 1 } }, + { MerkleTreeId::PUBLIC_DATA_TREE, + { fr("0x0278dcf9ff541da255ee722aecfad849b66af0d42c2924d949b5a509f2e1aec9"), 129 } }, + { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + { fr("0x24ffd0fab86555ab2e86cffc706d4cfb4b8c405c3966af805de954504ffc27ac"), 1 } }, + }; + + WorldStateStatusFull status = ws.sync_block( + block_state_ref, fr(1), { 42 }, { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }); + WorldStateStatusSummary expected{ 1, 0, 1, true }; + EXPECT_EQ(status.summary, expected); + + StateReference state_ref = ws.get_state_reference(WorldStateRevision::committed()); + + std::vector tree_ids{ + MerkleTreeId::NULLIFIER_TREE, + MerkleTreeId::NOTE_HASH_TREE, + MerkleTreeId::PUBLIC_DATA_TREE, + MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + }; + + for (const auto& id : tree_ids) { + std::vector> blockNumbers; + ws.get_block_numbers_for_leaf_indices( + WorldStateRevision::committed(), id, { state_ref[id].second - 1 }, blockNumbers); + + EXPECT_EQ(blockNumbers.size(), 1); + EXPECT_TRUE(blockNumbers[0].has_value()); + EXPECT_EQ(blockNumbers[0].value(), 1); + } +} diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp index d8ac68d0db1..f3290da5e9e 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp @@ -150,6 +150,11 @@ WorldStateAddon::WorldStateAddon(const Napi::CallbackInfo& info) WorldStateMessageType::GET_SIBLING_PATH, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_sibling_path(obj, buffer); }); + _dispatcher.registerTarget(WorldStateMessageType::GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { + return get_block_numbers_for_leaf_indices(obj, buffer); + }); + _dispatcher.registerTarget( WorldStateMessageType::FIND_LEAF_INDEX, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_leaf_index(obj, buffer); }); @@ -387,6 +392,24 @@ bool WorldStateAddon::get_sibling_path(msgpack::object& obj, msgpack::sbuffer& b return true; } +bool WorldStateAddon::get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + + GetBlockNumbersForLeafIndicesResponse response; + _ws->get_block_numbers_for_leaf_indices( + request.value.revision, request.value.treeId, request.value.leafIndices, response.blockNumbers); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg( + WorldStateMessageType::GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, header, response); + + msgpack::pack(buffer, resp_msg); + + return true; +} + bool WorldStateAddon::find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const { TypedMessage request; diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp index 1a077a946e9..d0b33be2532 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp @@ -38,6 +38,7 @@ class WorldStateAddon : public Napi::ObjectWrap { bool get_leaf_value(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool get_leaf_preimage(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool get_sibling_path(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool find_low_leaf(msgpack::object& obj, msgpack::sbuffer& buffer) const; diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp index c876f5993bc..b98a8c6a69d 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp @@ -1,5 +1,6 @@ #pragma once #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/msgpack.hpp" @@ -20,6 +21,7 @@ enum WorldStateMessageType { GET_LEAF_VALUE, GET_LEAF_PREIMAGE, GET_SIBLING_PATH, + GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, FIND_LEAF_INDEX, FIND_LOW_LEAF, @@ -54,7 +56,7 @@ struct TreeIdOnlyRequest { struct CreateForkRequest { bool latest; - index_t blockNumber; + block_number_t blockNumber; MSGPACK_FIELDS(latest, blockNumber); }; @@ -129,6 +131,18 @@ struct GetSiblingPathRequest { MSGPACK_FIELDS(treeId, revision, leafIndex); }; +struct GetBlockNumbersForLeafIndicesRequest { + MerkleTreeId treeId; + WorldStateRevision revision; + std::vector leafIndices; + MSGPACK_FIELDS(treeId, revision, leafIndices); +}; + +struct GetBlockNumbersForLeafIndicesResponse { + std::vector> blockNumbers; + MSGPACK_FIELDS(blockNumbers); +}; + template struct FindLeafIndexRequest { MerkleTreeId treeId; WorldStateRevision revision; diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 57f576c55d3..4434785d41c 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -435,6 +435,22 @@ export class AztecNodeService implements AztecNode { return await Promise.all(leafValues.map(leafValue => committedDb.findLeafIndex(treeId, leafValue.toBuffer()))); } + /** + * Find the block numbers of the given leaf indices in the given tree. + * @param blockNumber - The block number at which to get the data or 'latest' for latest data + * @param treeId - The tree to search in. + * @param leafIndices - The values to search for + * @returns The indexes of the given leaves in the given tree or undefined if not found. + */ + public async findBlockNumbersForIndexes( + blockNumber: L2BlockNumber, + treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + const committedDb = await this.#getWorldState(blockNumber); + return await committedDb.getBlockNumbersForLeafIndices(treeId, leafIndices); + } + public async findNullifiersIndexesWithBlock( blockNumber: L2BlockNumber, nullifiers: Fr[], diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index 4e3f9bd4a8d..c2db85e5250 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -91,6 +91,11 @@ describe('AztecNodeApiSchema', () => { expect(response).toEqual([1n, undefined]); }); + it('findBlockNumbersForIndexes', async () => { + const response = await context.client.findBlockNumbersForIndexes(1, MerkleTreeId.ARCHIVE, [5n, 58n]); + expect(response).toEqual([3n, 9n]); + }); + it('findNullifiersIndexesWithBlock', async () => { const response = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); expect(response).toEqual([ @@ -349,6 +354,15 @@ class MockAztecNode implements AztecNode { expect(leafValues[1]).toBeInstanceOf(Fr); return Promise.resolve([1n, undefined]); } + + findBlockNumbersForIndexes( + _blockNumber: number | 'latest', + _treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + expect(leafIndices).toEqual([5n, 58n]); + return Promise.resolve([3n, 9n]); + } findNullifiersIndexesWithBlock( blockNumber: number | 'latest', nullifiers: Fr[], diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 8e5b2bda55f..457d188acf0 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -61,7 +61,7 @@ export interface AztecNode * Find the indexes of the given leaves in the given tree. * @param blockNumber - The block number at which to get the data or 'latest' for latest data * @param treeId - The tree to search in. - * @param leafValue - The values to search for + * @param leafValues - The values to search for * @returns The indexes of the given leaves in the given tree or undefined if not found. */ findLeavesIndexes( @@ -70,6 +70,19 @@ export interface AztecNode leafValues: Fr[], ): Promise<(bigint | undefined)[]>; + /** + * Find the indexes of the given leaves in the given tree. + * @param blockNumber - The block number at which to get the data or 'latest' for latest data + * @param treeId - The tree to search in. + * @param leafIndices - The values to search for + * @returns The indexes of the given leaves in the given tree or undefined if not found. + */ + findBlockNumbersForIndexes( + blockNumber: L2BlockNumber, + treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]>; + /** * Returns the indexes of the given nullifiers in the nullifier tree, * scoped to the block they were included in. @@ -430,6 +443,11 @@ export const AztecNodeApiSchema: ApiSchemaFor = { .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) .returns(z.array(optional(schemas.BigInt))), + findBlockNumbersForIndexes: z + .function() + .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.BigInt)) + .returns(z.array(optional(schemas.BigInt))), + findNullifiersIndexesWithBlock: z .function() .args(L2BlockNumberSchema, z.array(schemas.Fr)) diff --git a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts index 8520da7ca55..9017c1a6a84 100644 --- a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts @@ -199,6 +199,16 @@ export interface MerkleTreeReadOperations { treeId: ID, index: bigint, ): Promise | undefined>; + + /** + * Get the block numbers for a set of leaf indices + * @param treeId - The tree for which the block numbers should be returned. + * @param leafIndices - The indices to be queried. + */ + getBlockNumbersForLeafIndices( + treeId: ID, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]>; } export interface MerkleTreeWriteOperations extends MerkleTreeReadOperations { diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 01f3e483497..853ce0bb58f 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -173,6 +173,24 @@ export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state. export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.leaf_indices.message'; export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.leaf_indices.note_hash'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NULLIFIER = + 'aztec.world_state.db_used_size.block_indices.nullifier'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_PUBLIC_DATA = + 'aztec.world_state.db_used_size.block_indices.public_data'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.block_indices.archive'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.block_indices.message'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NOTE_HASH = + 'aztec.world_state.db_used_size.block_indices.note_hash'; + +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NULLIFIER = + 'aztec.world_state.db_num_items.block_indices.nullifier'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_PUBLIC_DATA = + 'aztec.world_state.db_num_items.block_indices.public_data'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.block_indices.archive'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.block_indices.message'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NOTE_HASH = + 'aztec.world_state.db_num_items.block_indices.note_hash'; + export const PROOF_VERIFIER_COUNT = 'aztec.proof_verifier.count'; export const VALIDATOR_RE_EXECUTION_TIME = 'aztec.validator.re_execution_time'; diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index 49d53e8f3a5..9ce07806d0e 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -166,6 +166,19 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations { treeId, }; } + + async getBlockNumbersForLeafIndices( + treeId: ID, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + const response = await this.instance.call(WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, { + treeId, + revision: this.revision, + leafIndices, + }); + + return response.blockNumbers.map(x => (x === undefined || x === null ? undefined : BigInt(x))); + } } export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTreeWriteOperations { diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index 12c4c410edc..6f95755ce7c 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -54,6 +54,7 @@ export enum WorldStateMessageType { GET_LEAF_VALUE, GET_LEAF_PREIMAGE, GET_SIBLING_PATH, + GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, FIND_LEAF_INDEX, FIND_LOW_LEAF, @@ -139,6 +140,8 @@ export interface TreeDBStats { leafPreimagesDBStats: DBStats; /** Stats for the 'leaf indices' DB */ leafIndicesDBStats: DBStats; + /** Stats for the 'block indices' DB */ + blockIndicesDBStats: DBStats; } export interface WorldStateMeta { @@ -189,6 +192,7 @@ export function buildEmptyTreeDBStats() { leafIndicesDBStats: buildEmptyDBStats(), leafKeysDBStats: buildEmptyDBStats(), leafPreimagesDBStats: buildEmptyDBStats(), + blockIndicesDBStats: buildEmptyDBStats(), } as TreeDBStats; } @@ -271,6 +275,7 @@ export function sanitiseTreeDBStats(stats: TreeDBStats) { stats.blocksDBStats = sanitiseDBStats(stats.blocksDBStats); stats.leafIndicesDBStats = sanitiseDBStats(stats.leafIndicesDBStats); stats.leafPreimagesDBStats = sanitiseDBStats(stats.leafPreimagesDBStats); + stats.blockIndicesDBStats = sanitiseDBStats(stats.blockIndicesDBStats); stats.nodesDBStats = sanitiseDBStats(stats.nodesDBStats); stats.mapSize = BigInt(stats.mapSize); return stats; @@ -344,6 +349,14 @@ interface GetTreeInfoResponse { root: Buffer; } +interface GetBlockNumbersForLeafIndicesRequest extends WithTreeId, WithWorldStateRevision { + leafIndices: bigint[]; +} + +interface GetBlockNumbersForLeafIndicesResponse { + blockNumbers: bigint[]; +} + interface GetSiblingPathRequest extends WithTreeId, WithLeafIndex, WithWorldStateRevision {} type GetSiblingPathResponse = Buffer[]; @@ -446,6 +459,7 @@ export type WorldStateRequest = { [WorldStateMessageType.GET_LEAF_VALUE]: GetLeafRequest; [WorldStateMessageType.GET_LEAF_PREIMAGE]: GetLeafPreImageRequest; [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathRequest; + [WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]: GetBlockNumbersForLeafIndicesRequest; [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexRequest; [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafRequest; @@ -481,6 +495,7 @@ export type WorldStateResponse = { [WorldStateMessageType.GET_LEAF_VALUE]: GetLeafResponse; [WorldStateMessageType.GET_LEAF_PREIMAGE]: GetLeafPreImageResponse; [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathResponse; + [WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]: GetBlockNumbersForLeafIndicesResponse; [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexResponse; [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafResponse; diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index d18593fcbf0..91044fdef56 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -490,10 +490,75 @@ describe('NativeWorldState', () => { }); }); + describe('block numbers for indices', () => { + let block: L2Block; + let messages: Fr[]; + let noteHashes: number; + let nullifiers: number; + let publicTree: number; + + beforeAll(async () => { + await rm(dataDir, { recursive: true }); + }); + + it('correctly reports block numbers', async () => { + const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); + const statuses = []; + const numBlocks = 2; + const txsPerBlock = 2; + for (let i = 0; i < numBlocks; i++) { + const fork = await ws.fork(); + ({ block, messages } = await mockBlock(1, txsPerBlock, fork)); + noteHashes = block.body.txEffects[0].noteHashes.length; + nullifiers = block.body.txEffects[0].nullifiers.length; + publicTree = block.body.txEffects[0].publicDataWrites.length; + await fork.close(); + const status = await ws.handleL2BlockAndMessages(block, messages); + statuses.push(status); + } + + const checkTree = async ( + treeId: MerkleTreeId, + itemsLength: number, + blockNumber: number, + initialSize: number, + numPerBlock: number, + ) => { + const before = initialSize + itemsLength * blockNumber * numPerBlock - 2; + const on = before + 1; + const after = on + 1; + const blockNumbers = await ws.getCommitted().getBlockNumbersForLeafIndices( + treeId, + [before, on, after].map(x => BigInt(x)), + ); + expect(blockNumbers).toEqual([blockNumber, blockNumber, blockNumber + 1].map(x => BigInt(x))); + }; + + for (let i = 0; i < numBlocks - 1; i++) { + await checkTree(MerkleTreeId.NOTE_HASH_TREE, noteHashes, i + 1, 0, 2); + await checkTree(MerkleTreeId.NULLIFIER_TREE, nullifiers, i + 1, 128, 2); + await checkTree(MerkleTreeId.PUBLIC_DATA_TREE, publicTree, i + 1, 128, 2); + await checkTree(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, messages.length, i + 1, 0, 1); + } + + const lastStatus = statuses[statuses.length - 1]; + const before = Number(lastStatus.meta.noteHashTreeMeta.committedSize) - 2; + const blockNumbers = await ws.getCommitted().getBlockNumbersForLeafIndices( + MerkleTreeId.NOTE_HASH_TREE, + [before, before + 1, before + 2].map(x => BigInt(x)), + ); + expect(blockNumbers).toEqual([2, 2, undefined].map(x => (x == undefined ? x : BigInt(x)))); + }); + }); + describe('status reporting', () => { let block: L2Block; let messages: Fr[]; + beforeAll(async () => { + await rm(dataDir, { recursive: true }); + }); + it('correctly reports status', async () => { const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); const statuses = []; diff --git a/yarn-project/world-state/src/synchronizer/instrumentation.ts b/yarn-project/world-state/src/synchronizer/instrumentation.ts index d52dca0aef4..9b4fb6a3480 100644 --- a/yarn-project/world-state/src/synchronizer/instrumentation.ts +++ b/yarn-project/world-state/src/synchronizer/instrumentation.ts @@ -5,7 +5,7 @@ import { type Gauge, type Meter, type TelemetryClient, ValueType } from '@aztec/ import { type DBStats, type TreeDBStats, type TreeMeta, type WorldStateStatusFull } from '../native/message.js'; type TreeTypeString = 'nullifier' | 'note_hash' | 'archive' | 'message' | 'public_data'; -type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks'; +type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks' | 'block_indices'; class TreeDBInstrumentation { private dbNumItems: Gauge; @@ -70,6 +70,7 @@ class TreeInstrumentation { this.treeDbInstrumentation.set('nodes', new TreeDBInstrumentation(meter, treeName, 'nodes')); this.treeDbInstrumentation.set('leaf_preimage', new TreeDBInstrumentation(meter, treeName, 'leaf_preimage')); this.treeDbInstrumentation.set('leaf_indices', new TreeDBInstrumentation(meter, treeName, 'leaf_indices')); + this.treeDbInstrumentation.set('block_indices', new TreeDBInstrumentation(meter, treeName, 'block_indices')); } private updateDBMetrics(dbName: DBTypeString, dbStats: DBStats) { @@ -92,6 +93,7 @@ class TreeInstrumentation { this.updateDBMetrics('leaf_preimage', treeDbStats.leafPreimagesDBStats); this.updateDBMetrics('blocks', treeDbStats.blocksDBStats); this.updateDBMetrics('nodes', treeDbStats.nodesDBStats); + this.updateDBMetrics('block_indices', treeDbStats.blockIndicesDBStats); } } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts index 82f5934fb18..a1b93999290 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts @@ -181,6 +181,13 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations throw new Error('Method not implemented in legacy merkle tree'); } + getBlockNumbersForLeafIndices( + _treeId: ID, + _leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + throw new Error('Method not implemented in legacy merkle tree'); + } + close(): Promise { return Promise.resolve(); } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts index 90560ef0960..7f4e4bc9d6a 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts @@ -101,6 +101,10 @@ export class MerkleTreeSnapshotOperationsFacade implements MerkleTreeReadOperati }; } + getBlockNumbersForLeafIndices(_a: ID, _b: bigint[]): Promise<(bigint | undefined)[]> { + throw new Error('Not implemented'); + } + async getStateReference(): Promise { const snapshots = await Promise.all([ this.#getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE),