Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Allow querying block number for tree indices #10332

Merged
merged 14 commits into from
Dec 2, 2024
Merged

feat: Allow querying block number for tree indices #10332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
1742ca9
WIP
PhilWindle Nov 29, 2024
bdb551d
WIP
PhilWindle Nov 29, 2024
e8965af
Merge remote-tracking branch 'origin/master' into pw/blocks-by-index
PhilWindle Nov 29, 2024
143e643
WIP
PhilWindle Nov 29, 2024
a40a5a7
WIP
PhilWindle Dec 1, 2024
13a5a0b
WIP
PhilWindle Dec 1, 2024
fc330dc
Merge branch 'master' into pw/blocks-by-index
PhilWindle Dec 1, 2024
b91cde5
Fixes
PhilWindle Dec 1, 2024
6e7f021
Formatting
PhilWindle Dec 1, 2024
54059fd
WIP
PhilWindle Dec 2, 2024
9376754
WIP
PhilWindle Dec 2, 2024
35a595f
WIP
PhilWindle Dec 2, 2024
6628ac8
Modified test
PhilWindle Dec 2, 2024
3ac651e
Test fix
PhilWindle Dec 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion barretenberg/cpp/scripts/merkle_tree_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ set -e
# run commands relative to parent directory
cd $(dirname $0)/..

DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.*
DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBTreeStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.*
TEST=${1:-$DEFAULT_TESTS}
PRESET=${PRESET:-clang16}

Expand Down
120 changes: 95 additions & 25 deletions ...c/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,17 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
using StoreType = Store;

// Asynchronous methods accept these callback function types as arguments
using AppendCompletionCallback = std::function<void(const TypedResponse<AddDataResponse>&)>;
using MetaDataCallback = std::function<void(const TypedResponse<TreeMetaResponse>&)>;
using HashPathCallback = std::function<void(const TypedResponse<GetSiblingPathResponse>&)>;
using FindLeafCallback = std::function<void(const TypedResponse<FindLeafIndexResponse>&)>;
using GetLeafCallback = std::function<void(const TypedResponse<GetLeafResponse>&)>;
using AppendCompletionCallback = std::function<void(TypedResponse<AddDataResponse>&)>;
using MetaDataCallback = std::function<void(TypedResponse<TreeMetaResponse>&)>;
using HashPathCallback = std::function<void(TypedResponse<GetSiblingPathResponse>&)>;
using FindLeafCallback = std::function<void(TypedResponse<FindLeafIndexResponse>&)>;
using GetLeafCallback = std::function<void(TypedResponse<GetLeafResponse>&)>;
using CommitCallback = std::function<void(TypedResponse<CommitResponse>&)>;
using RollbackCallback = std::function<void(const Response&)>;
using RollbackCallback = std::function<void(Response&)>;
using RemoveHistoricBlockCallback = std::function<void(TypedResponse<RemoveHistoricResponse>&)>;
using UnwindBlockCallback = std::function<void(TypedResponse<UnwindResponse>&)>;
using FinaliseBlockCallback = std::function<void(const Response&)>;
using FinaliseBlockCallback = std::function<void(Response&)>;
using GetBlockForIndexCallback = std::function<void(TypedResponse<BlockForIndexResponse>&)>;

// Only construct from provided store and thread pool, no copies or moves
ContentAddressedAppendOnlyTree(std::unique_ptr<Store> store,
Expand Down Expand Up @@ -90,7 +91,7 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
* @param includeUncommitted Whether to include uncommitted changes
*/
void get_sibling_path(const index_t& index,
const index_t& blockNumber,
const block_number_t& blockNumber,
const HashPathCallback& on_completion,
bool includeUncommitted) const;

Expand Down Expand Up @@ -131,7 +132,7 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
* @param includeUncommitted Whether to include uncommitted changes
* @param on_completion Callback to be called on completion
*/
void get_meta_data(const index_t& blockNumber,
void get_meta_data(const block_number_t& blockNumber,
bool includeUncommitted,
const MetaDataCallback& on_completion) const;

Expand All @@ -151,7 +152,7 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
* @param on_completion Callback to be called on completion
*/
void get_leaf(const index_t& index,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const GetLeafCallback& completion) const;

Expand All @@ -164,7 +165,7 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
* @brief Returns the index of the provided leaf in the tree
*/
void find_leaf_index(const fr& leaf,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const FindLeafCallback& on_completion) const;

Expand All @@ -181,10 +182,23 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
*/
void find_leaf_index_from(const fr& leaf,
const index_t& start_index,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const FindLeafCallback& on_completion) const;

/**
* @brief Returns the block numbers that correspond to the given indices values
*/
void find_block_numbers(const std::vector<index_t>& indices, const GetBlockForIndexCallback& on_completion) const;

/**
* @brief Returns the block numbers that correspond to the given indices values, from the perspective of a
* historical block number
*/
void find_block_numbers(const std::vector<index_t>& indices,
const block_number_t& blockNumber,
const GetBlockForIndexCallback& on_completion) const;

/**
* @brief Commit the tree to the backing store
*/
Expand All @@ -200,11 +214,11 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
*/
uint32_t depth() const { return depth_; }

void remove_historic_block(const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion);
void remove_historic_block(const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion);

void unwind_block(const index_t& blockNumber, const UnwindBlockCallback& on_completion);
void unwind_block(const block_number_t& blockNumber, const UnwindBlockCallback& on_completion);

void finalise_block(const index_t& blockNumber, const FinaliseBlockCallback& on_completion);
void finalise_block(const block_number_t& blockNumber, const FinaliseBlockCallback& on_completion);

protected:
using ReadTransaction = typename Store::ReadTransaction;
Expand Down Expand Up @@ -326,7 +340,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_meta_data(bool in
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_meta_data(const index_t& blockNumber,
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_meta_data(const block_number_t& blockNumber,
bool includeUncommitted,
const MetaDataCallback& on_completion) const
{
Expand Down Expand Up @@ -361,7 +375,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_sibling_path(cons

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_sibling_path(const index_t& index,
const index_t& blockNumber,
const block_number_t& blockNumber,
const HashPathCallback& on_completion,
bool includeUncommitted) const
{
Expand Down Expand Up @@ -393,6 +407,62 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_sibling_path(cons
workers_->enqueue(job);
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_block_numbers(
const std::vector<index_t>& indices, const GetBlockForIndexCallback& on_completion) const
{
auto job = [=, this]() {
execute_and_report<BlockForIndexResponse>(
[=, this](TypedResponse<BlockForIndexResponse>& response) {
response.inner.blockNumbers.reserve(indices.size());
TreeMeta meta;
ReadTransactionPtr tx = store_->create_read_transaction();
store_->get_meta(meta, *tx, true);
index_t maxIndex = meta.committedSize;
for (index_t index : indices) {
bool outOfRange = index >= maxIndex;
std::optional<block_number_t> block =
outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx);
response.inner.blockNumbers.emplace_back(block);
}
},
on_completion);
};
workers_->enqueue(job);
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_block_numbers(
const std::vector<index_t>& indices,
const block_number_t& blockNumber,
const GetBlockForIndexCallback& on_completion) const
{
auto job = [=, this]() {
execute_and_report<BlockForIndexResponse>(
[=, this](TypedResponse<BlockForIndexResponse>& response) {
response.inner.blockNumbers.reserve(indices.size());
TreeMeta meta;
BlockPayload blockPayload;
ReadTransactionPtr tx = store_->create_read_transaction();
store_->get_meta(meta, *tx, true);
if (!store_->get_block_data(blockNumber, blockPayload, *tx)) {
throw std::runtime_error(format("Unable to find block numbers for indices for block ",
blockNumber,
", failed to get block data."));
}
index_t maxIndex = std::min(meta.committedSize, blockPayload.size);
for (index_t index : indices) {
bool outOfRange = index >= maxIndex;
std::optional<block_number_t> block =
outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx);
response.inner.blockNumbers.emplace_back(block);
}
},
on_completion);
};
workers_->enqueue(job);
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_subtree_sibling_path(
uint32_t subtree_depth, const HashPathCallback& on_completion, bool includeUncommitted) const
Expand Down Expand Up @@ -473,7 +543,7 @@ std::optional<fr> ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_lea
NodePayload nodePayload;
bool success = store_->get_node_by_hash(hash, nodePayload, tx, requestContext.includeUncommitted);
if (!success) {
// std::cout << "No root" << std::endl;
// std::cout << "No root " << hash << std::endl;
return std::nullopt;
}
// std::cout << "Found root at depth " << i << " : " << hash << std::endl;
Expand Down Expand Up @@ -601,7 +671,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_leaf(const index_

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_leaf(const index_t& leaf_index,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const GetLeafCallback& on_completion) const
{
Expand Down Expand Up @@ -657,7 +727,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_leaf_index(const

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_leaf_index(const fr& leaf,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const FindLeafCallback& on_completion) const
{
Expand Down Expand Up @@ -696,7 +766,7 @@ template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_leaf_index_from(
const fr& leaf,
const index_t& start_index,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const FindLeafCallback& on_completion) const
{
Expand Down Expand Up @@ -788,7 +858,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::rollback(const Rollba

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::remove_historic_block(
const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion)
const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion)
{
auto job = [=, this]() {
execute_and_report<RemoveHistoricResponse>(
Expand All @@ -804,7 +874,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::remove_historic_block
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::unwind_block(const index_t& blockNumber,
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::unwind_block(const block_number_t& blockNumber,
const UnwindBlockCallback& on_completion)
{
auto job = [=, this]() {
Expand All @@ -821,7 +891,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::unwind_block(const in
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::finalise_block(const index_t& blockNumber,
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::finalise_block(const block_number_t& blockNumber,
const FinaliseBlockCallback& on_completion)
{
auto job = [=, this]() {
Expand Down Expand Up @@ -981,7 +1051,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::add_batch_internal(
new_root = new_hash;
meta.root = new_hash;
meta.size = new_size;
// std::cout << "New size: " << meta.size << std::endl;
// std::cout << "New size: " << meta.size << ", root " << meta.root << std::endl;
store_->put_meta(meta);
}

Expand Down
Loading