From f0265de85a5e514598e559daab29cfbe4bffbbf5 Mon Sep 17 00:00:00 2001 From: IlyasRidhuan Date: Thu, 7 Mar 2024 10:59:18 +0000 Subject: [PATCH 01/63] chore: move alpine containers to ubuntu --- barretenberg/cpp/dockerfiles/Dockerfile.bench | 2 +- .../dockerfiles/Dockerfile.x86_64-linux-clang | 37 +++++++++------ .../Dockerfile.x86_64-linux-clang-assert | 47 +++++++++++-------- .../Dockerfile.x86_64-linux-clang-benchmarks | 40 +++++++++------- .../Dockerfile.x86_64-linux-clang-fuzzing | 39 ++++++++------- .../dockerfiles/Dockerfile.x86_64-linux-gcc | 33 ++++++++----- 6 files changed, 119 insertions(+), 79 deletions(-) diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.bench b/barretenberg/cpp/dockerfiles/Dockerfile.bench index 42226685814..94d84f1338b 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.bench +++ b/barretenberg/cpp/dockerfiles/Dockerfile.bench @@ -1,4 +1,4 @@ FROM aztecprotocol/barretenberg-x86_64-linux-clang WORKDIR /usr/src/barretenberg/cpp -RUN apk update && apk add curl libstdc++ jq +RUN apt update && apt install curl libstdc++6 jq -y RUN ./scripts/ci/ultra_honk_bench.sh diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang index fe984e74e35..568f0fcd9e4 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang +++ b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang @@ -1,23 +1,32 @@ -FROM alpine:3.18 AS builder -RUN apk update \ - && apk upgrade \ - && apk add --no-cache \ - build-base \ - clang16 \ - cmake \ - ninja \ - git \ - curl \ - perl +FROM ubuntu:lunar as builder + +RUN apt update && apt install -y \ + build-essential \ + curl \ + git \ + cmake \ + lsb-release \ + wget \ + software-properties-common \ + gnupg \ + ninja-build \ + npm \ + libssl-dev \ + jq \ + bash \ + libstdc++6 + +RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 + WORKDIR /usr/src/barretenberg/cpp COPY . . # Build bb binary and targets needed for benchmarking. # Everything else is built as part linux-clang-assert. # Benchmark targets want to run without asserts, so get built alongside bb. -RUN cmake --preset default -RUN cmake --build --preset default --target ultra_honk_rounds_bench --target bb --target grumpkin_srs_gen +RUN cmake --preset clang16 +RUN cmake --build --preset clang16 --target ultra_honk_rounds_bench --target bb --target grumpkin_srs_gen -FROM alpine:3.18 +FROM ubuntu:lunar WORKDIR /usr/src/barretenberg/cpp COPY . . COPY --from=builder /usr/src/barretenberg/cpp/scripts/ci /usr/src/barretenberg/cpp/scripts/ci diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert index d487335b597..9276245c00c 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert +++ b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert @@ -1,24 +1,33 @@ -# We have to stay on 3.17 for now, to get clang-format 15, as code is not yet formatted to 16. -FROM alpine:3.17 AS builder -RUN apk update \ - && apk upgrade \ - && apk add --no-cache \ - build-base \ - clang15 \ - cmake \ - ninja \ - git \ - curl \ - perl \ - clang-extra-tools \ - bash +FROM ubuntu:lunar as builder + +RUN apt update && apt install -y \ + build-essential \ + curl \ + git \ + cmake \ + lsb-release \ + wget \ + software-properties-common \ + gnupg \ + ninja-build \ + npm \ + libssl-dev \ + jq \ + bash \ + libstdc++6 \ + clang-format + +RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 + WORKDIR /usr/src/barretenberg/cpp COPY . . # Build everything to ensure everything builds. All tests will be run from the result of this build. -RUN ./format.sh check && cmake --preset default -DCMAKE_BUILD_TYPE=RelWithAssert -DCI=ON && cmake --build --preset default +RUN ./format.sh check && cmake --preset clang16 -DCMAKE_BUILD_TYPE=RelWithAssert -DCI=ON && cmake --build --preset clang16 RUN srs_db/download_grumpkin.sh -FROM alpine:3.17 -RUN apk update && apk add curl libstdc++ -COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db -COPY --from=builder /usr/src/barretenberg/cpp/build/bin /usr/src/barretenberg/cpp/build/bin \ No newline at end of file +CMD ["/bin/bash"] + +# FROM ubuntu:lunar +# RUN apt update && apt install curl libstdc++6 -y +# COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db +# COPY --from=builder /usr/src/barretenberg/cpp/build/bin /usr/src/barretenberg/cpp/build/bin diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-benchmarks b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-benchmarks index 00d896c6342..b2bbebeb48a 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-benchmarks +++ b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-benchmarks @@ -1,23 +1,29 @@ -FROM alpine:3.18 AS builder -RUN apk update \ - && apk upgrade \ - && apk add --no-cache \ - build-base \ - clang16 \ - openmp-dev \ - cmake \ - ninja \ - git \ - curl \ - perl +FROM ubuntu:lunar as builder -WORKDIR /usr/src/barretenberg/cpp +RUN apt update && apt install -y \ + build-essential \ + curl \ + git \ + cmake \ + lsb-release \ + wget \ + software-properties-common \ + gnupg \ + ninja-build \ + npm \ + libssl-dev \ + jq \ + bash \ + libstdc++6 + +RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 +WORKDIR /usr/src/barretenberg/cpp COPY . . # Build everything to ensure everything builds. All tests will be run from the result of this build. -RUN cmake --preset default && cmake --build --preset default --target external_bench +RUN cmake --preset clang16 && cmake --build --preset clang16 --target external_bench -FROM alpine:3.18 -RUN apk update && apk add curl openmp +FROM ubuntu:lunar +RUN apt update && apt install curl libomp-dev -y COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db -COPY --from=builder /usr/src/barretenberg/cpp/build/bin/*_bench /usr/src/barretenberg/cpp/build/bin/ \ No newline at end of file +COPY --from=builder /usr/src/barretenberg/cpp/build/bin/*_bench /usr/src/barretenberg/cpp/build/bin/ diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-fuzzing b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-fuzzing index f9dd7717b23..cc66d7958a0 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-fuzzing +++ b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-fuzzing @@ -1,23 +1,28 @@ -FROM alpine:3.18 AS builder -RUN apk update && \ - apk upgrade && \ - apk add --no-cache \ - build-base \ - clang16 \ - compiler-rt \ - openmp-dev \ - cmake \ - ninja \ - git \ - curl \ - perl +FROM ubuntu:lunar as builder -WORKDIR /usr/src/barretenberg/cpp +RUN apt update && apt install -y \ + build-essential \ + curl \ + git \ + cmake \ + lsb-release \ + wget \ + software-properties-common \ + gnupg \ + ninja-build \ + npm \ + libssl-dev \ + jq \ + bash \ + libstdc++6 + +RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 +WORKDIR /usr/src/barretenberg/cpp COPY . . -# Build the entire project, as we want to check everything builds under clang +# Build the entire project, as we want to check everything builds under clang-fuzzing with clang-16. RUN cmake --preset fuzzing && cmake --build --preset fuzzing -FROM alpine:3.18 -RUN apk update && apk add openmp +FROM ubuntu:lunar +RUN apt update && apt install libomp-dev -y COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-gcc b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-gcc index 5bcdd5009be..3532d222f34 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-gcc +++ b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-gcc @@ -1,17 +1,28 @@ -FROM alpine:3.18 AS builder -RUN apk update \ - && apk upgrade \ - && apk add --no-cache \ - build-base \ - cmake \ - ninja \ - git \ - curl +FROM ubuntu:lunar as builder + +RUN apt update && apt install -y \ + build-essential \ + curl \ + git \ + cmake \ + lsb-release \ + wget \ + software-properties-common \ + gnupg \ + ninja-build \ + npm \ + libssl-dev \ + jq \ + bash \ + libstdc++6 + +RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 + WORKDIR /usr/src/barretenberg/cpp COPY . . # Build the entire project, as we want to check everything builds under gcc. RUN cmake --preset gcc -DCI=ON && cmake --build --preset gcc -FROM alpine:3.18 -RUN apk update && apk add libstdc++ +FROM ubuntu:lunar +RUN apt update && apt install libstdc++6 -y COPY --from=builder /usr/src/barretenberg/cpp/build-gcc/bin/bb /usr/src/barretenberg/cpp/build/bin/bb From 181e0b41eb1beeaa8708345db993a6b1a81884e9 Mon Sep 17 00:00:00 2001 From: IlyasRidhuan Date: Thu, 7 Mar 2024 15:22:52 +0000 Subject: [PATCH 02/63] chore: more dockerfiles from noir --- barretenberg/acir_tests/Dockerfile.bb | 4 +- barretenberg/acir_tests/Dockerfile.bb.sol | 15 +++-- .../Dockerfile.x86_64-linux-clang-assert | 10 ++-- barretenberg/sol/Dockerfile | 55 ++++++++++++------- barretenberg/ts/Dockerfile | 4 +- l1-contracts/Dockerfile | 13 ++++- noir/Dockerfile.native | 2 +- 7 files changed, 66 insertions(+), 37 deletions(-) diff --git a/barretenberg/acir_tests/Dockerfile.bb b/barretenberg/acir_tests/Dockerfile.bb index e0267b01345..f7123707612 100644 --- a/barretenberg/acir_tests/Dockerfile.bb +++ b/barretenberg/acir_tests/Dockerfile.bb @@ -1,8 +1,8 @@ FROM aztecprotocol/barretenberg-x86_64-linux-clang-assert FROM aztecprotocol/noir-compile-acir-tests as noir-acir-tests -FROM node:18.19.0-alpine -RUN apk update && apk add git bash curl jq coreutils +FROM node:18.19.0 +RUN apt update && apt install git bash curl jq coreutils -y COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build COPY --from=noir-acir-tests /usr/src/noir/noir-repo/test_programs /usr/src/noir/noir-repo/test_programs WORKDIR /usr/src/barretenberg/acir_tests diff --git a/barretenberg/acir_tests/Dockerfile.bb.sol b/barretenberg/acir_tests/Dockerfile.bb.sol index 98cb85c364a..0883035bb4b 100644 --- a/barretenberg/acir_tests/Dockerfile.bb.sol +++ b/barretenberg/acir_tests/Dockerfile.bb.sol @@ -2,14 +2,21 @@ FROM aztecprotocol/barretenberg-x86_64-linux-clang-assert FROM aztecprotocol/barretenberg-x86_64-linux-clang-sol FROM aztecprotocol/noir-compile-acir-tests as noir-acir-tests -FROM node:18.19.0-alpine -RUN apk update && apk add git bash curl jq +FROM node:18.19.0 +RUN apt update && apt install git bash curl jq -y COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build COPY --from=1 /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol COPY --from=noir-acir-tests /usr/src/noir/noir-repo/test_programs /usr/src/noir/noir-repo/test_programs -COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/anvil /usr/local/bin/anvil +# COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/anvil /usr/local/bin/anvil + +RUN curl -L https://foundry.paradigm.xyz | bash +ENV PATH="${PATH}:/root/.foundry/bin" +RUN foundryup + WORKDIR /usr/src/barretenberg/acir_tests COPY . . # Run every acir test through a solidity verifier. RUN (cd sol-test && yarn) -RUN PARALLEL=1 FLOW=sol ./run_acir_tests.sh +CMD ["/bin/bash"] +# RUN PARALLEL=1 FLOW=sol ./run_acir_tests.sh + diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert index 9276245c00c..a958ead8b83 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert +++ b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert @@ -25,9 +25,7 @@ COPY . . RUN ./format.sh check && cmake --preset clang16 -DCMAKE_BUILD_TYPE=RelWithAssert -DCI=ON && cmake --build --preset clang16 RUN srs_db/download_grumpkin.sh -CMD ["/bin/bash"] - -# FROM ubuntu:lunar -# RUN apt update && apt install curl libstdc++6 -y -# COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db -# COPY --from=builder /usr/src/barretenberg/cpp/build/bin /usr/src/barretenberg/cpp/build/bin +FROM ubuntu:lunar +RUN apt update && apt install curl libstdc++6 -y +COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db +COPY --from=builder /usr/src/barretenberg/cpp/build/bin /usr/src/barretenberg/cpp/build/bin diff --git a/barretenberg/sol/Dockerfile b/barretenberg/sol/Dockerfile index df99298fb85..b44f7547e44 100644 --- a/barretenberg/sol/Dockerfile +++ b/barretenberg/sol/Dockerfile @@ -1,15 +1,22 @@ -FROM alpine:3.18 -RUN apk update \ - && apk upgrade \ - && apk add --no-cache \ - build-base \ - clang16 \ - openmp-dev \ - cmake \ - ninja \ - git \ - curl \ - perl +FROM ubuntu:lunar as builder + +RUN apt update && apt install -y \ + build-essential \ + curl \ + git \ + cmake \ + lsb-release \ + wget \ + software-properties-common \ + gnupg \ + ninja-build \ + npm \ + libssl-dev \ + jq \ + bash \ + libstdc++6 + +RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 WORKDIR /usr/src/barretenberg/cpp @@ -17,20 +24,30 @@ COPY ./cpp . # Build everything to ensure everything builds. All tests will be run from the result of this build. RUN cmake --preset clang16 && cmake --build --preset clang16 --target solidity_key_gen solidity_proof_gen -FROM docker.io/frolvlad/alpine-glibc:alpine-3.17_glibc-2.34 as builder -RUN apk update && apk add git curl build-base openmp-dev bash - -COPY --from=0 /usr/src/barretenberg/cpp/build/bin /usr/src/barretenberg/cpp/build/bin -COPY --from=0 /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db +# FROM docker.io/frolvlad/alpine-glibc:alpine-3.17_glibc-2.34 as builder +# RUN apk update && apk add git curl build-base openmp-dev bash +FROM ubuntu:lunar +RUN apt update && apt install -y \ + build-essential \ + curl \ + git \ + bash \ + libomp-dev + +COPY --from=builder /usr/src/barretenberg/cpp/build/bin /usr/src/barretenberg/cpp/build/bin +COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db WORKDIR /usr/src/barretenberg/sol COPY ./sol . # Copy forge binary directly from foundry -COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/forge /usr/local/bin/forge +# COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/forge /usr/local/bin/forge +RUN curl -L https://foundry.paradigm.xyz | bash +ENV PATH="${PATH}:/root/.foundry/bin" +RUN foundryup RUN cd ../cpp/srs_db && ./download_ignition.sh 3 && cd ../../sol RUN ./scripts/init.sh # TestBase is excluded as it is just boilerplate -RUN forge test --no-match-contract TestBase \ No newline at end of file +RUN forge test --no-match-contract TestBase diff --git a/barretenberg/ts/Dockerfile b/barretenberg/ts/Dockerfile index 855dc17be37..af5c2511a4a 100644 --- a/barretenberg/ts/Dockerfile +++ b/barretenberg/ts/Dockerfile @@ -1,6 +1,6 @@ FROM aztecprotocol/barretenberg-wasm-linux-clang -FROM node:18.19.0-alpine +FROM node:18.19.0 COPY --from=0 /usr/src/barretenberg /usr/src/barretenberg # Create a standalone container that can run bb.js (and tests). @@ -17,4 +17,4 @@ RUN yarn formatting && SKIP_CPP_BUILD=1 yarn build CMD ["yarn", "test"] # We want to create a pure package, as would be published to npm, for consuming projects. -RUN yarn pack && tar zxf package.tgz && rm package.tgz && mv package ../ts \ No newline at end of file +RUN yarn pack && tar zxf package.tgz && rm package.tgz && mv package ../ts diff --git a/l1-contracts/Dockerfile b/l1-contracts/Dockerfile index 671da76b806..44ef16c676e 100644 --- a/l1-contracts/Dockerfile +++ b/l1-contracts/Dockerfile @@ -1,6 +1,13 @@ # Building requires foundry. -FROM ghcr.io/foundry-rs/foundry:nightly-4a643801d0b3855934cdec778e33e79c79971783 -RUN apk update && apk add git jq bash nodejs npm yarn python3 py3-pip && pip3 install slither-analyzer==0.10.0 slitherin==0.5.0 +# FROM ghcr.io/foundry-rs/foundry:nightly-4a643801d0b3855934cdec778e33e79c79971783 +FROM ubuntu:lunar +RUN apt update && apt install git jq bash nodejs npm yarn python3 py3-pip -y && pip3 install slither-analyzer==0.10.0 slitherin==0.5.0 + +RUN curl -L https://foundry.paradigm.xyz | bash +RUN source ~/.bashrc +ENV PATH="~/.foundry/bin:${PATH}" +RUN foundryup + WORKDIR /usr/src/l1-contracts COPY . . RUN git init @@ -10,4 +17,4 @@ RUN git add . && yarn slither && yarn slither-has-diff RUN forge build FROM scratch -COPY --from=0 /usr/src/l1-contracts/out /usr/src/l1-contracts/out \ No newline at end of file +COPY --from=0 /usr/src/l1-contracts/out /usr/src/l1-contracts/out diff --git a/noir/Dockerfile.native b/noir/Dockerfile.native index 73a29b3de21..0ba100fe726 100644 --- a/noir/Dockerfile.native +++ b/noir/Dockerfile.native @@ -7,7 +7,7 @@ COPY . . RUN ./scripts/bootstrap_native.sh # When running the container, mount the users home directory to same location. -FROM ubuntu:focal +FROM ubuntu:lunar # Install Tini as nargo doesn't handle signals properly. # Install git as nargo needs it to clone. RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean From 3a5712f0b467549c585658f63cca4c7b421d54ae Mon Sep 17 00:00:00 2001 From: IlyasRidhuan Date: Thu, 7 Mar 2024 15:27:12 +0000 Subject: [PATCH 03/63] fix: python pip ubuntu --- l1-contracts/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/l1-contracts/Dockerfile b/l1-contracts/Dockerfile index 44ef16c676e..0d0baf7e8e6 100644 --- a/l1-contracts/Dockerfile +++ b/l1-contracts/Dockerfile @@ -1,7 +1,7 @@ # Building requires foundry. # FROM ghcr.io/foundry-rs/foundry:nightly-4a643801d0b3855934cdec778e33e79c79971783 FROM ubuntu:lunar -RUN apt update && apt install git jq bash nodejs npm yarn python3 py3-pip -y && pip3 install slither-analyzer==0.10.0 slitherin==0.5.0 +RUN apt update && apt install git jq bash nodejs npm yarn python3 python3-pip -y && pip3 install slither-analyzer==0.10.0 slitherin==0.5.0 RUN curl -L https://foundry.paradigm.xyz | bash RUN source ~/.bashrc From aef55d2f1d8c1379f86e21e27a9d5f08aa07cec8 Mon Sep 17 00:00:00 2001 From: IlyasRidhuan Date: Thu, 7 Mar 2024 17:22:43 +0000 Subject: [PATCH 04/63] fix: wrangle python --- barretenberg/acir_tests/Dockerfile.bb.sol | 1 - l1-contracts/Dockerfile | 15 +++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/barretenberg/acir_tests/Dockerfile.bb.sol b/barretenberg/acir_tests/Dockerfile.bb.sol index 0883035bb4b..ddeb4fb774a 100644 --- a/barretenberg/acir_tests/Dockerfile.bb.sol +++ b/barretenberg/acir_tests/Dockerfile.bb.sol @@ -7,7 +7,6 @@ RUN apt update && apt install git bash curl jq -y COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build COPY --from=1 /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol COPY --from=noir-acir-tests /usr/src/noir/noir-repo/test_programs /usr/src/noir/noir-repo/test_programs -# COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/anvil /usr/local/bin/anvil RUN curl -L https://foundry.paradigm.xyz | bash ENV PATH="${PATH}:/root/.foundry/bin" diff --git a/l1-contracts/Dockerfile b/l1-contracts/Dockerfile index 0d0baf7e8e6..48064e89441 100644 --- a/l1-contracts/Dockerfile +++ b/l1-contracts/Dockerfile @@ -1,18 +1,25 @@ # Building requires foundry. -# FROM ghcr.io/foundry-rs/foundry:nightly-4a643801d0b3855934cdec778e33e79c79971783 FROM ubuntu:lunar -RUN apt update && apt install git jq bash nodejs npm yarn python3 python3-pip -y && pip3 install slither-analyzer==0.10.0 slitherin==0.5.0 +RUN apt update && apt install curl git jq bash nodejs npm python3.11-full python3-pip -y + +# Use virtualenv, do not try to use pipx, it's not working. +RUN python3 -m venv /root/.venv +RUN /root/.venv/bin/pip3 install slither-analyzer==0.10.0 slitherin==0.5.0 RUN curl -L https://foundry.paradigm.xyz | bash -RUN source ~/.bashrc -ENV PATH="~/.foundry/bin:${PATH}" + +# Set env variables for foundry and venv +ENV PATH="${PATH}:/root/.foundry/bin:/root/.venv/bin" RUN foundryup WORKDIR /usr/src/l1-contracts COPY . . RUN git init RUN forge clean && forge fmt --check && forge build && forge test + +RUN npm install --global yarn RUN yarn && yarn lint + RUN git add . && yarn slither && yarn slither-has-diff RUN forge build From acfccb487f7a5884a6268da9a9254a5ca90f6f21 Mon Sep 17 00:00:00 2001 From: IlyasRidhuan Date: Fri, 8 Mar 2024 10:26:56 +0000 Subject: [PATCH 05/63] chore: clean up some leftover comments --- barretenberg/acir_tests/Dockerfile.bb.sol | 3 +-- barretenberg/sol/Dockerfile | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/barretenberg/acir_tests/Dockerfile.bb.sol b/barretenberg/acir_tests/Dockerfile.bb.sol index ddeb4fb774a..9b391a524e9 100644 --- a/barretenberg/acir_tests/Dockerfile.bb.sol +++ b/barretenberg/acir_tests/Dockerfile.bb.sol @@ -16,6 +16,5 @@ WORKDIR /usr/src/barretenberg/acir_tests COPY . . # Run every acir test through a solidity verifier. RUN (cd sol-test && yarn) -CMD ["/bin/bash"] -# RUN PARALLEL=1 FLOW=sol ./run_acir_tests.sh +RUN PARALLEL=1 FLOW=sol ./run_acir_tests.sh diff --git a/barretenberg/sol/Dockerfile b/barretenberg/sol/Dockerfile index b44f7547e44..fc57077ff89 100644 --- a/barretenberg/sol/Dockerfile +++ b/barretenberg/sol/Dockerfile @@ -24,8 +24,6 @@ COPY ./cpp . # Build everything to ensure everything builds. All tests will be run from the result of this build. RUN cmake --preset clang16 && cmake --build --preset clang16 --target solidity_key_gen solidity_proof_gen -# FROM docker.io/frolvlad/alpine-glibc:alpine-3.17_glibc-2.34 as builder -# RUN apk update && apk add git curl build-base openmp-dev bash FROM ubuntu:lunar RUN apt update && apt install -y \ build-essential \ @@ -39,8 +37,7 @@ COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/s WORKDIR /usr/src/barretenberg/sol COPY ./sol . -# Copy forge binary directly from foundry -# COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/forge /usr/local/bin/forge +# Download and install foundry RUN curl -L https://foundry.paradigm.xyz | bash ENV PATH="${PATH}:/root/.foundry/bin" RUN foundryup From f0e671bd992f78074b9e509f413e8115d021b65d Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 8 Mar 2024 17:53:17 +0000 Subject: [PATCH 06/63] WIP --- .../crypto/merkle_tree/store/lmdb_store.hpp | 141 ++++++++++++++++++ .../merkle_tree/store/lmdb_store.test.cpp | 33 ++++ 2 files changed, 174 insertions(+) create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.test.cpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.hpp new file mode 100644 index 00000000000..c924c2db027 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.hpp @@ -0,0 +1,141 @@ +#pragma once +#include +#include +#include + +namespace bb::crypto::merkle_tree { + +class LMDBEnvironment { + public: + LMDBEnvironment(const std::string& directory) + { + if (mdb_env_create(&_mdbEnv) != 0) { + // throw here + } + mdb_env_set_mapsize(_mdbEnv, (size_t)1048576 * (size_t)100000); // 1MB * 100000 + mdb_env_set_maxdbs(_mdbEnv, 1); + if (mdb_env_open(_mdbEnv, directory.c_str(), 0, S_IRWXU | S_IRWXG | S_IRWXO) != 0) { + mdb_env_close(_mdbEnv); + // throw here + } + } + + ~LMDBEnvironment() { mdb_env_close(_mdbEnv); } + + MDB_env* underlying() const { return _mdbEnv; } + + private: + MDB_env* _mdbEnv; +}; + +class LMDBTransaction { + public: + LMDBTransaction(const LMDBEnvironment& env, bool readOnly = true) + : _readOnly(readOnly) + { + if (mdb_txn_begin(env.underlying(), nullptr, _readOnly ? MDB_RDONLY : 0, &_transaction) != 0) { + // throw + } + } + + ~LMDBTransaction() + { + if (_readOnly) { + mdb_txn_abort(_transaction); + } else { + mdb_txn_commit(_transaction); + } + } + + MDB_txn* underlying() const { return _transaction; } + + private: + MDB_txn* _transaction; + bool _readOnly; +}; + +class LMDBDatabase { + public: + LMDBDatabase(const LMDBEnvironment& env, const LMDBTransaction& transaction, const std::string& name) + : _environment(env) + { + if (mdb_dbi_open(transaction.underlying(), name.c_str(), MDB_INTEGERKEY | MDB_CREATE, &_dbi) != 0) { + // throw here + } + } + + ~LMDBDatabase() { mdb_dbi_close(_environment.underlying(), _dbi); } + + MDB_dbi& underlying() { return _dbi; } + + private: + MDB_dbi _dbi; + const LMDBEnvironment& _environment; +}; + +class LMDBStore { + + public: + LMDBStore(const std::string& directory, const std::string& name) + : _environment(directory) + , _name(name) + {} + ~LMDBStore() {} + + void put(size_t level, size_t index, const std::vector& data) + { + const LMDBTransaction transaction(_environment, false); + LMDBDatabase database(_environment, transaction, _name); + size_t key = (1 << level) + index - 1; + + MDB_val dbKey; + dbKey.mv_size = sizeof(key); + dbKey.mv_data = &key; + + MDB_val dbVal; + dbVal.mv_size = data.size(); + dbVal.mv_data = static_cast(&data[0]); + int error = mdb_put(transaction.underlying(), database.underlying(), &dbKey, &dbVal, 0); + + if (error == 0) { + return; + } + std::cout << "ERROR When writing to DB" << std::endl; + } + bool get(size_t level, size_t index, std::vector& data) const + { + const LMDBTransaction transaction(_environment, true); + LMDBDatabase database(_environment, transaction, _name); + size_t key = (1 << level) + index - 1; + + MDB_val dbKey; + dbKey.mv_size = sizeof(key); + dbKey.mv_data = &key; + + MDB_val dbVal; + int error = mdb_get(transaction.underlying(), database.underlying(), &dbKey, &dbVal); + + if (error == 0) { + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + return; + } + + switch (error) { + case MDB_NOTFOUND: + std::cout << "Key " << key << " not found" << std::endl; + break; + case EINVAL: + std::cout << "INVALID PARAMETER" << std::endl; + break; + default: + std::cout << "Received error " << error << std::endl; + break; + } + } + + private: + LMDBEnvironment _environment; + const std::string _name; +}; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.test.cpp new file mode 100644 index 00000000000..1fd58431f91 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.test.cpp @@ -0,0 +1,33 @@ +#include + +#include "barretenberg/common/streams.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/numeric/random/engine.hpp" + +#include "lmdb_store.hpp" + +using namespace bb::stdlib; +using namespace bb::crypto::merkle_tree; + +using Builder = UltraCircuitBuilder; + +using field_ct = field_t; +using witness_ct = witness_t; +namespace { +auto& engine = numeric::get_debug_randomness(); +auto& random_engine = numeric::get_randomness(); +} // namespace + +static std::vector VALUES = []() { + std::vector values(1024); + for (size_t i = 0; i < 1024; ++i) { + values[i] = i; + } + return values; +}(); + +TEST(lmdb_store, test_open) +{ + + LMDBStore store("tmp/lmdb"); +} \ No newline at end of file From 08fec83399cae6767e07dd3eb8c3d8b649136cee Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 11 Mar 2024 14:36:31 +0000 Subject: [PATCH 07/63] WIP --- barretenberg/cpp/src/CMakeLists.txt | 1 + .../crypto/merkle_tree/store/lmdb_store.hpp | 141 ------------------ .../merkle_tree/store/lmdb_store.test.cpp | 33 ---- .../src/barretenberg/db_cli/CMakeLists.txt | 7 + .../barretenberg/db_cli/store/lmdb_store.cpp | 112 ++++++++++++++ .../barretenberg/db_cli/store/lmdb_store.hpp | 78 ++++++++++ .../db_cli/store/lmdb_store.test.cpp | 47 ++++++ 7 files changed, 245 insertions(+), 174 deletions(-) delete mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.test.cpp create mode 100644 barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp create mode 100644 barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp create mode 100644 barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 94cef6e7947..ca6e8b6a6d2 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -80,6 +80,7 @@ add_subdirectory(barretenberg/translator_vm) add_subdirectory(barretenberg/ultra_honk) add_subdirectory(barretenberg/vm) add_subdirectory(barretenberg/wasi) +add_subdirectory(barretenberg/db_cli) if(SMT) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.hpp deleted file mode 100644 index c924c2db027..00000000000 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.hpp +++ /dev/null @@ -1,141 +0,0 @@ -#pragma once -#include -#include -#include - -namespace bb::crypto::merkle_tree { - -class LMDBEnvironment { - public: - LMDBEnvironment(const std::string& directory) - { - if (mdb_env_create(&_mdbEnv) != 0) { - // throw here - } - mdb_env_set_mapsize(_mdbEnv, (size_t)1048576 * (size_t)100000); // 1MB * 100000 - mdb_env_set_maxdbs(_mdbEnv, 1); - if (mdb_env_open(_mdbEnv, directory.c_str(), 0, S_IRWXU | S_IRWXG | S_IRWXO) != 0) { - mdb_env_close(_mdbEnv); - // throw here - } - } - - ~LMDBEnvironment() { mdb_env_close(_mdbEnv); } - - MDB_env* underlying() const { return _mdbEnv; } - - private: - MDB_env* _mdbEnv; -}; - -class LMDBTransaction { - public: - LMDBTransaction(const LMDBEnvironment& env, bool readOnly = true) - : _readOnly(readOnly) - { - if (mdb_txn_begin(env.underlying(), nullptr, _readOnly ? MDB_RDONLY : 0, &_transaction) != 0) { - // throw - } - } - - ~LMDBTransaction() - { - if (_readOnly) { - mdb_txn_abort(_transaction); - } else { - mdb_txn_commit(_transaction); - } - } - - MDB_txn* underlying() const { return _transaction; } - - private: - MDB_txn* _transaction; - bool _readOnly; -}; - -class LMDBDatabase { - public: - LMDBDatabase(const LMDBEnvironment& env, const LMDBTransaction& transaction, const std::string& name) - : _environment(env) - { - if (mdb_dbi_open(transaction.underlying(), name.c_str(), MDB_INTEGERKEY | MDB_CREATE, &_dbi) != 0) { - // throw here - } - } - - ~LMDBDatabase() { mdb_dbi_close(_environment.underlying(), _dbi); } - - MDB_dbi& underlying() { return _dbi; } - - private: - MDB_dbi _dbi; - const LMDBEnvironment& _environment; -}; - -class LMDBStore { - - public: - LMDBStore(const std::string& directory, const std::string& name) - : _environment(directory) - , _name(name) - {} - ~LMDBStore() {} - - void put(size_t level, size_t index, const std::vector& data) - { - const LMDBTransaction transaction(_environment, false); - LMDBDatabase database(_environment, transaction, _name); - size_t key = (1 << level) + index - 1; - - MDB_val dbKey; - dbKey.mv_size = sizeof(key); - dbKey.mv_data = &key; - - MDB_val dbVal; - dbVal.mv_size = data.size(); - dbVal.mv_data = static_cast(&data[0]); - int error = mdb_put(transaction.underlying(), database.underlying(), &dbKey, &dbVal, 0); - - if (error == 0) { - return; - } - std::cout << "ERROR When writing to DB" << std::endl; - } - bool get(size_t level, size_t index, std::vector& data) const - { - const LMDBTransaction transaction(_environment, true); - LMDBDatabase database(_environment, transaction, _name); - size_t key = (1 << level) + index - 1; - - MDB_val dbKey; - dbKey.mv_size = sizeof(key); - dbKey.mv_data = &key; - - MDB_val dbVal; - int error = mdb_get(transaction.underlying(), database.underlying(), &dbKey, &dbVal); - - if (error == 0) { - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); - return; - } - - switch (error) { - case MDB_NOTFOUND: - std::cout << "Key " << key << " not found" << std::endl; - break; - case EINVAL: - std::cout << "INVALID PARAMETER" << std::endl; - break; - default: - std::cout << "Received error " << error << std::endl; - break; - } - } - - private: - LMDBEnvironment _environment; - const std::string _name; -}; -} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.test.cpp deleted file mode 100644 index 1fd58431f91..00000000000 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/store/lmdb_store.test.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include - -#include "barretenberg/common/streams.hpp" -#include "barretenberg/common/test.hpp" -#include "barretenberg/numeric/random/engine.hpp" - -#include "lmdb_store.hpp" - -using namespace bb::stdlib; -using namespace bb::crypto::merkle_tree; - -using Builder = UltraCircuitBuilder; - -using field_ct = field_t; -using witness_ct = witness_t; -namespace { -auto& engine = numeric::get_debug_randomness(); -auto& random_engine = numeric::get_randomness(); -} // namespace - -static std::vector VALUES = []() { - std::vector values(1024); - for (size_t i = 0; i < 1024; ++i) { - values[i] = i; - } - return values; -}(); - -TEST(lmdb_store, test_open) -{ - - LMDBStore store("tmp/lmdb"); -} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt new file mode 100644 index 00000000000..3e2f30e230d --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt @@ -0,0 +1,7 @@ +barretenberg_module(db_cli) +target_link_libraries( + db_cli + PRIVATE + lmdb + env +) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp new file mode 100644 index 00000000000..9fa1d093ccc --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp @@ -0,0 +1,112 @@ +#include "lmdb_store.hpp" + +namespace bb::crypto::merkle_tree { + +LMDBEnvironment::LMDBEnvironment(const std::string& directory) +{ + if (!call_lmdb_func(mdb_env_create, &_mdbEnv)) { + // throw here + } + call_lmdb_func(mdb_env_set_mapsize, _mdbEnv, (size_t)1048576); + call_lmdb_func(mdb_env_set_maxdbs, _mdbEnv, (MDB_dbi)5); + if (!call_lmdb_func(mdb_env_open, _mdbEnv, directory.c_str(), 0U, mdb_mode_t(S_IRWXU | S_IRWXG | S_IRWXO))) { + call_lmdb_func(mdb_env_close, _mdbEnv); + // throw here + } +} + +LMDBTransaction::LMDBTransaction(const LMDBEnvironment& env, bool readOnly = true) + : _readOnly(readOnly) + , _committed(false) +{ + unsigned int flags = _readOnly ? MDB_RDONLY : 0; + MDB_txn* p = nullptr; + if (!call_lmdb_func(mdb_txn_begin, env.underlying(), p, flags, &_transaction)) { + // throw here + } +} + +LMDBTransaction::~LMDBTransaction() +{ + if (_readOnly) { + call_lmdb_func(mdb_txn_abort, _transaction); + } else if (!_committed) { + call_lmdb_func(mdb_txn_commit, _transaction); + } +} + +MDB_txn* LMDBTransaction::underlying() const +{ + return _transaction; +} + +bool LMDBTransaction::commit() +{ + if (_committed) { + return true; + } + _committed = call_lmdb_func(mdb_txn_commit, _transaction); + return _committed; +} + +LMDBDatabase::LMDBDatabase(const LMDBEnvironment& env, const LMDBTransaction& transaction, const std::string& name) + : _environment(env) +{ + unsigned int flags = MDB_CREATE; + if (!call_lmdb_func(mdb_dbi_open, transaction.underlying(), name.c_str(), flags, &_dbi)) { + // throw here + } +} + +LMDBDatabase::~LMDBDatabase() +{ + call_lmdb_func(mdb_dbi_close, _environment.underlying(), _dbi); +} + +const MDB_dbi& LMDBDatabase::underlying() const +{ + return _dbi; +} + +LMDBStore::LMDBStore(LMDBEnvironment& environment, const std::string& name) + : _environment(environment) + , _name(name) + , _database(_environment, LMDBTransaction(_environment, false), _name) +{} +LMDBSTORE::~LMDBStore() {} + +void LMDBStore::put(size_t level, size_t index, std::vector& data) +{ + LMDBTransaction transaction(_environment, false); + size_t key = (1 << level) + index - 1; + std::cout << "Key " << key << std::endl; + + MDB_val dbKey; + dbKey.mv_size = sizeof(key); + dbKey.mv_data = &key; + + MDB_val dbVal; + dbVal.mv_size = data.size(); + dbVal.mv_data = static_cast(&data[0]); + call_lmdb_func(mdb_put, transaction.underlying(), _database.underlying(), &dbKey, &dbVal, 0U); +} + +bool LMDBStore::get(size_t level, size_t index, std::vector& data) const +{ + LMDBTransaction transaction(_environment, false); + size_t key = (1 << level) + index - 1; + std::cout << "Key " << key << std::endl; + + MDB_val dbKey; + dbKey.mv_size = sizeof(key); + dbKey.mv_data = &key; + + MDB_val dbVal; + if (!call_lmdb_func(mdb_get, transaction.underlying(), _database.underlying(), &dbKey, &dbVal)) { + return false; + } + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + return true; +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp new file mode 100644 index 00000000000..c5948ce5a16 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp @@ -0,0 +1,78 @@ +#pragma once +#include +#include +#include +#include + +namespace bb::crypto::merkle_tree { + +template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) +{ + int error = f(args...); + if (error != 0) { + std::cout << "ERROR: " << mdb_strerror(error) << std::endl; + } + return error == 0; +} + +template void call_lmdb_func(void (*f)(TArgs...), TArgs... args) +{ + f(args...); +} + +class LMDBEnvironment { + public: + LMDBEnvironment(const std::string& directory); + + ~LMDBEnvironment(); + + MDB_env* underlying() const; + + private: + MDB_env* _mdbEnv; +}; + +class LMDBTransaction { + public: + LMDBTransaction(const LMDBEnvironment& env, bool readOnly = true); + + ~LMDBTransaction(); + + MDB_txn* underlying() const; + + bool commit(); + + private: + MDB_txn* _transaction; + bool _readOnly; + bool _committed; +}; + +class LMDBDatabase { + public: + LMDBDatabase(const LMDBEnvironment& env, const LMDBTransaction& transaction, const std::string& name); + + ~LMDBDatabase(); + + const MDB_dbi& underlying() const; + + private: + MDB_dbi _dbi; + const LMDBEnvironment& _environment; +}; + +class LMDBStore { + + public: + LMDBStore(LMDBEnvironment& environment, const std::string& name); + ~LMDBStore(); + + void put(size_t level, size_t index, std::vector& data); + bool get(size_t level, size_t index, std::vector& data) const; + + private: + LMDBEnvironment& _environment; + const std::string _name; + LMDBDatabase _database; +}; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp new file mode 100644 index 00000000000..8b58d054ae7 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp @@ -0,0 +1,47 @@ +#include + +#include + +#include "barretenberg/common/streams.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include "barretenberg/stdlib/primitives/field/field.hpp" +#include "lmdb_store.hpp" + +using namespace bb::stdlib; +using namespace bb::crypto::merkle_tree; + +using Builder = bb::UltraCircuitBuilder; + +using field_ct = field_t; +using witness_ct = witness_t; +namespace { +auto& engine = bb::numeric::get_debug_randomness(); +auto& random_engine = bb::numeric::get_randomness(); +} // namespace + +static std::vector VALUES = []() { + std::vector values(1024); + for (size_t i = 0; i < 1024; ++i) { + values[i] = i; + } + return values; +}(); + +TEST(lmdb_store, test_open) +{ + std::filesystem::create_directories("/tmp/lmdb"); + std::cout << "Hello!" << std::endl; + LMDBEnvironment environment("/tmp/lmdb"); + LMDBStore store(environment, "note hash tree"); + std::cout << "HERE" << std::endl; + std::vector buf; + write(buf, VALUES[0]); + store.put(0, 0, buf); + + std::vector buf2; + bool success = store.get(0, 0, buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[0]); +} \ No newline at end of file From c9d381925520cb3d88e9bc040911183d108e9982 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 11 Mar 2024 21:35:59 +0000 Subject: [PATCH 08/63] WIP --- .../src/barretenberg/db_cli/CMakeLists.txt | 7 ++ .../barretenberg/db_cli/store/lmdb_store.cpp | 70 +++++++---- .../barretenberg/db_cli/store/lmdb_store.hpp | 18 ++- .../db_cli/store/lmdb_store.test.cpp | 115 ++++++++++++++++-- 4 files changed, 173 insertions(+), 37 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt index 3e2f30e230d..be45fdf9e0f 100644 --- a/barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt @@ -4,4 +4,11 @@ target_link_libraries( PRIVATE lmdb env +) + +target_link_libraries( + db_cli + PRIVATE + barretenberg + env ) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp index 9fa1d093ccc..d0fa210960e 100644 --- a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp +++ b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp @@ -1,6 +1,6 @@ #include "lmdb_store.hpp" -namespace bb::crypto::merkle_tree { +namespace bb::db_cli { LMDBEnvironment::LMDBEnvironment(const std::string& directory) { @@ -15,9 +15,20 @@ LMDBEnvironment::LMDBEnvironment(const std::string& directory) } } -LMDBTransaction::LMDBTransaction(const LMDBEnvironment& env, bool readOnly = true) +LMDBEnvironment::~LMDBEnvironment() +{ + call_lmdb_func(mdb_env_close, _mdbEnv); +} + +MDB_env* LMDBEnvironment::underlying() const +{ + return _mdbEnv; +} + +LMDBTransaction::LMDBTransaction(const LMDBEnvironment& env, const LMDBDatabase& database, bool readOnly) : _readOnly(readOnly) , _committed(false) + , _database(database) { unsigned int flags = _readOnly ? MDB_RDONLY : 0; MDB_txn* p = nullptr; @@ -49,10 +60,34 @@ bool LMDBTransaction::commit() return _committed; } +void LMDBTransaction::put(size_t level, size_t index, std::vector& data) +{ + put(level, index, data, data.size()); +} + +void LMDBTransaction::put(size_t level, size_t start_index, std::vector& data, size_t element_size) +{ + MDB_cursor* cursor; + call_lmdb_func(mdb_cursor_open, underlying(), _database, &cursor); + size_t key = (1 << level) + start_index - 1; + size_t num_elements = data.size() / element_size; + + for (size_t i = 0; i < num_elements; ++i, ++key) { + MDB_val dbKey; + dbKey.mv_size = sizeof(key); + dbKey.mv_data = &key; + + MDB_val dbVal; + dbVal.mv_size = element_size; + dbVal.mv_data = static_cast(&data[i * element_size]); + call_lmdb_func(mdb_cursor_put, cursor, &dbKey, &dbVal, 0U); + } +} + LMDBDatabase::LMDBDatabase(const LMDBEnvironment& env, const LMDBTransaction& transaction, const std::string& name) : _environment(env) { - unsigned int flags = MDB_CREATE; + unsigned int flags = MDB_CREATE | MDB_INTEGERKEY; if (!call_lmdb_func(mdb_dbi_open, transaction.underlying(), name.c_str(), flags, &_dbi)) { // throw here } @@ -73,29 +108,12 @@ LMDBStore::LMDBStore(LMDBEnvironment& environment, const std::string& name) , _name(name) , _database(_environment, LMDBTransaction(_environment, false), _name) {} -LMDBSTORE::~LMDBStore() {} - -void LMDBStore::put(size_t level, size_t index, std::vector& data) -{ - LMDBTransaction transaction(_environment, false); - size_t key = (1 << level) + index - 1; - std::cout << "Key " << key << std::endl; - - MDB_val dbKey; - dbKey.mv_size = sizeof(key); - dbKey.mv_data = &key; - - MDB_val dbVal; - dbVal.mv_size = data.size(); - dbVal.mv_data = static_cast(&data[0]); - call_lmdb_func(mdb_put, transaction.underlying(), _database.underlying(), &dbKey, &dbVal, 0U); -} +LMDBStore::~LMDBStore() {} bool LMDBStore::get(size_t level, size_t index, std::vector& data) const { - LMDBTransaction transaction(_environment, false); + LMDBTransaction transaction(_environment, true); size_t key = (1 << level) + index - 1; - std::cout << "Key " << key << std::endl; MDB_val dbKey; dbKey.mv_size = sizeof(key); @@ -109,4 +127,10 @@ bool LMDBStore::get(size_t level, size_t index, std::vector& data) cons std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); return true; } -} // namespace bb::crypto::merkle_tree \ No newline at end of file + +std::unique_ptr LMDBStore::createWriteTransaction() +{ + return new LMDBTransaction(_environment, false); +} + +} // namespace bb::db_cli \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp index c5948ce5a16..419a4e4bcfc 100644 --- a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp +++ b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp @@ -1,10 +1,13 @@ #pragma once #include #include +#include #include +#include #include +#include -namespace bb::crypto::merkle_tree { +namespace bb::db_cli { template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) { @@ -34,18 +37,21 @@ class LMDBEnvironment { class LMDBTransaction { public: - LMDBTransaction(const LMDBEnvironment& env, bool readOnly = true); + LMDBTransaction(const LMDBEnvironment& env, const LMDBDatabase& database, bool readOnly = true); ~LMDBTransaction(); MDB_txn* underlying() const; + void put(size_t level, size_t index, std::vector& data); + void put(size_t level, size_t start_index, std::vector& data, size_t element_size); bool commit(); private: - MDB_txn* _transaction; bool _readOnly; bool _committed; + const LMDBDatabase& _database; + MDB_txn* _transaction; }; class LMDBDatabase { @@ -67,12 +73,14 @@ class LMDBStore { LMDBStore(LMDBEnvironment& environment, const std::string& name); ~LMDBStore(); - void put(size_t level, size_t index, std::vector& data); bool get(size_t level, size_t index, std::vector& data) const; + std::unique_ptr createWriteTransaction(); + private: LMDBEnvironment& _environment; const std::string _name; LMDBDatabase _database; + std::unique_ptr _writeTransaction; }; -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::db_cli \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp index 8b58d054ae7..8df98988e11 100644 --- a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp @@ -1,5 +1,6 @@ #include +#include #include #include "barretenberg/common/streams.hpp" @@ -9,7 +10,7 @@ #include "lmdb_store.hpp" using namespace bb::stdlib; -using namespace bb::crypto::merkle_tree; +using namespace bb::db_cli; using Builder = bb::UltraCircuitBuilder; @@ -20,21 +21,49 @@ auto& engine = bb::numeric::get_debug_randomness(); auto& random_engine = bb::numeric::get_randomness(); } // namespace +const int SAMPLE_DATA_SIZE = 1024; + static std::vector VALUES = []() { - std::vector values(1024); - for (size_t i = 0; i < 1024; ++i) { - values[i] = i; + std::vector values(SAMPLE_DATA_SIZE); + for (size_t i = 0; i < SAMPLE_DATA_SIZE; ++i) { + values[i] = bb::fr::random_element(&engine); } return values; }(); -TEST(lmdb_store, test_open) +const std::string directory = "/tmp/lmdb"; + +class LMDBStoreTest : public testing::Test { + protected: + static void SetUpTestSuite() { std::filesystem::create_directories(directory); } + + // static void TearDownTestSuite() { + // } +}; + +TEST(lmdb_store, can_create_store) +{ + LMDBEnvironment environment(directory); + LMDBStore store(environment, "note hash tree"); + std::vector buf; + write(buf, VALUES[0]); + + store.createWriteTransaction(); + store.put(0, 0, buf); + store.commitWriteTransaction() + + std::vector + buf2; + bool success = store.get(0, 0, buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[0]); +} + +TEST(lmdb_store, can_write_to_and_read_from_store) { - std::filesystem::create_directories("/tmp/lmdb"); - std::cout << "Hello!" << std::endl; - LMDBEnvironment environment("/tmp/lmdb"); + LMDBEnvironment environment(directory); LMDBStore store(environment, "note hash tree"); - std::cout << "HERE" << std::endl; std::vector buf; write(buf, VALUES[0]); store.put(0, 0, buf); @@ -44,4 +73,72 @@ TEST(lmdb_store, test_open) EXPECT_EQ(success, true); bb::fr value = from_buffer(buf2, 0); EXPECT_EQ(value, VALUES[0]); +} + +TEST(lmdb_store, can_write_and_read_multiple) +{ + LMDBEnvironment environment(directory); + LMDBStore store(environment, "note hash tree"); + + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf; + write(buf, VALUES[i]); + store.put(10, i, buf); + } + + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf2; + bool success = store.get(10, i, buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[i]); + } +} + +TEST(lmdb_store, can_write_batch_and_read_back) +{ + LMDBEnvironment environment(directory); + LMDBStore store(environment, "note hash tree"); + + std::vector buf; + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector temp; + write(temp, VALUES[i]); + int old_size = int(buf.size()); + buf.resize(buf.size() + temp.size()); + copy(temp.begin(), temp.end(), buf.begin() + old_size); + } + store.put(10, 0, buf, 32); + + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf2; + bool success = store.get(10, i, buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[i]); + } +} + +TEST(lmdb_store, can_write_and_read_at_random_keys) +{ + LMDBEnvironment environment(directory); + LMDBStore store(environment, "note hash tree"); + + std::vector keys; + + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf; + write(buf, VALUES[i]); + size_t key = size_t(rand() % 10000000); + keys.push_back(key); + store.put(0, key, buf); + } + + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf2; + bool success = store.get(0, keys[i], buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[i]); + } } \ No newline at end of file From 8b0d3841bb60237b377315f0f6e18b8622b96c47 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 12 Mar 2024 15:47:55 +0000 Subject: [PATCH 09/63] WIP --- .../barretenberg/db_cli/store/lmdb_store.cpp | 105 ++----- .../barretenberg/db_cli/store/lmdb_store.hpp | 140 ++++++++-- .../db_cli/store/lmdb_store.test.cpp | 260 +++++++++++++----- 3 files changed, 340 insertions(+), 165 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp index d0fa210960e..ac83e71a481 100644 --- a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp +++ b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp @@ -2,73 +2,46 @@ namespace bb::db_cli { -LMDBEnvironment::LMDBEnvironment(const std::string& directory) +LMDBEnvironment::LMDBEnvironment(const std::string& directory, + unsigned long mapSizeMB, + unsigned long maxNumDBs, + unsigned int maxNumReaders) + : _maxReaders(maxNumReaders) + , _numReaders(0) { if (!call_lmdb_func(mdb_env_create, &_mdbEnv)) { // throw here } - call_lmdb_func(mdb_env_set_mapsize, _mdbEnv, (size_t)1048576); - call_lmdb_func(mdb_env_set_maxdbs, _mdbEnv, (MDB_dbi)5); + size_t totalMapSize = 1024 * 1024 * mapSizeMB; + call_lmdb_func(mdb_env_set_mapsize, _mdbEnv, totalMapSize); + call_lmdb_func(mdb_env_set_maxdbs, _mdbEnv, (MDB_dbi)maxNumDBs); + call_lmdb_func(mdb_env_set_maxreaders, _mdbEnv, maxNumReaders); if (!call_lmdb_func(mdb_env_open, _mdbEnv, directory.c_str(), 0U, mdb_mode_t(S_IRWXU | S_IRWXG | S_IRWXO))) { call_lmdb_func(mdb_env_close, _mdbEnv); // throw here } } -LMDBEnvironment::~LMDBEnvironment() +void LMDBEnvironment::waitForReader() { - call_lmdb_func(mdb_env_close, _mdbEnv); -} - -MDB_env* LMDBEnvironment::underlying() const -{ - return _mdbEnv; -} - -LMDBTransaction::LMDBTransaction(const LMDBEnvironment& env, const LMDBDatabase& database, bool readOnly) - : _readOnly(readOnly) - , _committed(false) - , _database(database) -{ - unsigned int flags = _readOnly ? MDB_RDONLY : 0; - MDB_txn* p = nullptr; - if (!call_lmdb_func(mdb_txn_begin, env.underlying(), p, flags, &_transaction)) { - // throw here - } -} - -LMDBTransaction::~LMDBTransaction() -{ - if (_readOnly) { - call_lmdb_func(mdb_txn_abort, _transaction); - } else if (!_committed) { - call_lmdb_func(mdb_txn_commit, _transaction); - } -} - -MDB_txn* LMDBTransaction::underlying() const -{ - return _transaction; -} - -bool LMDBTransaction::commit() -{ - if (_committed) { - return true; + std::unique_lock lock(_readersLock); + if (_numReaders >= _maxReaders) { + _readersCondition.wait(lock, [&] { return _numReaders < _maxReaders; }); } - _committed = call_lmdb_func(mdb_txn_commit, _transaction); - return _committed; + std::cout << " Try Num Readers: " << ++_numReaders << std::endl; } -void LMDBTransaction::put(size_t level, size_t index, std::vector& data) +void LMDBEnvironment::releaseReader() { - put(level, index, data, data.size()); + std::unique_lock lock(_readersLock); + std::cout << " Release Num Readers: " << --_numReaders << std::endl; + _readersCondition.notify_one(); } -void LMDBTransaction::put(size_t level, size_t start_index, std::vector& data, size_t element_size) +void LMDBWriteTransaction::put(size_t level, size_t start_index, std::vector& data, size_t element_size) { MDB_cursor* cursor; - call_lmdb_func(mdb_cursor_open, underlying(), _database, &cursor); + call_lmdb_func(mdb_cursor_open, underlying(), _database.underlying(), &cursor); size_t key = (1 << level) + start_index - 1; size_t num_elements = data.size() / element_size; @@ -84,35 +57,8 @@ void LMDBTransaction::put(size_t level, size_t start_index, std::vector } } -LMDBDatabase::LMDBDatabase(const LMDBEnvironment& env, const LMDBTransaction& transaction, const std::string& name) - : _environment(env) -{ - unsigned int flags = MDB_CREATE | MDB_INTEGERKEY; - if (!call_lmdb_func(mdb_dbi_open, transaction.underlying(), name.c_str(), flags, &_dbi)) { - // throw here - } -} - -LMDBDatabase::~LMDBDatabase() -{ - call_lmdb_func(mdb_dbi_close, _environment.underlying(), _dbi); -} - -const MDB_dbi& LMDBDatabase::underlying() const -{ - return _dbi; -} - -LMDBStore::LMDBStore(LMDBEnvironment& environment, const std::string& name) - : _environment(environment) - , _name(name) - , _database(_environment, LMDBTransaction(_environment, false), _name) -{} -LMDBStore::~LMDBStore() {} - -bool LMDBStore::get(size_t level, size_t index, std::vector& data) const +bool LMDBReadTransaction::get(size_t level, size_t index, std::vector& data) const { - LMDBTransaction transaction(_environment, true); size_t key = (1 << level) + index - 1; MDB_val dbKey; @@ -120,7 +66,7 @@ bool LMDBStore::get(size_t level, size_t index, std::vector& data) cons dbKey.mv_data = &key; MDB_val dbVal; - if (!call_lmdb_func(mdb_get, transaction.underlying(), _database.underlying(), &dbKey, &dbVal)) { + if (!call_lmdb_func(mdb_get, underlying(), _database.underlying(), &dbKey, &dbVal)) { return false; } data.resize(dbVal.mv_size); @@ -128,9 +74,10 @@ bool LMDBStore::get(size_t level, size_t index, std::vector& data) cons return true; } -std::unique_ptr LMDBStore::createWriteTransaction() +LMDBReadTransaction::Ptr LMDBStore::createReadTransaction() { - return new LMDBTransaction(_environment, false); + _environment.waitForReader(); + return std::make_unique(_environment, _database); } } // namespace bb::db_cli \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp index 419a4e4bcfc..08a961e5d00 100644 --- a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp +++ b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -12,7 +13,7 @@ namespace bb::db_cli { template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) { int error = f(args...); - if (error != 0) { + if (error != 0 && error != MDB_NOTFOUND) { std::cout << "ERROR: " << mdb_strerror(error) << std::endl; } return error == 0; @@ -25,42 +26,134 @@ template void call_lmdb_func(void (*f)(TArgs...), TArgs... a class LMDBEnvironment { public: - LMDBEnvironment(const std::string& directory); + LMDBEnvironment(const std::string& directory, + unsigned long mapSizeMB, + unsigned long maxNumDBs, + unsigned int maxNumReaders); - ~LMDBEnvironment(); + ~LMDBEnvironment() { call_lmdb_func(mdb_env_close, _mdbEnv); } - MDB_env* underlying() const; + MDB_env* underlying() const { return _mdbEnv; } + + void waitForReader(); + + void releaseReader(); private: MDB_env* _mdbEnv; + unsigned int _maxReaders; + unsigned int _numReaders; + std::mutex _readersLock; + std::condition_variable _readersCondition; }; +class LMDBDatabase; + class LMDBTransaction { public: - LMDBTransaction(const LMDBEnvironment& env, const LMDBDatabase& database, bool readOnly = true); + LMDBTransaction(const LMDBEnvironment& env, bool readOnly = false) + : _environment(env) + { + MDB_txn* p = nullptr; + if (readOnly) { + std::cout << "Starting Read" << std::endl; + } + if (!call_lmdb_func(mdb_txn_begin, _environment.underlying(), p, readOnly ? MDB_RDONLY : 0U, &_transaction)) { + // throw here + } + } + + virtual ~LMDBTransaction() {} + + MDB_txn* underlying() const { return _transaction; } + + protected: + LMDBEnvironment& _environment; + MDB_txn* _transaction; +}; + +class LMDBReadTransaction : public LMDBTransaction { + public: + typedef std::unique_ptr Ptr; + + LMDBReadTransaction(LMDBEnvironment& env, const LMDBDatabase& database) + : LMDBTransaction(env, true) + , _environment(env) + , _database(database) + {} + + virtual ~LMDBReadTransaction() { abort(); } + + bool get(size_t level, size_t index, std::vector& data) const; - ~LMDBTransaction(); + private: + const LMDBDatabase& _database; - MDB_txn* underlying() const; + void abort() + { + std::cout << "Aborting Read" << std::endl; + call_lmdb_func(mdb_txn_abort, _transaction); + _environment.releaseReader(); + } +}; - void put(size_t level, size_t index, std::vector& data); +class LMDBWriteTransaction : public LMDBTransaction { + public: + typedef std::unique_ptr Ptr; + + LMDBWriteTransaction(const LMDBEnvironment& env, const LMDBDatabase& database) + : LMDBTransaction(env) + , _committed(false) + , _database(database) + {} + + virtual ~LMDBWriteTransaction() { commit(); } + + void put(size_t level, size_t index, std::vector& data) { put(level, index, data, data.size()); } void put(size_t level, size_t start_index, std::vector& data, size_t element_size); - bool commit(); + + bool commit() + { + if (_committed) { + return true; + } + _committed = call_lmdb_func(mdb_txn_commit, _transaction); + return _committed; + } private: - bool _readOnly; bool _committed; const LMDBDatabase& _database; - MDB_txn* _transaction; +}; + +class LMDBDatabaseCreationTransaction : public LMDBTransaction { + public: + typedef std::unique_ptr Ptr; + + LMDBDatabaseCreationTransaction(const LMDBEnvironment& env) + : LMDBTransaction(env) + {} + + virtual ~LMDBDatabaseCreationTransaction() { commit(); } + + private: + bool commit() { return call_lmdb_func(mdb_txn_commit, _transaction); } }; class LMDBDatabase { public: - LMDBDatabase(const LMDBEnvironment& env, const LMDBTransaction& transaction, const std::string& name); + LMDBDatabase(const LMDBEnvironment& env, const LMDBTransaction& transaction, const std::string& name) + : _environment(env) + { + unsigned int flags = MDB_CREATE | MDB_INTEGERKEY; + if (!call_lmdb_func(mdb_dbi_open, transaction.underlying(), name.c_str(), flags, &_dbi)) { + // throw here + } + } - ~LMDBDatabase(); + ~LMDBDatabase() { call_lmdb_func(mdb_dbi_close, _environment.underlying(), _dbi); } - const MDB_dbi& underlying() const; + const MDB_dbi& underlying() const { return _dbi; } private: MDB_dbi _dbi; @@ -70,17 +163,22 @@ class LMDBDatabase { class LMDBStore { public: - LMDBStore(LMDBEnvironment& environment, const std::string& name); - ~LMDBStore(); - - bool get(size_t level, size_t index, std::vector& data) const; - - std::unique_ptr createWriteTransaction(); + LMDBStore(LMDBEnvironment& environment, const std::string& name) + : _environment(environment) + , _name(name) + , _database(_environment, LMDBDatabaseCreationTransaction(_environment), _name) + {} + ~LMDBStore() {} + + LMDBWriteTransaction::Ptr createWriteTransaction() const + { + return std::make_unique(_environment, _database); + } + LMDBReadTransaction::Ptr createReadTransaction(); private: LMDBEnvironment& _environment; const std::string _name; LMDBDatabase _database; - std::unique_ptr _writeTransaction; }; } // namespace bb::db_cli \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp index 8df98988e11..9d1cbad6aba 100644 --- a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp @@ -1,5 +1,6 @@ #include +#include #include #include @@ -31,74 +32,100 @@ static std::vector VALUES = []() { return values; }(); -const std::string directory = "/tmp/lmdb"; - class LMDBStoreTest : public testing::Test { protected: - static void SetUpTestSuite() { std::filesystem::create_directories(directory); } + static void SetUpTestSuite() + { + const int random_directory_value = rand(); + std::stringstream ss; + ss << "/tmp/lmdb" << random_directory_value; + _directory = ss.str(); + std::filesystem::create_directories(_directory); + } + + static void TearDownTestSuite() { std::filesystem::remove_all(_directory); } + + void SetUp() + { + // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers + _environment.reset(new LMDBEnvironment(_directory, 1, 1, 2)); + } + + static std::string _directory; - // static void TearDownTestSuite() { - // } + std::unique_ptr _environment; }; -TEST(lmdb_store, can_create_store) -{ - LMDBEnvironment environment(directory); - LMDBStore store(environment, "note hash tree"); - std::vector buf; - write(buf, VALUES[0]); - - store.createWriteTransaction(); - store.put(0, 0, buf); - store.commitWriteTransaction() - - std::vector - buf2; - bool success = store.get(0, 0, buf2); - EXPECT_EQ(success, true); - bb::fr value = from_buffer(buf2, 0); - EXPECT_EQ(value, VALUES[0]); -} +std::string LMDBStoreTest::_directory = ""; -TEST(lmdb_store, can_write_to_and_read_from_store) +TEST_F(LMDBStoreTest, can_write_to_and_read_from_store) { - LMDBEnvironment environment(directory); - LMDBStore store(environment, "note hash tree"); - std::vector buf; - write(buf, VALUES[0]); - store.put(0, 0, buf); - - std::vector buf2; - bool success = store.get(0, 0, buf2); - EXPECT_EQ(success, true); - bb::fr value = from_buffer(buf2, 0); - EXPECT_EQ(value, VALUES[0]); + LMDBStore store(*_environment, "note hash tree"); + + { + std::vector buf; + write(buf, VALUES[0]); + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + transaction->put(0, 0, buf); + } + + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector buf2; + bool success = transaction->get(0, 0, buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[0]); + } } -TEST(lmdb_store, can_write_and_read_multiple) +TEST_F(LMDBStoreTest, reading_an_empty_key_reports_correctly) { - LMDBEnvironment environment(directory); - LMDBStore store(environment, "note hash tree"); + LMDBStore store(*_environment, "note hash tree"); - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + { std::vector buf; - write(buf, VALUES[i]); - store.put(10, i, buf); + write(buf, VALUES[0]); + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + transaction->put(0, 0, buf); } - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); std::vector buf2; - bool success = store.get(10, i, buf2); - EXPECT_EQ(success, true); - bb::fr value = from_buffer(buf2, 0); - EXPECT_EQ(value, VALUES[i]); + bool success = transaction->get(0, 1, buf2); + EXPECT_EQ(success, false); + } +} + +TEST_F(LMDBStoreTest, can_write_and_read_multiple) +{ + LMDBStore store(*_environment, "note hash tree"); + + { + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf; + write(buf, VALUES[i]); + transaction->put(10, i, buf); + } + } + + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf2; + bool success = transaction->get(10, i, buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[i]); + } } } -TEST(lmdb_store, can_write_batch_and_read_back) +TEST_F(LMDBStoreTest, can_write_batch_and_read_back) { - LMDBEnvironment environment(directory); - LMDBStore store(environment, "note hash tree"); + LMDBStore store(*_environment, "note hash tree"); std::vector buf; for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { @@ -108,37 +135,140 @@ TEST(lmdb_store, can_write_batch_and_read_back) buf.resize(buf.size() + temp.size()); copy(temp.begin(), temp.end(), buf.begin() + old_size); } - store.put(10, 0, buf, 32); + + { + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + transaction->put(10, 0, buf, 32); + } for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { std::vector buf2; - bool success = store.get(10, i, buf2); + bool success = store.createReadTransaction()->get(10, i, buf2); EXPECT_EQ(success, true); bb::fr value = from_buffer(buf2, 0); EXPECT_EQ(value, VALUES[i]); } } -TEST(lmdb_store, can_write_and_read_at_random_keys) +TEST_F(LMDBStoreTest, can_write_and_read_at_random_keys) { - LMDBEnvironment environment(directory); - LMDBStore store(environment, "note hash tree"); + LMDBStore store(*_environment, "note hash tree"); std::vector keys; - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { - std::vector buf; - write(buf, VALUES[i]); - size_t key = size_t(rand() % 10000000); - keys.push_back(key); - store.put(0, key, buf); + { + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf; + write(buf, VALUES[i]); + size_t key = size_t(rand() % 10000000); + keys.push_back(key); + transaction->put(0, key, buf); + } } - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { - std::vector buf2; - bool success = store.get(0, keys[i], buf2); + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf2; + bool success = transaction->get(0, keys[i], buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[i]); + } + } +} + +TEST_F(LMDBStoreTest, can_recreate_the_store_and_use_again) +{ + std::vector keys; + { + LMDBStore store(*_environment, "note hash tree"); + + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf; + write(buf, VALUES[i]); + size_t key = size_t(rand() % 10000000); + keys.push_back(key); + transaction->put(0, key, buf); + } + } + + { + LMDBStore store(*_environment, "note hash tree"); + + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf2; + bool success = transaction->get(0, keys[i], buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[i]); + } + } +} + +void read_loop(LMDBStore& store, size_t key, std::atomic& flag, bb::fr starting_value) +{ + bool seen = false; + while (true) { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector buf; + bool success = transaction->get(0, key, buf); EXPECT_EQ(success, true); - bb::fr value = from_buffer(buf2, 0); - EXPECT_EQ(value, VALUES[i]); + bb::fr value = from_buffer(buf, 0); + if (value == starting_value && !seen) { + // acknowledge that we have seen the old value + flag--; + seen = true; + } + if (value == starting_value + bb::fr(1)) { + // exit now that we have seen the new value + break; + } + } +} + +TEST_F(LMDBStoreTest, can_read_from_multiple_threads) +{ + std::cout << "Multiple Threads" << std::endl; + LMDBStore store(*_environment, "note hash tree"); + const int num_threads = 5; + + size_t key = size_t(rand() % 1000000); + + { + // we write VALUES[0] to a slot + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + std::vector buf; + write(buf, VALUES[0]); + transaction->put(0, key, buf); + } + + { + // we setup multiple threads to read the slot and check they shutdown when the value changes + std::vector threads; + std::atomic flag = num_threads; + for (size_t i = 0; i < num_threads; i++) { + threads.push_back(std::thread(read_loop, std::ref(store), key, std::ref(flag), VALUES[0])); + } + // wait until all threads have seen the old value + while (flag != 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + { + // we write VALUES[0] + 1 to the slot + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + std::vector buf; + write(buf, VALUES[0] + 1); + transaction->put(0, key, buf); + } + // now wait for all threads to exit having seen the new value + for (size_t i = 0; i < 5; i++) { + threads[i].join(); + } } } \ No newline at end of file From 9e826410c681d73e16684364706aad4aa721d78e Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 12 Jun 2024 12:20:53 +0100 Subject: [PATCH 10/63] WIP --- barretenberg/cpp/src/CMakeLists.txt | 5 +- .../src/barretenberg/db_cli/CMakeLists.txt | 14 -- .../barretenberg/lmdb_store/CMakeLists.txt | 2 + .../store => lmdb_store}/lmdb_store.cpp | 14 +- .../store => lmdb_store}/lmdb_store.hpp | 38 +++-- .../store => lmdb_store}/lmdb_store.test.cpp | 15 +- .../src/barretenberg/messaging/CMakeLists.txt | 2 + .../cpp/src/barretenberg/messaging/header.hpp | 37 +++++ .../barretenberg/messaging/stream_parser.cpp | 62 ++++++++ .../barretenberg/messaging/stream_parser.hpp | 23 +++ .../messaging/stream_parser.test.cpp | 147 ++++++++++++++++++ .../barretenberg/world_state/CMakeLists.txt | 1 + .../service/world_state_service.cpp | 23 +++ .../service/world_state_service.hpp | 14 ++ .../world_state_service/CMakeLists.txt | 7 + .../barretenberg/world_state_service/main.cpp | 20 +++ .../world_state_service/std_io.hpp | 43 +++++ 17 files changed, 418 insertions(+), 49 deletions(-) delete mode 100644 barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt rename barretenberg/cpp/src/barretenberg/{db_cli/store => lmdb_store}/lmdb_store.cpp (86%) rename barretenberg/cpp/src/barretenberg/{db_cli/store => lmdb_store}/lmdb_store.hpp (81%) rename barretenberg/cpp/src/barretenberg/{db_cli/store => lmdb_store}/lmdb_store.test.cpp (95%) create mode 100644 barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/messaging/header.hpp create mode 100644 barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp create mode 100644 barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp create mode 100644 barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.cpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/world_state_service/main.cpp create mode 100644 barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index bce0dce3575..dc824372a29 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -79,7 +79,10 @@ add_subdirectory(barretenberg/translator_vm) add_subdirectory(barretenberg/ultra_honk) add_subdirectory(barretenberg/vm) add_subdirectory(barretenberg/wasi) -add_subdirectory(barretenberg/db_cli) +add_subdirectory(barretenberg/world_state) +add_subdirectory(barretenberg/world_state_service) +add_subdirectory(barretenberg/messaging) +add_subdirectory(barretenberg/lmdb_store) if(SMT) diff --git a/barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt deleted file mode 100644 index be45fdf9e0f..00000000000 --- a/barretenberg/cpp/src/barretenberg/db_cli/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -barretenberg_module(db_cli) -target_link_libraries( - db_cli - PRIVATE - lmdb - env -) - -target_link_libraries( - db_cli - PRIVATE - barretenberg - env -) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt new file mode 100644 index 00000000000..e6d4d0d9c1d --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt @@ -0,0 +1,2 @@ +barretenberg_module(lmdb_store lmdb barretenberg) + diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp similarity index 86% rename from barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp rename to barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp index ac83e71a481..2b8eed13880 100644 --- a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.cpp +++ b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp @@ -1,6 +1,6 @@ #include "lmdb_store.hpp" -namespace bb::db_cli { +namespace bb::lmdb { LMDBEnvironment::LMDBEnvironment(const std::string& directory, unsigned long mapSizeMB, @@ -14,9 +14,11 @@ LMDBEnvironment::LMDBEnvironment(const std::string& directory, } size_t totalMapSize = 1024 * 1024 * mapSizeMB; call_lmdb_func(mdb_env_set_mapsize, _mdbEnv, totalMapSize); - call_lmdb_func(mdb_env_set_maxdbs, _mdbEnv, (MDB_dbi)maxNumDBs); + call_lmdb_func(mdb_env_set_maxdbs, _mdbEnv, static_cast(maxNumDBs)); call_lmdb_func(mdb_env_set_maxreaders, _mdbEnv, maxNumReaders); - if (!call_lmdb_func(mdb_env_open, _mdbEnv, directory.c_str(), 0U, mdb_mode_t(S_IRWXU | S_IRWXG | S_IRWXO))) { + unsigned int flags = MDB_NOTLS; + if (!call_lmdb_func( + mdb_env_open, _mdbEnv, directory.c_str(), flags, static_cast(S_IRWXU | S_IRWXG | S_IRWXO))) { call_lmdb_func(mdb_env_close, _mdbEnv); // throw here } @@ -28,13 +30,13 @@ void LMDBEnvironment::waitForReader() if (_numReaders >= _maxReaders) { _readersCondition.wait(lock, [&] { return _numReaders < _maxReaders; }); } - std::cout << " Try Num Readers: " << ++_numReaders << std::endl; + ++_numReaders; } void LMDBEnvironment::releaseReader() { std::unique_lock lock(_readersLock); - std::cout << " Release Num Readers: " << --_numReaders << std::endl; + --_numReaders; _readersCondition.notify_one(); } @@ -80,4 +82,4 @@ LMDBReadTransaction::Ptr LMDBStore::createReadTransaction() return std::make_unique(_environment, _database); } -} // namespace bb::db_cli \ No newline at end of file +} // namespace bb::lmdb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp similarity index 81% rename from barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp rename to barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp index 08a961e5d00..00b59b9391c 100644 --- a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.hpp +++ b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp @@ -5,16 +5,20 @@ #include #include #include +#include #include +#include #include -namespace bb::db_cli { +namespace bb::lmdb { template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) { int error = f(args...); if (error != 0 && error != MDB_NOTFOUND) { - std::cout << "ERROR: " << mdb_strerror(error) << std::endl; + std::stringstream ss; + ss << "ERROR: " << mdb_strerror(error) << std::endl; + std::cout << ss.str(); } return error == 0; } @@ -51,19 +55,16 @@ class LMDBDatabase; class LMDBTransaction { public: - LMDBTransaction(const LMDBEnvironment& env, bool readOnly = false) + LMDBTransaction(LMDBEnvironment& env, bool readOnly = false) : _environment(env) { MDB_txn* p = nullptr; - if (readOnly) { - std::cout << "Starting Read" << std::endl; - } if (!call_lmdb_func(mdb_txn_begin, _environment.underlying(), p, readOnly ? MDB_RDONLY : 0U, &_transaction)) { // throw here } } - virtual ~LMDBTransaction() {} + virtual ~LMDBTransaction() = default; MDB_txn* underlying() const { return _transaction; } @@ -74,15 +75,14 @@ class LMDBTransaction { class LMDBReadTransaction : public LMDBTransaction { public: - typedef std::unique_ptr Ptr; + using Ptr = std::unique_ptr; LMDBReadTransaction(LMDBEnvironment& env, const LMDBDatabase& database) : LMDBTransaction(env, true) - , _environment(env) , _database(database) {} - virtual ~LMDBReadTransaction() { abort(); } + ~LMDBReadTransaction() override { abort(); } bool get(size_t level, size_t index, std::vector& data) const; @@ -91,7 +91,6 @@ class LMDBReadTransaction : public LMDBTransaction { void abort() { - std::cout << "Aborting Read" << std::endl; call_lmdb_func(mdb_txn_abort, _transaction); _environment.releaseReader(); } @@ -99,11 +98,10 @@ class LMDBReadTransaction : public LMDBTransaction { class LMDBWriteTransaction : public LMDBTransaction { public: - typedef std::unique_ptr Ptr; + using Ptr = std::unique_ptr; - LMDBWriteTransaction(const LMDBEnvironment& env, const LMDBDatabase& database) + LMDBWriteTransaction(LMDBEnvironment& env, const LMDBDatabase& database) : LMDBTransaction(env) - , _committed(false) , _database(database) {} @@ -122,15 +120,15 @@ class LMDBWriteTransaction : public LMDBTransaction { } private: - bool _committed; + bool _committed{}; const LMDBDatabase& _database; }; class LMDBDatabaseCreationTransaction : public LMDBTransaction { public: - typedef std::unique_ptr Ptr; + using Ptr = std::unique_ptr; - LMDBDatabaseCreationTransaction(const LMDBEnvironment& env) + LMDBDatabaseCreationTransaction(LMDBEnvironment& env) : LMDBTransaction(env) {} @@ -163,9 +161,9 @@ class LMDBDatabase { class LMDBStore { public: - LMDBStore(LMDBEnvironment& environment, const std::string& name) + LMDBStore(LMDBEnvironment& environment, std::string name) : _environment(environment) - , _name(name) + , _name(std::move(name)) , _database(_environment, LMDBDatabaseCreationTransaction(_environment), _name) {} ~LMDBStore() {} @@ -181,4 +179,4 @@ class LMDBStore { const std::string _name; LMDBDatabase _database; }; -} // namespace bb::db_cli \ No newline at end of file +} // namespace bb::lmdb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp rename to barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp index 9d1cbad6aba..70d2bb3d048 100644 --- a/barretenberg/cpp/src/barretenberg/db_cli/store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp @@ -11,7 +11,7 @@ #include "lmdb_store.hpp" using namespace bb::stdlib; -using namespace bb::db_cli; +using namespace bb::lmdb; using Builder = bb::UltraCircuitBuilder; @@ -56,7 +56,7 @@ class LMDBStoreTest : public testing::Test { std::unique_ptr _environment; }; -std::string LMDBStoreTest::_directory = ""; +std::string LMDBStoreTest::_directory; TEST_F(LMDBStoreTest, can_write_to_and_read_from_store) { @@ -131,7 +131,7 @@ TEST_F(LMDBStoreTest, can_write_batch_and_read_back) for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { std::vector temp; write(temp, VALUES[i]); - int old_size = int(buf.size()); + int old_size = static_cast(buf.size()); buf.resize(buf.size() + temp.size()); copy(temp.begin(), temp.end(), buf.begin() + old_size); } @@ -162,7 +162,7 @@ TEST_F(LMDBStoreTest, can_write_and_read_at_random_keys) for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { std::vector buf; write(buf, VALUES[i]); - size_t key = size_t(rand() % 10000000); + size_t key = static_cast(rand() % 10000000); keys.push_back(key); transaction->put(0, key, buf); } @@ -191,7 +191,7 @@ TEST_F(LMDBStoreTest, can_recreate_the_store_and_use_again) for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { std::vector buf; write(buf, VALUES[i]); - size_t key = size_t(rand() % 10000000); + size_t key = static_cast(rand() % 10000000); keys.push_back(key); transaction->put(0, key, buf); } @@ -234,11 +234,10 @@ void read_loop(LMDBStore& store, size_t key, std::atomic& flag, bb::fr s TEST_F(LMDBStoreTest, can_read_from_multiple_threads) { - std::cout << "Multiple Threads" << std::endl; LMDBStore store(*_environment, "note hash tree"); const int num_threads = 5; - size_t key = size_t(rand() % 1000000); + size_t key = static_cast(rand() % 1000000); { // we write VALUES[0] to a slot @@ -253,7 +252,7 @@ TEST_F(LMDBStoreTest, can_read_from_multiple_threads) std::vector threads; std::atomic flag = num_threads; for (size_t i = 0; i < num_threads; i++) { - threads.push_back(std::thread(read_loop, std::ref(store), key, std::ref(flag), VALUES[0])); + threads.emplace_back(read_loop, std::ref(store), key, std::ref(flag), VALUES[0]); } // wait until all threads have seen the old value while (flag != 0) { diff --git a/barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt new file mode 100644 index 00000000000..d4f98426dc7 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt @@ -0,0 +1,2 @@ +barretenberg_module(messaging barretenberg) + diff --git a/barretenberg/cpp/src/barretenberg/messaging/header.hpp b/barretenberg/cpp/src/barretenberg/messaging/header.hpp new file mode 100644 index 00000000000..d2b9586ea35 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/messaging/header.hpp @@ -0,0 +1,37 @@ +#pragma once +#include +#include + +const uint32_t MAGIC_STRING_LENGTH = 8; +const uint8_t MAGIC_STRING[] = "AZTEC!!"; + +enum SystemMsgTypes { TERMINATE = 0, PING = 1, PONG = 2 }; + +const uint32_t FIRST_APP_MSG_TYPE = 100; + +#pragma pack(push, 1) +struct MsgHeader { + uint8_t msgStart[MAGIC_STRING_LENGTH]; // MAGIC_STRING + uint32_t msgLength; // Length including the header + uint16_t msgType; // The type of message + uint32_t msgId; // Unique Id for the message + uint32_t requestId; // Id of the message this is responding too (may not be used) + + MsgHeader(uint16_t type, uint32_t reqId) + : msgLength(sizeof(MsgHeader)) + , msgType(type) + , requestId(reqId) + { + std::memcpy(msgStart, MAGIC_STRING, MAGIC_STRING_LENGTH); + } + + MsgHeader(uint16_t type, uint32_t reqId, uint32_t msgId, uint32_t totalLength) + : MsgHeader(type, reqId) + { + this->msgId = msgId; + this->msgLength = totalLength; + } +}; +#pragma pack(pop) + +const uint32_t HEADER_SIZE = sizeof(MsgHeader); diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp new file mode 100644 index 00000000000..155c920c617 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp @@ -0,0 +1,62 @@ +#include "stream_parser.hpp" +#include "barretenberg/messaging/header.hpp" +#include +#include + +bool StreamParser::onNewData(char* data, uint32_t length) +{ + uint32_t newSize = bufferLength + length; + if (newSize > buffer.size()) { + buffer.resize(buffer.size() + length); + } + std::memcpy(&buffer[bufferLength], data, length); + bufferLength += length; + + while (true) { + uint32_t dataRemaining = bufferLength - readPointer; + if (dataRemaining < MAGIC_STRING_LENGTH) { + // not even enough to query the magic string + shrinkBuffer(); + break; + } + if (std::memcmp(&buffer[readPointer], MAGIC_STRING, MAGIC_STRING_LENGTH) != 0) { + ++readPointer; + continue; + } + // We have found the magic string, now see if there is a full header + if (dataRemaining < HEADER_SIZE) { + // Less than header bytes available, wait for more data + shrinkBuffer(); + break; + } + // We have a full header at least + const char* messageStart = &buffer[readPointer]; + const MsgHeader* header = reinterpret_cast(messageStart); + // Now see if we have a full message + if (header->msgLength > dataRemaining) { + // We don't, wait for more data + shrinkBuffer(); + break; + } + + // We have a full message! + readPointer += header->msgLength; + bool continueStream = messageHandler(header, messageStart + HEADER_SIZE); + // If we are told the stream has ended, propagate that information up + if (!continueStream) { + return false; + } + } + return true; +} + +void StreamParser::shrinkBuffer() +{ + uint32_t dataRemaining = bufferLength - readPointer; + if (dataRemaining > 0 && readPointer > 0) { + // use memove as ranges could overlap + std::memmove(&buffer[0], &buffer[readPointer], dataRemaining); + } + readPointer = 0; + bufferLength = dataRemaining; +} diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp new file mode 100644 index 00000000000..f106254f28b --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include "barretenberg/messaging/header.hpp" +#include +#include +#include +#include + +class StreamParser { + private: + std::vector buffer; + uint32_t bufferLength = 0; + uint32_t readPointer = 0; + std::function messageHandler; + + void shrinkBuffer(); + + public: + StreamParser(std::function handler) + : messageHandler(std::move(handler)) + {} + bool onNewData(char* data, uint32_t length); +}; \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp new file mode 100644 index 00000000000..7f587516418 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp @@ -0,0 +1,147 @@ +#include "stream_parser.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/messaging/header.hpp" +#include "gtest/gtest.h" +#include +#include +#include +#include + +std::vector createMessage(uint16_t type, uint32_t messageId, uint32_t requestId, uint32_t dataSize) +{ + MsgHeader header(type, requestId, messageId, HEADER_SIZE + dataSize); + std::vector message(header.msgLength); + std::memcpy(&message[0], &header, HEADER_SIZE); + return message; +} + +void assertHandledMessage(std::vector& inputMessage, const MsgHeader* parsedHeader, const char* parsedData) +{ + EXPECT_EQ(0, std::memcmp(&inputMessage[0], parsedHeader, HEADER_SIZE)); + EXPECT_EQ(0, std::memcmp(&inputMessage[HEADER_SIZE], parsedData, inputMessage.size() - HEADER_SIZE)); +} + +TEST(messaging_stream_parser, correctly_parses_basic_message) +{ + const MsgHeader* parsedHeader = nullptr; + const char* receivedData = nullptr; + int numCalls = 0; + + std::function handler = [&](const MsgHeader* header, const char* data) { + parsedHeader = header; + receivedData = data; + numCalls++; + return true; + }; + StreamParser parser(handler); + std::vector message = createMessage(5, 57, 56, 14); + parser.onNewData(&message[0], static_cast(message.size())); + EXPECT_EQ(numCalls, 1); + assertHandledMessage(message, parsedHeader, receivedData); +} + +TEST(messaging_stream_parser, correctly_parses_split_message) +{ + const uint32_t dataSize = 14; + const MsgHeader* parsedHeader = nullptr; + const char* receivedData = nullptr; + int numCalls = 0; + + std::function handler = [&](const MsgHeader* header, const char* data) { + parsedHeader = header; + receivedData = data; + numCalls++; + return true; + }; + StreamParser parser(handler); + std::vector message = createMessage(5, 57, 56, dataSize); + + size_t readPointer = 0; + // add less than the magic string + parser.onNewData(&message[readPointer], static_cast(MAGIC_STRING_LENGTH - 1)); + readPointer += MAGIC_STRING_LENGTH - 1; + + // handler not yet called + EXPECT_EQ(numCalls, 0); + + // now add more than magic string, less than header + parser.onNewData(&message[readPointer], 2); + readPointer += 2; + + // should still not have been called + EXPECT_EQ(numCalls, 0); + + // now add up to the header + parser.onNewData(&message[readPointer], static_cast(HEADER_SIZE - readPointer)); + readPointer = HEADER_SIZE; + + // should still not have been called + EXPECT_EQ(numCalls, 0); + + // now add part of the data + parser.onNewData(&message[readPointer], static_cast(dataSize / 2)); + readPointer += dataSize / 2; + + // should still not have been called + EXPECT_EQ(numCalls, 0); + + // now add the rest of the data + parser.onNewData(&message[readPointer], static_cast(dataSize / 2)); + readPointer += dataSize / 2; + + // should still not have been called + EXPECT_EQ(numCalls, 1); + + assertHandledMessage(message, parsedHeader, receivedData); +} + +TEST(messaging_stream_parser, correctly_parses_multiple_messages) +{ + std::array, 3> messages = { createMessage(5, 57, 56, 14), + createMessage(6, 58, 57, 18), + createMessage(7, 59, 58, 20) }; + std::vector fullStream; + for (const auto& message : messages) { + fullStream.insert(fullStream.end(), message.begin(), message.end()); + } + uint32_t numCalls = 0; + std::function handler = [&](const MsgHeader* header, const char* data) { + assertHandledMessage(messages[numCalls], header, data); + numCalls++; + return true; + }; + StreamParser parser(handler); + parser.onNewData(&fullStream[0], static_cast(fullStream.size())); + EXPECT_EQ(numCalls, 3); +} + +TEST(messaging_stream_parser, correctly_parses_split_multiple_messages) +{ + std::array, 3> messages = { createMessage(5, 57, 56, 14), + createMessage(6, 58, 57, 14), + createMessage(7, 59, 58, 14) }; + std::vector fullStream; + for (const auto& message : messages) { + fullStream.insert(fullStream.end(), message.begin(), message.end()); + } + uint32_t numCalls = 0; + std::function handler = [&](const MsgHeader* header, const char* data) { + assertHandledMessage(messages[numCalls], header, data); + numCalls++; + return true; + }; + StreamParser parser(handler); + + // add the data in chunks + std::array chunks = { static_cast(fullStream.size() / 6), + static_cast((fullStream.size() / 6) * 2), + static_cast((fullStream.size() / 6) * 2) }; + uint32_t readPointer = 0; + for (size_t i = 0; i < chunks.size(); i++) { + parser.onNewData(&fullStream[readPointer], chunks[i]); + readPointer += chunks[i]; + } + uint32_t remaining = static_cast(fullStream.size()) - readPointer; + parser.onNewData(&fullStream[readPointer], remaining); + EXPECT_EQ(numCalls, 3); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt new file mode 100644 index 00000000000..ff3d7abe47f --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(world_state lmdb barretenberg) diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.cpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.cpp new file mode 100644 index 00000000000..fd94a35db90 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.cpp @@ -0,0 +1,23 @@ + + +#include "world_state_service.hpp" +#include "barretenberg/messaging/header.hpp" +#include + +template +bool WorldStateService::processMessage(const MsgHeader* header, const char*) +{ + if (header->msgType == SystemMsgTypes::TERMINATE) { + return false; + } + if (header->msgType == SystemMsgTypes::PONG) { + sendPong(header->msgId); + } + return true; +} + +template void WorldStateService::sendPong(uint32_t pingId) +{ + MsgHeader header(SystemMsgTypes::PONG, pingId); + outputStream.send(&header); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp new file mode 100644 index 00000000000..4a137f09eb4 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp @@ -0,0 +1,14 @@ +#pragma once +#include "barretenberg/messaging/header.hpp" + +template class WorldStateService { + private: + OutputStream& outputStream; + void sendPong(uint32_t pingId); + + public: + WorldStateService(OutputStream& out) + : outputStream(out) + {} + bool processMessage(const MsgHeader* header, const char* data); +}; \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt new file mode 100644 index 00000000000..4fcfaf91826 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt @@ -0,0 +1,7 @@ +add_executable(world_state_service main.cpp) + +target_link_libraries( + world_state_service + PRIVATE + world_state +) diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp b/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp new file mode 100644 index 00000000000..1a6df30d1a7 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp @@ -0,0 +1,20 @@ +#include "barretenberg/messaging/header.hpp" +#include "barretenberg/messaging/stream_parser.hpp" +#include "barretenberg/world_state/service/world_state_service.hpp" +#include "std_io.hpp" +#include +#include + +int main(int, char**) +{ + std::cout << "Hello World" << std::endl; + SynchronisedStdOutput outputStream(std::cout); + WorldStateService service(outputStream); + + std::function handler = [&service](const MsgHeader* header, const char* data) { + return service.processMessage(header, data); + }; + StreamParser parser(handler); + std::cout << "Hello World" << std::endl; + waitForInput(std::cin, parser); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp b/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp new file mode 100644 index 00000000000..d9841b4d5cd --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "barretenberg/messaging/stream_parser.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +void waitForInput(std::basic_istream& inputStream, StreamParser& parser) +{ + bool terminate = false; + while (terminate) { + std::array data; + std::cout << "Getting input " << std::endl; + inputStream.getline(data.data(), 1024); + std::cout << "After input" << std::endl; + uint32_t length = static_cast(inputStream.gcount()); + terminate = parser.onNewData(data.data(), length); + } +} + +class SynchronisedStdOutput { + private: + std::basic_ostream& stream; + std::mutex mutex; + uint32_t msgId = 1; + + public: + SynchronisedStdOutput(std::basic_ostream& str) + : stream(str) + {} + void send(MsgHeader* header) + { + std::unique_lock lock(mutex); + header->msgId = msgId++; + const char* data = reinterpret_cast(header); + stream.write(data, header->msgLength); + } +}; \ No newline at end of file From ab929a4f8b227103a41fe871be1883872cd4bca7 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 12 Jun 2024 16:09:43 +0100 Subject: [PATCH 11/63] WIP --- .../barretenberg/lmdb_store/CMakeLists.txt | 2 +- .../barretenberg/lmdb_store/lmdb_store.cpp | 4 +- .../barretenberg/lmdb_store/lmdb_store.hpp | 4 +- .../lmdb_store/lmdb_store.test.cpp | 2 +- .../src/barretenberg/messaging/CMakeLists.txt | 2 +- .../cpp/src/barretenberg/messaging/header.hpp | 2 + .../barretenberg/messaging/stream_parser.cpp | 2 + .../barretenberg/messaging/stream_parser.hpp | 4 +- .../messaging/stream_parser.test.cpp | 37 +++++++++++++++++++ .../barretenberg/world_state/CMakeLists.txt | 4 +- .../service/world_state_service.cpp | 23 ------------ .../service/world_state_service.hpp | 24 +++++++++++- .../world_state_service/CMakeLists.txt | 11 +++--- .../barretenberg/world_state_service/main.cpp | 4 +- .../world_state_service/std_io.hpp | 8 ++-- 15 files changed, 87 insertions(+), 46 deletions(-) delete mode 100644 barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.cpp diff --git a/barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt index e6d4d0d9c1d..9050a6601f6 100644 --- a/barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt @@ -1,2 +1,2 @@ -barretenberg_module(lmdb_store lmdb barretenberg) +barretenberg_module(lmdb_store lmdb stdlib_primitives) diff --git a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp index 2b8eed13880..72e34ec22ab 100644 --- a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp +++ b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp @@ -1,6 +1,6 @@ #include "lmdb_store.hpp" -namespace bb::lmdb { +namespace bb::lmdb_store { LMDBEnvironment::LMDBEnvironment(const std::string& directory, unsigned long mapSizeMB, @@ -82,4 +82,4 @@ LMDBReadTransaction::Ptr LMDBStore::createReadTransaction() return std::make_unique(_environment, _database); } -} // namespace bb::lmdb \ No newline at end of file +} // namespace bb::lmdb_store \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp index 00b59b9391c..856f3d303f9 100644 --- a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp +++ b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp @@ -10,7 +10,7 @@ #include #include -namespace bb::lmdb { +namespace bb::lmdb_store { template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) { @@ -179,4 +179,4 @@ class LMDBStore { const std::string _name; LMDBDatabase _database; }; -} // namespace bb::lmdb \ No newline at end of file +} // namespace bb::lmdb_store \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp index 70d2bb3d048..eca52cc7e81 100644 --- a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp @@ -11,7 +11,7 @@ #include "lmdb_store.hpp" using namespace bb::stdlib; -using namespace bb::lmdb; +using namespace bb::lmdb_store; using Builder = bb::UltraCircuitBuilder; diff --git a/barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt index d4f98426dc7..cf4d754e97b 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt @@ -1,2 +1,2 @@ -barretenberg_module(messaging barretenberg) +barretenberg_module(messaging) diff --git a/barretenberg/cpp/src/barretenberg/messaging/header.hpp b/barretenberg/cpp/src/barretenberg/messaging/header.hpp index d2b9586ea35..bf2dbcd2570 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/header.hpp +++ b/barretenberg/cpp/src/barretenberg/messaging/header.hpp @@ -2,6 +2,7 @@ #include #include +namespace bb::messaging { const uint32_t MAGIC_STRING_LENGTH = 8; const uint8_t MAGIC_STRING[] = "AZTEC!!"; @@ -35,3 +36,4 @@ struct MsgHeader { #pragma pack(pop) const uint32_t HEADER_SIZE = sizeof(MsgHeader); +} // namespace bb::messaging diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp index 155c920c617..1b9513e38ec 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp +++ b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp @@ -3,6 +3,7 @@ #include #include +namespace bb::messaging { bool StreamParser::onNewData(char* data, uint32_t length) { uint32_t newSize = bufferLength + length; @@ -60,3 +61,4 @@ void StreamParser::shrinkBuffer() readPointer = 0; bufferLength = dataRemaining; } +} // namespace bb::messaging diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp index f106254f28b..5ed3443868c 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp +++ b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp @@ -6,6 +6,7 @@ #include #include +namespace bb::messaging { class StreamParser { private: std::vector buffer; @@ -20,4 +21,5 @@ class StreamParser { : messageHandler(std::move(handler)) {} bool onNewData(char* data, uint32_t length); -}; \ No newline at end of file +}; +} // namespace bb::messaging diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp index 7f587516418..2c171ab699e 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp +++ b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp @@ -4,9 +4,12 @@ #include "gtest/gtest.h" #include #include +#include #include #include +using namespace bb::messaging; + std::vector createMessage(uint16_t type, uint32_t messageId, uint32_t requestId, uint32_t dataSize) { MsgHeader header(type, requestId, messageId, HEADER_SIZE + dataSize); @@ -132,6 +135,40 @@ TEST(messaging_stream_parser, correctly_parses_split_multiple_messages) }; StreamParser parser(handler); + // add the data in chunks + std::array chunks = { static_cast(fullStream.size() / 6), + static_cast((fullStream.size() / 6) * 2), + static_cast((fullStream.size() / 6) * 2) }; + uint32_t readPointer = 0; + for (size_t i = 0; i < chunks.size(); i++) { + parser.onNewData(&fullStream[readPointer], chunks[i]); + readPointer += chunks[i]; + } + uint32_t remaining = static_cast(fullStream.size()) - readPointer; + parser.onNewData(&fullStream[readPointer], remaining); + EXPECT_EQ(numCalls, 3); +} + +TEST(messaging_stream_parser, correctly_parses_split_multiple_messages_with_garbage) +{ + std::array, 3> messages = { createMessage(5, 57, 56, 14), + createMessage(6, 58, 57, 14), + createMessage(7, 59, 58, 14) }; + std::vector fullStream; + // build the full stream with garbage bytes inserted before each real message + for (const auto& message : messages) { + std::string garbage{ "epfnwpcndcqwedoceodinc" }; + fullStream.insert(fullStream.end(), garbage.begin(), garbage.end()); + fullStream.insert(fullStream.end(), message.begin(), message.end()); + } + uint32_t numCalls = 0; + std::function handler = [&](const MsgHeader* header, const char* data) { + assertHandledMessage(messages[numCalls], header, data); + numCalls++; + return true; + }; + StreamParser parser(handler); + // add the data in chunks std::array chunks = { static_cast(fullStream.size() / 6), static_cast((fullStream.size() / 6) * 2), diff --git a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt index ff3d7abe47f..8e659628ed5 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt @@ -1 +1,3 @@ -barretenberg_module(world_state lmdb barretenberg) +barretenberg_module(world_state lmdb) + + diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.cpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.cpp deleted file mode 100644 index fd94a35db90..00000000000 --- a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.cpp +++ /dev/null @@ -1,23 +0,0 @@ - - -#include "world_state_service.hpp" -#include "barretenberg/messaging/header.hpp" -#include - -template -bool WorldStateService::processMessage(const MsgHeader* header, const char*) -{ - if (header->msgType == SystemMsgTypes::TERMINATE) { - return false; - } - if (header->msgType == SystemMsgTypes::PONG) { - sendPong(header->msgId); - } - return true; -} - -template void WorldStateService::sendPong(uint32_t pingId) -{ - MsgHeader header(SystemMsgTypes::PONG, pingId); - outputStream.send(&header); -} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp index 4a137f09eb4..d87f319a63e 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp @@ -1,6 +1,9 @@ #pragma once #include "barretenberg/messaging/header.hpp" +using namespace bb::messaging; + +namespace bb::world_state { template class WorldStateService { private: OutputStream& outputStream; @@ -11,4 +14,23 @@ template class WorldStateService { : outputStream(out) {} bool processMessage(const MsgHeader* header, const char* data); -}; \ No newline at end of file +}; + +template +bool WorldStateService::processMessage(const MsgHeader* header, const char*) +{ + if (header->msgType == SystemMsgTypes::TERMINATE) { + return false; + } + if (header->msgType == SystemMsgTypes::PONG) { + sendPong(header->msgId); + } + return true; +} + +template void WorldStateService::sendPong(uint32_t pingId) +{ + MsgHeader header(SystemMsgTypes::PONG, pingId); + outputStream.send(&header); +} +} // namespace bb::world_state \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt index 4fcfaf91826..2dbeaa768c7 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt @@ -1,7 +1,6 @@ -add_executable(world_state_service main.cpp) +if (NOT(FUZZING)) + add_executable(world_state main.cpp) -target_link_libraries( - world_state_service - PRIVATE - world_state -) + target_link_libraries(world_state PRIVATE messaging) + + endif() diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp b/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp index 1a6df30d1a7..8ee4d7c5b58 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp @@ -5,9 +5,10 @@ #include #include +using namespace bb::world_state; + int main(int, char**) { - std::cout << "Hello World" << std::endl; SynchronisedStdOutput outputStream(std::cout); WorldStateService service(outputStream); @@ -15,6 +16,5 @@ int main(int, char**) return service.processMessage(header, data); }; StreamParser parser(handler); - std::cout << "Hello World" << std::endl; waitForInput(std::cin, parser); } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp b/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp index d9841b4d5cd..20782cc4e02 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp @@ -10,14 +10,12 @@ #include #include -void waitForInput(std::basic_istream& inputStream, StreamParser& parser) +void waitForInput(std::basic_istream& inputStream, bb::messaging::StreamParser& parser) { bool terminate = false; - while (terminate) { + while (!terminate) { std::array data; - std::cout << "Getting input " << std::endl; inputStream.getline(data.data(), 1024); - std::cout << "After input" << std::endl; uint32_t length = static_cast(inputStream.gcount()); terminate = parser.onNewData(data.data(), length); } @@ -33,7 +31,7 @@ class SynchronisedStdOutput { SynchronisedStdOutput(std::basic_ostream& str) : stream(str) {} - void send(MsgHeader* header) + void send(bb::messaging::MsgHeader* header) { std::unique_lock lock(mutex); header->msgId = msgId++; From 09d10ad1c20bafaf064515875f83cd8aa03fddfd Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 12 Jun 2024 16:10:21 +0100 Subject: [PATCH 12/63] WIP --- barretenberg/cpp/bootstrap.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 80ee4024b63..783a6e2d12c 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -62,6 +62,11 @@ function build_native { cmake --build --preset $PRESET --target bb } +function build_native { + cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert + cmake --build --preset $PRESET --target world_state_service +} + function build_wasm { cmake --preset wasm cmake --build --preset wasm From 09c0d39eba2e91bc1b487a5d80a6aef6ae4ff5fd Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 12 Jun 2024 22:16:51 +0100 Subject: [PATCH 13/63] WIP --- .../merkle-tree/src/factory/tree_factory.ts | 104 ++++++++++++++++++ yarn-project/merkle-tree/src/index.ts | 4 +- .../src/interfaces/tree_factory.ts | 45 ++++++++ yarn-project/merkle-tree/src/load_tree.ts | 33 ------ .../merkle-tree/src/native/native_client.ts | 21 ++++ yarn-project/merkle-tree/src/new_tree.ts | 29 ----- .../src/sparse_tree/sparse_tree.test.ts | 3 +- .../src/standard_tree/standard_tree.test.ts | 3 +- .../src/world-state-db/merkle_trees.ts | 96 ++++------------ 9 files changed, 194 insertions(+), 144 deletions(-) create mode 100644 yarn-project/merkle-tree/src/factory/tree_factory.ts create mode 100644 yarn-project/merkle-tree/src/interfaces/tree_factory.ts delete mode 100644 yarn-project/merkle-tree/src/load_tree.ts create mode 100644 yarn-project/merkle-tree/src/native/native_client.ts delete mode 100644 yarn-project/merkle-tree/src/new_tree.ts diff --git a/yarn-project/merkle-tree/src/factory/tree_factory.ts b/yarn-project/merkle-tree/src/factory/tree_factory.ts new file mode 100644 index 00000000000..56ab65ac6b6 --- /dev/null +++ b/yarn-project/merkle-tree/src/factory/tree_factory.ts @@ -0,0 +1,104 @@ +import { MerkleTreeId } from "@aztec/circuit-types"; +import { Fr } from "@aztec/circuits.js"; +import { Bufferable, FromBuffer } from "@aztec/foundation/serialize"; +import { AztecKVStore } from "@aztec/kv-store"; +import { Hasher } from "@aztec/types/interfaces"; +import { NullifierTree, PublicDataTree, TreeFactory } from "../interfaces/tree_factory.js"; +import { StandardTree } from "../standard_tree/standard_tree.js"; +import { TreeBase, getTreeMeta } from "../tree_base.js"; + + +/** + * Creates a new tree. + * @param c - The class of the tree to be instantiated. + * @param db - A database used to store the Merkle tree data. + * @param hasher - A hasher used to compute hash paths. + * @param name - Name of the tree. + * @param depth - Depth of the tree. + * @param prefilledSize - A number of leaves that are prefilled with values. + * @returns The newly created tree. + */ +export async function newTree, D extends FromBuffer>( + c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, deserializer: D) => T, + store: AztecKVStore, + hasher: Hasher, + name: string, + deserializer: D, + depth: number, + prefilledSize = 1, +): Promise { + const tree = new c(store, hasher, name, depth, 0n, deserializer); + await tree.init(prefilledSize); + return tree; +} + +/** + * Creates a new tree and sets its root, depth and size based on the meta data which are associated with the name. + * @param c - The class of the tree to be instantiated. + * @param db - A database used to store the Merkle tree data. + * @param hasher - A hasher used to compute hash paths. + * @param name - Name of the tree. + * @returns The newly created tree. + */ +export function loadTree, D extends FromBuffer>( + c: new ( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint, + deserializer: D, + root: Buffer, + ) => T, + store: AztecKVStore, + hasher: Hasher, + name: string, + deserializer: D, +): Promise { + const { root, depth, size } = getTreeMeta(store, name); + const tree = new c(store, hasher, name, depth, size, deserializer, root); + return Promise.resolve(tree); +} + + +type InitFunc = , D extends FromBuffer>( + c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, deserializer: D) => T, + store: AztecKVStore, + hasher: Hasher, + name: string, + deserializer: D, + depth: number, + prefilledSize: number, +) => Promise; + + + +export class JSTreeFactory implements TreeFactory { + constructor(private kvStore: AztecKVStore, private hasher: Hasher, private initFunction: InitFunc) {} + + public async init(kvStore: AztecKVStore, hasher: Hasher) { + const factoryMethod = JSTreeFactory.isDbPopulated(kvStore) ? loadTree : newTree; + return new JSTreeFactory(kvStore, hasher, factoryMethod); + } + public async createStandardTree(name: string, depth: number, prefilledSize = 1): Promise> { + return await this.initFunction(StandardTree, this.kvStore, this.hasher, name, Fr, depth, prefilledSize); + } + public async createNullifierTree(name: string, depth:number, prefilledSize = 1): Promise { + return await this.initFunction(NullifierTree, this.kvStore, this.hasher, name, {}, depth, prefilledSize); + } + public async createPublicDataTree(name: string, depth:number, prefilledSize = 1): Promise { + return await this.initFunction(PublicDataTree, this.kvStore, this.hasher, name, {}, depth, prefilledSize); + } + + private static isDbPopulated(kvStore: AztecKVStore): boolean { + try { + getTreeMeta(kvStore, MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]); + // Tree meta was found --> db is populated + return true; + } catch (e) { + // Tree meta was not found --> db is not populated + return false; + } + } + +} \ No newline at end of file diff --git a/yarn-project/merkle-tree/src/index.ts b/yarn-project/merkle-tree/src/index.ts index 7ac5cb2dc6e..dc347123a24 100644 --- a/yarn-project/merkle-tree/src/index.ts +++ b/yarn-project/merkle-tree/src/index.ts @@ -2,6 +2,7 @@ export * from './interfaces/append_only_tree.js'; export * from './interfaces/indexed_tree.js'; export * from './interfaces/merkle_tree.js'; export * from './interfaces/update_only_tree.js'; +export * from './interfaces/tree_factory.js'; export * from './pedersen.js'; export * from './sha_256.js'; export * from './sparse_tree/sparse_tree.js'; @@ -9,8 +10,7 @@ export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tr export { StandardIndexedTreeWithAppend } from './standard_indexed_tree/test/standard_indexed_tree_with_append.js'; export * from './standard_tree/standard_tree.js'; export { INITIAL_LEAF, getTreeMeta } from './tree_base.js'; -export { newTree } from './new_tree.js'; -export { loadTree } from './load_tree.js'; +export * from './factory/tree_factory.js' export * from './snapshots/snapshot_builder.js'; export * from './snapshots/full_snapshot.js'; export * from './snapshots/append_only_snapshot.js'; diff --git a/yarn-project/merkle-tree/src/interfaces/tree_factory.ts b/yarn-project/merkle-tree/src/interfaces/tree_factory.ts new file mode 100644 index 00000000000..72d51c380f7 --- /dev/null +++ b/yarn-project/merkle-tree/src/interfaces/tree_factory.ts @@ -0,0 +1,45 @@ +import { AztecKVStore } from "@aztec/kv-store"; +import { Hasher } from "@aztec/types/interfaces"; +import { StandardTree } from "../standard_tree/standard_tree.js"; +import { Fr, NullifierLeaf, NullifierLeafPreimage, PublicDataTreeLeaf, PublicDataTreeLeafPreimage } from "@aztec/circuits.js"; +import { StandardIndexedTree } from "../standard_indexed_tree/standard_indexed_tree.js"; + +/** + * The nullifier tree is an indexed tree. + */ +export class NullifierTree extends StandardIndexedTree { + constructor( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint = 0n, + _noop: any, + root?: Buffer, + ) { + super(store, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); + } +} + +/** + * The public data tree is an indexed tree. + */ +export class PublicDataTree extends StandardIndexedTree { + constructor( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint = 0n, + _noop: any, + root?: Buffer, + ) { + super(store, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); + } +} + +export interface TreeFactory { + createStandardTree(name: string, depth: number, prefilledSize: number): Promise>; + createNullifierTree(name: string, depth: number, prefilledSize: number): Promise; + createPublicDataTree(name: string, depth: number, prefilledSize: number): Promise; +} \ No newline at end of file diff --git a/yarn-project/merkle-tree/src/load_tree.ts b/yarn-project/merkle-tree/src/load_tree.ts deleted file mode 100644 index b2a1d396b49..00000000000 --- a/yarn-project/merkle-tree/src/load_tree.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { type Bufferable, type FromBuffer } from '@aztec/foundation/serialize'; -import { type AztecKVStore } from '@aztec/kv-store'; -import { type Hasher } from '@aztec/types/interfaces'; - -import { type TreeBase, getTreeMeta } from './tree_base.js'; - -/** - * Creates a new tree and sets its root, depth and size based on the meta data which are associated with the name. - * @param c - The class of the tree to be instantiated. - * @param db - A database used to store the Merkle tree data. - * @param hasher - A hasher used to compute hash paths. - * @param name - Name of the tree. - * @returns The newly created tree. - */ -export function loadTree, D extends FromBuffer>( - c: new ( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint, - deserializer: D, - root: Buffer, - ) => T, - store: AztecKVStore, - hasher: Hasher, - name: string, - deserializer: D, -): Promise { - const { root, depth, size } = getTreeMeta(store, name); - const tree = new c(store, hasher, name, depth, size, deserializer, root); - return Promise.resolve(tree); -} diff --git a/yarn-project/merkle-tree/src/native/native_client.ts b/yarn-project/merkle-tree/src/native/native_client.ts new file mode 100644 index 00000000000..29936196f79 --- /dev/null +++ b/yarn-project/merkle-tree/src/native/native_client.ts @@ -0,0 +1,21 @@ +import { createDebugLogger } from "@aztec/foundation/log"; +import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; + +export type NativeWorldStateConfig = { + worldStateBinaryPath: string; +} + +class NativeTrees { + constructor(private childProcess: ChildProcessWithoutNullStreams) { + + } + public static init(config: NativeWorldStateConfig, logger = createDebugLogger('aztec:native_client')) { + + const child = spawn(config.worldStateBinaryPath); + console.log(`${new Date()} : CHILD STARTED`); + child.stdout.on("data", (d) => console.log(`${new Date()} : STDOUT => ${d}`)); + child.stderr.on("data", (d) => console.log(`${new Date()} : STDERR => ${d}`)); + child.on("close", () => console.log(`${new Date()} : CHILD ENDED`)); + return new NativeTrees(child); + } +} \ No newline at end of file diff --git a/yarn-project/merkle-tree/src/new_tree.ts b/yarn-project/merkle-tree/src/new_tree.ts deleted file mode 100644 index d8040fc9f46..00000000000 --- a/yarn-project/merkle-tree/src/new_tree.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { type Bufferable, type FromBuffer } from '@aztec/foundation/serialize'; -import { type AztecKVStore } from '@aztec/kv-store'; -import { type Hasher } from '@aztec/types/interfaces'; - -import { type TreeBase } from './tree_base.js'; - -/** - * Creates a new tree. - * @param c - The class of the tree to be instantiated. - * @param db - A database used to store the Merkle tree data. - * @param hasher - A hasher used to compute hash paths. - * @param name - Name of the tree. - * @param depth - Depth of the tree. - * @param prefilledSize - A number of leaves that are prefilled with values. - * @returns The newly created tree. - */ -export async function newTree, D extends FromBuffer>( - c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, deserializer: D) => T, - store: AztecKVStore, - hasher: Hasher, - name: string, - deserializer: D, - depth: number, - prefilledSize = 1, -): Promise { - const tree = new c(store, hasher, name, depth, 0n, deserializer); - await tree.init(prefilledSize); - return tree; -} diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts index 56cbaee0a58..19b6c28a2e0 100644 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts +++ b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts @@ -6,9 +6,8 @@ import { type AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type Hasher } from '@aztec/types/interfaces'; -import { INITIAL_LEAF, newTree } from '../index.js'; +import { INITIAL_LEAF, loadTree, newTree } from '../index.js'; import { type UpdateOnlyTree } from '../interfaces/update_only_tree.js'; -import { loadTree } from '../load_tree.js'; import { Pedersen } from '../pedersen.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts index b01409eb14c..f78fe883f1c 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts @@ -4,13 +4,12 @@ import { type AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type Hasher } from '@aztec/types/interfaces'; -import { loadTree } from '../load_tree.js'; -import { newTree } from '../new_tree.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; import { PedersenWithCounter } from '../test/utils/pedersen_with_counter.js'; import { INITIAL_LEAF } from '../tree_base.js'; import { StandardTree } from './standard_tree.js'; +import { JSTreeFactory, loadTree, newTree } from '../factory/tree_factory.js'; const noopDeserializer: FromBuffer = { fromBuffer: (buffer: Buffer) => buffer, diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index be41f087fdc..b76e0cddbde 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -1,4 +1,4 @@ -import { type L2Block, MerkleTreeId, PublicDataWrite, type SiblingPath, TxEffect } from '@aztec/circuit-types'; +import { MerkleTreeId, PublicDataWrite, TxEffect, type L2Block, type SiblingPath } from '@aztec/circuit-types'; import { ARCHIVE_HEIGHT, AppendOnlyTreeSnapshot, @@ -14,33 +14,30 @@ import { NULLIFIER_SUBTREE_HEIGHT, NULLIFIER_TREE_HEIGHT, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, - NullifierLeaf, - NullifierLeafPreimage, PUBLIC_DATA_SUBTREE_HEIGHT, PUBLIC_DATA_TREE_HEIGHT, PartialStateReference, PublicDataTreeLeaf, - PublicDataTreeLeafPreimage, - StateReference, + StateReference } from '@aztec/circuits.js'; import { padArrayEnd } from '@aztec/foundation/collection'; import { SerialQueue } from '@aztec/foundation/fifo'; -import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; +import { createDebugLogger, type DebugLogger } from '@aztec/foundation/log'; import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { type AztecKVStore } from '@aztec/kv-store'; import { - type AppendOnlyTree, - type BatchInsertionResult, - type IndexedTree, Pedersen, StandardIndexedTree, StandardTree, - type UpdateOnlyTree, getTreeMeta, loadTree, newTree, + type AppendOnlyTree, + type BatchInsertionResult, + type IndexedTree, + type UpdateOnlyTree, + type TreeFactory, } from '@aztec/merkle-tree'; -import { type Hasher } from '@aztec/types/interfaces'; import { INITIAL_NULLIFIER_TREE_SIZE, @@ -58,39 +55,6 @@ import { } from './merkle_tree_operations.js'; import { MerkleTreeOperationsFacade } from './merkle_tree_operations_facade.js'; -/** - * The nullifier tree is an indexed tree. - */ -class NullifierTree extends StandardIndexedTree { - constructor( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint = 0n, - _noop: any, - root?: Buffer, - ) { - super(store, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); - } -} - -/** - * The public data tree is an indexed tree. - */ -class PublicDataTree extends StandardIndexedTree { - constructor( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint = 0n, - _noop: any, - root?: Buffer, - ) { - super(store, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); - } -} /** * A convenience class for managing multiple merkle trees. @@ -107,62 +71,42 @@ export class MerkleTrees implements MerkleTreeDb { * @param store - The db instance to use for data persistance. * @returns - A fully initialized MerkleTrees instance. */ - public static async new(store: AztecKVStore, log = createDebugLogger('aztec:merkle_trees')) { + public static async new(store: AztecKVStore, treeFactory: TreeFactory, log = createDebugLogger('aztec:merkle_trees')) { const merkleTrees = new MerkleTrees(store, log); - await merkleTrees.#init(); + await merkleTrees.#init(treeFactory); return merkleTrees; } /** * Initializes the collection of Merkle Trees. */ - async #init() { + async #init(treeFactory: TreeFactory) { const fromDb = this.#isDbPopulated(); const initializeTree = fromDb ? loadTree : newTree; const hasher = new Pedersen(); - const nullifierTree = await initializeTree( - NullifierTree, - this.store, - hasher, - `${MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]}`, - {}, - NULLIFIER_TREE_HEIGHT, - INITIAL_NULLIFIER_TREE_SIZE, - ); - const noteHashTree: AppendOnlyTree = await initializeTree( - StandardTree, - this.store, - hasher, + const nullifierTree = await treeFactory.createNullifierTree(`${MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]}`, NULLIFIER_TREE_HEIGHT, INITIAL_NULLIFIER_TREE_SIZE); + + const noteHashTree: AppendOnlyTree = await treeFactory.createStandardTree( `${MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE]}`, - Fr, NOTE_HASH_TREE_HEIGHT, + 1 ); - const publicDataTree = await initializeTree( - PublicDataTree, - this.store, - hasher, + const publicDataTree = await treeFactory.createPublicDataTree( `${MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE]}`, - {}, PUBLIC_DATA_TREE_HEIGHT, INITIAL_PUBLIC_DATA_TREE_SIZE, ); - const l1Tol2MessageTree: AppendOnlyTree = await initializeTree( - StandardTree, - this.store, - hasher, + const l1Tol2MessageTree: AppendOnlyTree = await treeFactory.createStandardTree( `${MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]}`, - Fr, L1_TO_L2_MSG_TREE_HEIGHT, + 1 ); - const archive: AppendOnlyTree = await initializeTree( - StandardTree, - this.store, - hasher, + const archive: AppendOnlyTree = await treeFactory.createStandardTree( `${MerkleTreeId[MerkleTreeId.ARCHIVE]}`, - Fr, ARCHIVE_HEIGHT, + 1, ); this.trees = [nullifierTree, noteHashTree, publicDataTree, l1Tol2MessageTree, archive]; From e4bdb28029ef0aed87711b492cb93b9d5c671a8b Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 12 Jun 2024 22:17:19 +0100 Subject: [PATCH 14/63] Fix --- barretenberg/cpp/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 8b8b3ef6309..2d301d6c6d1 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -61,7 +61,7 @@ function build_native { function build_native { cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert - cmake --build --preset $PRESET --target world_state_service + cmake --build --preset $PRESET --target world_state } function build_wasm { From 3a102bd2123b81069feab858b1ab1d508e390d05 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 13 Jun 2024 10:04:08 +0100 Subject: [PATCH 15/63] WIP --- yarn-project/aztec-node/src/aztec-node/server.ts | 8 +++++--- .../src/composed/integration_l1_publisher.test.ts | 5 +++-- yarn-project/merkle-tree/src/factory/tree_factory.ts | 3 ++- yarn-project/merkle-tree/src/index.ts | 2 +- yarn-project/prover-client/package.json | 1 + yarn-project/prover-client/src/mocks/test_context.ts | 6 ++++-- .../src/orchestrator/orchestrator_mixed_blocks_2.test.ts | 4 +++- .../src/orchestrator/orchestrator_single_blocks.test.ts | 4 +++- .../src/orchestrator/orchestrator_workflow.test.ts | 4 +++- 9 files changed, 25 insertions(+), 12 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 5ccbf6266c7..be0ca099f43 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -53,7 +53,7 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { type AztecKVStore } from '@aztec/kv-store'; import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils'; -import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; +import { JSTreeFactory, SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { AztecKVTxPool, type P2P, createP2PClient } from '@aztec/p2p'; import { getCanonicalClassRegisterer } from '@aztec/protocol-contracts/class-registerer'; import { getCanonicalGasToken } from '@aztec/protocol-contracts/gas-token'; @@ -156,7 +156,8 @@ export class AztecNodeService implements AztecNode { const p2pClient = await createP2PClient(store, config, new AztecKVTxPool(store), archiver); // now create the merkle trees and the world state synchronizer - const merkleTrees = await MerkleTrees.new(store); + const treeFactory = await JSTreeFactory.init(store); + const merkleTrees = await MerkleTrees.new(store, treeFactory); const worldStateConfig: WorldStateConfig = getWorldStateConfig(); const worldStateSynchronizer = new ServerWorldStateSynchronizer(store, merkleTrees, archiver, worldStateConfig); @@ -748,7 +749,8 @@ export class AztecNodeService implements AztecNode { // Instantiate merkle trees so uncommitted updates by this simulation are local to it. // TODO we should be able to remove this after https://github.com/AztecProtocol/aztec-packages/issues/1869 // So simulation of public functions doesn't affect the merkle trees. - const merkleTrees = await MerkleTrees.new(this.merkleTreesDb, this.log); + const treeFactory = await JSTreeFactory.init(this.merkleTreesDb); + const merkleTrees = await MerkleTrees.new(this.merkleTreesDb, treeFactory, this.log); const publicProcessorFactory = new PublicProcessorFactory( merkleTrees.asLatest(), diff --git a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts index 4445829658c..393c78c3f7d 100644 --- a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts @@ -37,7 +37,7 @@ import { type L1ContractAddresses, createEthereumChain } from '@aztec/ethereum'; import { makeTuple, range } from '@aztec/foundation/array'; import { openTmpStore } from '@aztec/kv-store/utils'; import { AvailabilityOracleAbi, InboxAbi, OutboxAbi, RollupAbi } from '@aztec/l1-artifacts'; -import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; +import { JSTreeFactory, SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { TxProver } from '@aztec/prover-client'; import { type L1Publisher, getL1Publisher } from '@aztec/sequencer-client'; import { MerkleTrees, ServerWorldStateSynchronizer, type WorldStateConfig } from '@aztec/world-state'; @@ -136,7 +136,8 @@ describe('L1Publisher integration', () => { }); const tmpStore = openTmpStore(); - builderDb = await MerkleTrees.new(tmpStore); + const treeFactory = await JSTreeFactory.init(tmpStore); + builderDb = await MerkleTrees.new(tmpStore, treeFactory); blockSource = mock(); blockSource.getBlocks.mockResolvedValue([]); const worldStateConfig: WorldStateConfig = { diff --git a/yarn-project/merkle-tree/src/factory/tree_factory.ts b/yarn-project/merkle-tree/src/factory/tree_factory.ts index 56ab65ac6b6..12e31859e3b 100644 --- a/yarn-project/merkle-tree/src/factory/tree_factory.ts +++ b/yarn-project/merkle-tree/src/factory/tree_factory.ts @@ -6,6 +6,7 @@ import { Hasher } from "@aztec/types/interfaces"; import { NullifierTree, PublicDataTree, TreeFactory } from "../interfaces/tree_factory.js"; import { StandardTree } from "../standard_tree/standard_tree.js"; import { TreeBase, getTreeMeta } from "../tree_base.js"; +import { Pedersen } from "../pedersen.js"; /** @@ -76,7 +77,7 @@ type InitFunc = , D extends FromBuffer(); const publicWorldStateDB = mock(); const publicKernel = new RealPublicKernelCircuitSimulator(new WASMSimulator()); - const actualDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + const kvStore = openTmpStore(); + const actualDb = await MerkleTrees.new(kvStore, await JSTreeFactory.init(kvStore)).then(t => t.asLatest()); const processor = new PublicProcessor( actualDb, publicExecutor, diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts index c9f8b930df2..ce89757e455 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts @@ -6,6 +6,7 @@ import { times } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; +import { JSTreeFactory } from '@aztec/merkle-tree'; import { makeBloatedProcessedTx, makeEmptyProcessedTestTx, updateExpectedTreesFromTxs } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; @@ -18,7 +19,8 @@ describe('prover/orchestrator/mixed-blocks', () => { beforeEach(async () => { context = await TestContext.new(logger); - expectsDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + const kvStore = openTmpStore(); + expectsDb = await MerkleTrees.new(kvStore, await JSTreeFactory.init(kvStore)).then(t => t.asLatest()); }); afterEach(async () => { diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts index 808aa3dae8d..3aa2bf43355 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts @@ -6,6 +6,7 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; +import { JSTreeFactory } from '@aztec/merkle-tree'; import { makeBloatedProcessedTx, makeEmptyProcessedTestTx, updateExpectedTreesFromTxs } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; @@ -18,7 +19,8 @@ describe('prover/orchestrator/blocks', () => { beforeEach(async () => { context = await TestContext.new(logger); - expectsDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + const kvStore = openTmpStore(); + expectsDb = await MerkleTrees.new(kvStore, await JSTreeFactory.init(kvStore)).then(t => t.asLatest()); }); afterEach(async () => { diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts index 07158f9aeec..c2151327d4c 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts @@ -12,6 +12,7 @@ import { promiseWithResolvers } from '@aztec/foundation/promise'; import { sleep } from '@aztec/foundation/sleep'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; +import { JSTreeFactory } from '@aztec/merkle-tree'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -23,7 +24,8 @@ describe('prover/orchestrator', () => { let mockProver: MockProxy; let actualDb: MerkleTreeOperations; beforeEach(async () => { - actualDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); + const kvStore = openTmpStore(); + actualDb = await MerkleTrees.new(kvStore, await JSTreeFactory.init(kvStore)).then(t => t.asLatest()); mockProver = mock(); orchestrator = new ProvingOrchestrator(actualDb, mockProver); }); From 94e960c31d0af269a971c6c217064ec9405189ec Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 13 Jun 2024 10:18:45 +0100 Subject: [PATCH 16/63] WIP --- .../merkle-tree/src/factory/tree_factory.ts | 30 +++++++--------- yarn-project/merkle-tree/src/index.ts | 4 +-- .../src/interfaces/tree_factory.ts | 19 +++++++---- .../merkle-tree/src/native/native_client.ts | 34 +++++++++---------- .../src/standard_tree/standard_tree.test.ts | 2 +- yarn-project/prover-client/package.json | 2 +- .../prover-client/src/mocks/test_context.ts | 4 +-- .../orchestrator_mixed_blocks_2.test.ts | 2 +- .../orchestrator_single_blocks.test.ts | 2 +- .../orchestrator_workflow.test.ts | 2 +- yarn-project/prover-client/tsconfig.json | 3 ++ .../src/world-state-db/merkle_trees.ts | 2 +- 12 files changed, 56 insertions(+), 50 deletions(-) diff --git a/yarn-project/merkle-tree/src/factory/tree_factory.ts b/yarn-project/merkle-tree/src/factory/tree_factory.ts index 12e31859e3b..f3ae092a717 100644 --- a/yarn-project/merkle-tree/src/factory/tree_factory.ts +++ b/yarn-project/merkle-tree/src/factory/tree_factory.ts @@ -1,13 +1,13 @@ -import { MerkleTreeId } from "@aztec/circuit-types"; -import { Fr } from "@aztec/circuits.js"; -import { Bufferable, FromBuffer } from "@aztec/foundation/serialize"; -import { AztecKVStore } from "@aztec/kv-store"; -import { Hasher } from "@aztec/types/interfaces"; -import { NullifierTree, PublicDataTree, TreeFactory } from "../interfaces/tree_factory.js"; -import { StandardTree } from "../standard_tree/standard_tree.js"; -import { TreeBase, getTreeMeta } from "../tree_base.js"; -import { Pedersen } from "../pedersen.js"; +import { MerkleTreeId } from '@aztec/circuit-types'; +import { Fr } from '@aztec/circuits.js'; +import { type Bufferable, type FromBuffer } from '@aztec/foundation/serialize'; +import { type AztecKVStore } from '@aztec/kv-store'; +import { type Hasher } from '@aztec/types/interfaces'; +import { NullifierTree, PublicDataTree, type TreeFactory } from '../interfaces/tree_factory.js'; +import { Pedersen } from '../pedersen.js'; +import { StandardTree } from '../standard_tree/standard_tree.js'; +import { type TreeBase, getTreeMeta } from '../tree_base.js'; /** * Creates a new tree. @@ -61,7 +61,6 @@ export function loadTree, D extends FromBuffer, D extends FromBuffer>( c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, deserializer: D) => T, store: AztecKVStore, @@ -72,22 +71,20 @@ type InitFunc = , D extends FromBuffer Promise; - - export class JSTreeFactory implements TreeFactory { constructor(private kvStore: AztecKVStore, private hasher: Hasher, private initFunction: InitFunc) {} - public static async init(kvStore: AztecKVStore, hasher: Hasher = new Pedersen()) { + public static init(kvStore: AztecKVStore, hasher: Hasher = new Pedersen()) { const factoryMethod = JSTreeFactory.isDbPopulated(kvStore) ? loadTree : newTree; return new JSTreeFactory(kvStore, hasher, factoryMethod); } public async createStandardTree(name: string, depth: number, prefilledSize = 1): Promise> { return await this.initFunction(StandardTree, this.kvStore, this.hasher, name, Fr, depth, prefilledSize); } - public async createNullifierTree(name: string, depth:number, prefilledSize = 1): Promise { + public async createNullifierTree(name: string, depth: number, prefilledSize = 1): Promise { return await this.initFunction(NullifierTree, this.kvStore, this.hasher, name, {}, depth, prefilledSize); } - public async createPublicDataTree(name: string, depth:number, prefilledSize = 1): Promise { + public async createPublicDataTree(name: string, depth: number, prefilledSize = 1): Promise { return await this.initFunction(PublicDataTree, this.kvStore, this.hasher, name, {}, depth, prefilledSize); } @@ -101,5 +98,4 @@ export class JSTreeFactory implements TreeFactory { return false; } } - -} \ No newline at end of file +} diff --git a/yarn-project/merkle-tree/src/index.ts b/yarn-project/merkle-tree/src/index.ts index a411f807c9d..89b52580867 100644 --- a/yarn-project/merkle-tree/src/index.ts +++ b/yarn-project/merkle-tree/src/index.ts @@ -10,8 +10,8 @@ export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tr export { StandardIndexedTreeWithAppend } from './standard_indexed_tree/test/standard_indexed_tree_with_append.js'; export * from './standard_tree/standard_tree.js'; export { INITIAL_LEAF, getTreeMeta } from './tree_base.js'; -export * from './factory/tree_factory.js' +export * from './factory/tree_factory.js'; export * from './snapshots/snapshot_builder.js'; export * from './snapshots/full_snapshot.js'; export * from './snapshots/append_only_snapshot.js'; -export * from './snapshots/indexed_tree_snapshot.js'; \ No newline at end of file +export * from './snapshots/indexed_tree_snapshot.js'; diff --git a/yarn-project/merkle-tree/src/interfaces/tree_factory.ts b/yarn-project/merkle-tree/src/interfaces/tree_factory.ts index 72d51c380f7..0e9f9f4097a 100644 --- a/yarn-project/merkle-tree/src/interfaces/tree_factory.ts +++ b/yarn-project/merkle-tree/src/interfaces/tree_factory.ts @@ -1,8 +1,15 @@ -import { AztecKVStore } from "@aztec/kv-store"; -import { Hasher } from "@aztec/types/interfaces"; -import { StandardTree } from "../standard_tree/standard_tree.js"; -import { Fr, NullifierLeaf, NullifierLeafPreimage, PublicDataTreeLeaf, PublicDataTreeLeafPreimage } from "@aztec/circuits.js"; -import { StandardIndexedTree } from "../standard_indexed_tree/standard_indexed_tree.js"; +import { + type Fr, + NullifierLeaf, + NullifierLeafPreimage, + PublicDataTreeLeaf, + PublicDataTreeLeafPreimage, +} from '@aztec/circuits.js'; +import { type AztecKVStore } from '@aztec/kv-store'; +import { type Hasher } from '@aztec/types/interfaces'; + +import { StandardIndexedTree } from '../standard_indexed_tree/standard_indexed_tree.js'; +import { type StandardTree } from '../standard_tree/standard_tree.js'; /** * The nullifier tree is an indexed tree. @@ -42,4 +49,4 @@ export interface TreeFactory { createStandardTree(name: string, depth: number, prefilledSize: number): Promise>; createNullifierTree(name: string, depth: number, prefilledSize: number): Promise; createPublicDataTree(name: string, depth: number, prefilledSize: number): Promise; -} \ No newline at end of file +} diff --git a/yarn-project/merkle-tree/src/native/native_client.ts b/yarn-project/merkle-tree/src/native/native_client.ts index 29936196f79..5f2b4568328 100644 --- a/yarn-project/merkle-tree/src/native/native_client.ts +++ b/yarn-project/merkle-tree/src/native/native_client.ts @@ -1,21 +1,21 @@ -import { createDebugLogger } from "@aztec/foundation/log"; -import { ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +// import { createDebugLogger } from "@aztec/foundation/log"; +// import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; -export type NativeWorldStateConfig = { - worldStateBinaryPath: string; -} +// export type NativeWorldStateConfig = { +// worldStateBinaryPath: string; +// } -class NativeTrees { - constructor(private childProcess: ChildProcessWithoutNullStreams) { +// class NativeTrees { +// constructor(private childProcess: ChildProcessWithoutNullStreams) { - } - public static init(config: NativeWorldStateConfig, logger = createDebugLogger('aztec:native_client')) { +// } +// public static init(config: NativeWorldStateConfig, logger = createDebugLogger('aztec:native_client')) { - const child = spawn(config.worldStateBinaryPath); - console.log(`${new Date()} : CHILD STARTED`); - child.stdout.on("data", (d) => console.log(`${new Date()} : STDOUT => ${d}`)); - child.stderr.on("data", (d) => console.log(`${new Date()} : STDERR => ${d}`)); - child.on("close", () => console.log(`${new Date()} : CHILD ENDED`)); - return new NativeTrees(child); - } -} \ No newline at end of file +// const child = spawn(config.worldStateBinaryPath); +// console.log(`${new Date()} : CHILD STARTED`); +// child.stdout.on("data", (d) => console.log(`${new Date()} : STDOUT => ${d}`)); +// child.stderr.on("data", (d) => console.log(`${new Date()} : STDERR => ${d}`)); +// child.on("close", () => console.log(`${new Date()} : CHILD ENDED`)); +// return new NativeTrees(child); +// } +// } diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts index f78fe883f1c..e3a4285620a 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts @@ -4,12 +4,12 @@ import { type AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type Hasher } from '@aztec/types/interfaces'; +import { loadTree, newTree } from '../factory/tree_factory.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; import { PedersenWithCounter } from '../test/utils/pedersen_with_counter.js'; import { INITIAL_LEAF } from '../tree_base.js'; import { StandardTree } from './standard_tree.js'; -import { JSTreeFactory, loadTree, newTree } from '../factory/tree_factory.js'; const noopDeserializer: FromBuffer = { fromBuffer: (buffer: Buffer) => buffer, diff --git a/yarn-project/prover-client/package.json b/yarn-project/prover-client/package.json index 3a9abe20562..78ec16a9283 100644 --- a/yarn-project/prover-client/package.json +++ b/yarn-project/prover-client/package.json @@ -55,7 +55,7 @@ "@aztec/circuits.js": "workspace:^", "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", - "@aztec/merkle-trees": "workspace:^", + "@aztec/merkle-tree": "workspace:^", "@aztec/noir-protocol-circuits-types": "workspace:^", "@aztec/simulator": "workspace:^", "@aztec/world-state": "workspace:^", diff --git a/yarn-project/prover-client/src/mocks/test_context.ts b/yarn-project/prover-client/src/mocks/test_context.ts index 8edd93216e2..68ffa788527 100644 --- a/yarn-project/prover-client/src/mocks/test_context.ts +++ b/yarn-project/prover-client/src/mocks/test_context.ts @@ -19,6 +19,7 @@ import { import { type Fr } from '@aztec/foundation/fields'; import { type DebugLogger } from '@aztec/foundation/log'; import { openTmpStore } from '@aztec/kv-store/utils'; +import { JSTreeFactory } from '@aztec/merkle-tree'; import { type ContractsDataSourcePublicDB, type PublicExecution, @@ -31,8 +32,7 @@ import { WASMSimulator, type WorldStatePublicDB, } from '@aztec/simulator'; -import { MerkleTrees, type MerkleTreeOperations } from '@aztec/world-state'; -import { JSTreeFactory } from '@aztec/merkle-tree'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import * as fs from 'fs/promises'; import { type MockProxy, mock } from 'jest-mock-extended'; diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts index ce89757e455..26be8d9ce37 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts @@ -5,8 +5,8 @@ import { range } from '@aztec/foundation/array'; import { times } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; import { openTmpStore } from '@aztec/kv-store/utils'; -import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { JSTreeFactory } from '@aztec/merkle-tree'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { makeBloatedProcessedTx, makeEmptyProcessedTestTx, updateExpectedTreesFromTxs } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts index 3aa2bf43355..1d4b2e53a00 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts @@ -5,8 +5,8 @@ import { range } from '@aztec/foundation/array'; import { createDebugLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; import { openTmpStore } from '@aztec/kv-store/utils'; -import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { JSTreeFactory } from '@aztec/merkle-tree'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { makeBloatedProcessedTx, makeEmptyProcessedTestTx, updateExpectedTreesFromTxs } from '../mocks/fixtures.js'; import { TestContext } from '../mocks/test_context.js'; diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts index c2151327d4c..764b37007dc 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts @@ -11,8 +11,8 @@ import { makeGlobalVariables, makeRootParityInput } from '@aztec/circuits.js/tes import { promiseWithResolvers } from '@aztec/foundation/promise'; import { sleep } from '@aztec/foundation/sleep'; import { openTmpStore } from '@aztec/kv-store/utils'; -import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { JSTreeFactory } from '@aztec/merkle-tree'; +import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { type MockProxy, mock } from 'jest-mock-extended'; diff --git a/yarn-project/prover-client/tsconfig.json b/yarn-project/prover-client/tsconfig.json index 5f4666ebf03..0cbffc97b3e 100644 --- a/yarn-project/prover-client/tsconfig.json +++ b/yarn-project/prover-client/tsconfig.json @@ -21,6 +21,9 @@ { "path": "../kv-store" }, + { + "path": "../merkle-tree" + }, { "path": "../noir-protocol-circuits-types" }, diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index b76e0cddbde..d781daddf6e 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -27,7 +27,7 @@ import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { type AztecKVStore } from '@aztec/kv-store'; import { Pedersen, - StandardIndexedTree, + type StandardIndexedTree, StandardTree, getTreeMeta, loadTree, From 827458226b76f5050818a92436650f6b67d00a07 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 13 Jun 2024 10:50:21 +0100 Subject: [PATCH 17/63] Lock file --- yarn-project/yarn.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index d1fe44fd9df..5e715fd4690 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -774,6 +774,7 @@ __metadata: "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" + "@aztec/merkle-tree": "workspace:^" "@aztec/noir-protocol-circuits-types": "workspace:^" "@aztec/simulator": "workspace:^" "@aztec/world-state": "workspace:^" From 79956df50eaa4b13946996bf4c8d5ff9e1e112ee Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 13 Jun 2024 11:07:07 +0100 Subject: [PATCH 18/63] Formatting --- .../stdlib_circuit_builders/ultra_circuit_builder.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp index d4df961d6d8..10317b7a009 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp @@ -486,10 +486,7 @@ class UltraCircuitBuilder_ : public CircuitBuilderBase Date: Thu, 13 Jun 2024 22:47:56 +0100 Subject: [PATCH 19/63] WIP --- .../cpp/src/barretenberg/messaging/header.hpp | 20 +++++------ .../barretenberg/messaging/stream_parser.cpp | 1 + .../world_state/service/message.hpp | 32 +++++++++++++++++ .../service/world_state_service.hpp | 34 ++++++++++++++++++- .../barretenberg/world_state_service/main.cpp | 1 + .../world_state_service/std_io.hpp | 15 ++++---- 6 files changed, 85 insertions(+), 18 deletions(-) create mode 100644 barretenberg/cpp/src/barretenberg/world_state/service/message.hpp diff --git a/barretenberg/cpp/src/barretenberg/messaging/header.hpp b/barretenberg/cpp/src/barretenberg/messaging/header.hpp index bf2dbcd2570..028a1b57ea5 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/header.hpp +++ b/barretenberg/cpp/src/barretenberg/messaging/header.hpp @@ -4,7 +4,7 @@ namespace bb::messaging { const uint32_t MAGIC_STRING_LENGTH = 8; -const uint8_t MAGIC_STRING[] = "AZTEC!!"; +const uint8_t MAGIC_STRING[] = "AZTEC!!!"; // The null terminator is not included enum SystemMsgTypes { TERMINATE = 0, PING = 1, PONG = 2 }; @@ -14,24 +14,24 @@ const uint32_t FIRST_APP_MSG_TYPE = 100; struct MsgHeader { uint8_t msgStart[MAGIC_STRING_LENGTH]; // MAGIC_STRING uint32_t msgLength; // Length including the header - uint16_t msgType; // The type of message + uint32_t msgType; // The type of message uint32_t msgId; // Unique Id for the message uint32_t requestId; // Id of the message this is responding too (may not be used) - MsgHeader(uint16_t type, uint32_t reqId) - : msgLength(sizeof(MsgHeader)) + MsgHeader(uint32_t type, uint32_t reqId, uint32_t lengthWithoutHeader = 0) + : msgLength(sizeof(MsgHeader) + lengthWithoutHeader) , msgType(type) , requestId(reqId) { std::memcpy(msgStart, MAGIC_STRING, MAGIC_STRING_LENGTH); } - MsgHeader(uint16_t type, uint32_t reqId, uint32_t msgId, uint32_t totalLength) - : MsgHeader(type, reqId) - { - this->msgId = msgId; - this->msgLength = totalLength; - } + // MsgHeader(uint32_t type, uint32_t reqId, uint32_t msgId, uint32_t totalLength) + // : MsgHeader(type, reqId) + // { + // this->msgId = msgId; + // this->msgLength = totalLength; + // } }; #pragma pack(pop) diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp index 1b9513e38ec..ab595ce3709 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp +++ b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp @@ -2,6 +2,7 @@ #include "barretenberg/messaging/header.hpp" #include #include +#include namespace bb::messaging { bool StreamParser::onNewData(char* data, uint32_t length) diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/message.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/message.hpp new file mode 100644 index 00000000000..6b2da95df0a --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/world_state/service/message.hpp @@ -0,0 +1,32 @@ +#pragma once +#include "barretenberg/messaging/header.hpp" +#include "barretenberg/serialize/msgpack.hpp" +#include + +namespace bb::world_state { + +using namespace bb::messaging; +enum WorldStateMsgTypes { + START_TREE_REQUEST = FIRST_APP_MSG_TYPE, + START_TREE_RESPONSE, + GET_TREE_INFO_REQUEST, + GET_TREE_INFO_RESPONSE +}; + +struct StartTreeRequest { + std::string name; + uint32_t depth; + + MSGPACK_FIELDS(name, depth); +}; + +struct StartTreeResponse { + std::string name; + uint32_t depth; + bool success; + std::string reason; + + MSGPACK_FIELDS(name, depth, success, reason); +}; + +} // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp index d87f319a63e..5795c34e01c 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp @@ -1,5 +1,12 @@ #pragma once #include "barretenberg/messaging/header.hpp" +#include "barretenberg/serialize/cbind.hpp" +#include "barretenberg/world_state/service/message.hpp" +#include "msgpack/v3/sbuffer_decl.hpp" +#include +#include +#include +#include using namespace bb::messaging; @@ -7,8 +14,11 @@ namespace bb::world_state { template class WorldStateService { private: OutputStream& outputStream; + std::unordered_map> messagesHandlers; void sendPong(uint32_t pingId); + void startTree(uint32_t requestId, const char* data, uint32_t length); + public: WorldStateService(OutputStream& out) : outputStream(out) @@ -20,9 +30,10 @@ template bool WorldStateService::processMessage(const MsgHeader* header, const char*) { if (header->msgType == SystemMsgTypes::TERMINATE) { + std::cerr << "Terminating!!" << std::endl; return false; } - if (header->msgType == SystemMsgTypes::PONG) { + if (header->msgType == SystemMsgTypes::PING) { sendPong(header->msgId); } return true; @@ -33,4 +44,25 @@ template void WorldStateService::sendPong( MsgHeader header(SystemMsgTypes::PONG, pingId); outputStream.send(&header); } + +template +void WorldStateService::startTree(uint32_t requestId, const char* data, uint32_t length) +{ + StartTreeRequest startTreeRequest; + msgpack::unpack(data, length).get().convert(startTreeRequest); + std::cout << "Received Start Tree Request " << startTreeRequest.name << " " << startTreeRequest.depth << std::endl; + + StartTreeResponse response; + response.depth = startTreeRequest.depth; + response.name = startTreeRequest.name; + response.reason = "Hello!"; + response.success = true; + + msgpack::sbuffer buffer; + msgpack::pack(buffer, response); + + MsgHeader header(WorldStateMsgTypes::START_TREE_RESPONSE, requestId, buffer.size()); + + outputStream.send(&header, buffer.data()); +} } // namespace bb::world_state \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp b/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp index 8ee4d7c5b58..d64b895b9c4 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp @@ -9,6 +9,7 @@ using namespace bb::world_state; int main(int, char**) { + std::cerr << "Hello from the world state" << std::endl; SynchronisedStdOutput outputStream(std::cout); WorldStateService service(outputStream); diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp b/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp index 20782cc4e02..df6b5eca24a 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp @@ -12,14 +12,14 @@ void waitForInput(std::basic_istream& inputStream, bb::messaging::StreamParser& parser) { - bool terminate = false; - while (!terminate) { - std::array data; - inputStream.getline(data.data(), 1024); - uint32_t length = static_cast(inputStream.gcount()); - terminate = parser.onNewData(data.data(), length); + std::vector buffer(1); + while (inputStream.read(&buffer[0], 1)) { + bool moreDataExpected = parser.onNewData(buffer.data(), 1); + if (!moreDataExpected) { + break; + } } -} +}; class SynchronisedStdOutput { private: @@ -37,5 +37,6 @@ class SynchronisedStdOutput { header->msgId = msgId++; const char* data = reinterpret_cast(header); stream.write(data, header->msgLength); + stream.flush(); } }; \ No newline at end of file From afda53a85a66d953c7ee962062d1e0b7820eb821 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 18 Jun 2024 12:40:19 +0100 Subject: [PATCH 20/63] WIP --- barretenberg/cpp/src/CMakeLists.txt | 1 - .../common/parallel_for_queued.cpp | 96 +-------- .../src/barretenberg/common/thread_pool.cpp | 73 +++++++ .../src/barretenberg/common/thread_pool.hpp | 33 +++ .../append_only_tree/append_only_tree.hpp | 139 ++++++------ .../append_only_tree.test.cpp | 94 ++++++--- .../crypto/merkle_tree/array_store.hpp | 15 +- .../merkle_tree/indexed_tree/indexed_leaf.hpp | 3 +- .../merkle_tree/indexed_tree/indexed_tree.hpp | 178 +++++++--------- .../indexed_tree/indexed_tree.test.cpp | 199 +++++++++++------- .../merkle_tree/indexed_tree/leaves_cache.hpp | 2 - .../nullifier_tree/nullifier_leaf.hpp | 3 +- .../nullifier_tree/nullifier_memory_tree.hpp | 3 +- .../barretenberg/crypto/merkle_tree/types.hpp | 8 + .../src/barretenberg/messaging/CMakeLists.txt | 2 - .../cpp/src/barretenberg/messaging/header.hpp | 71 ++++--- .../barretenberg/messaging/stream_parser.cpp | 65 ------ .../barretenberg/messaging/stream_parser.hpp | 64 +++++- .../barretenberg/world_state/CMakeLists.txt | 2 +- .../world_state/service/message.hpp | 50 ++++- .../service/world_state_service.hpp | 138 +++++++++--- .../world_state_service/CMakeLists.txt | 3 +- .../barretenberg/world_state_service/main.cpp | 38 +++- .../world_state_service/std_io.hpp | 49 +++-- yarn-project/merkle-tree/package.json | 1 + .../{tree_factory.ts => js_tree_factory.ts} | 0 yarn-project/merkle-tree/src/index.ts | 2 +- .../merkle-tree/src/interfaces/merkle_tree.ts | 8 + .../merkle-tree/src/native/message.ts | 69 ++++++ .../merkle-tree/src/native/native_client.ts | 126 +++++++++-- .../src/snapshots/append_only_snapshot.ts | 9 +- .../test/standard_indexed_tree_native.test.ts | 130 ++++++++++++ .../src/standard_tree/standard_tree.test.ts | 2 +- .../src/standard_tree/standard_tree_native.ts | 109 ++++++++++ .../merkle-tree/src/test/get_native_config.ts | 28 +++ yarn-project/merkle-tree/src/tree_base.ts | 6 +- yarn-project/prover-client/tsconfig.json | 2 +- yarn-project/yarn.lock | 8 + 38 files changed, 1260 insertions(+), 569 deletions(-) create mode 100644 barretenberg/cpp/src/barretenberg/common/thread_pool.cpp create mode 100644 barretenberg/cpp/src/barretenberg/common/thread_pool.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt delete mode 100644 barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp rename yarn-project/merkle-tree/src/factory/{tree_factory.ts => js_tree_factory.ts} (100%) create mode 100644 yarn-project/merkle-tree/src/native/message.ts create mode 100644 yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_native.test.ts create mode 100644 yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts create mode 100644 yarn-project/merkle-tree/src/test/get_native_config.ts diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index ab47b22e801..6ba950613b4 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -89,7 +89,6 @@ add_subdirectory(barretenberg/vm) add_subdirectory(barretenberg/wasi) add_subdirectory(barretenberg/world_state) add_subdirectory(barretenberg/world_state_service) -add_subdirectory(barretenberg/messaging) add_subdirectory(barretenberg/lmdb_store) diff --git a/barretenberg/cpp/src/barretenberg/common/parallel_for_queued.cpp b/barretenberg/cpp/src/barretenberg/common/parallel_for_queued.cpp index acaf920fd03..82a1a02d727 100644 --- a/barretenberg/cpp/src/barretenberg/common/parallel_for_queued.cpp +++ b/barretenberg/cpp/src/barretenberg/common/parallel_for_queued.cpp @@ -1,6 +1,7 @@ #ifndef NO_MULTITHREADING #include "log.hpp" #include "thread.hpp" +#include "thread_pool.hpp" #include #include #include @@ -9,101 +10,6 @@ #include #include -namespace { -class ThreadPool { - public: - ThreadPool(size_t num_threads); - ThreadPool(const ThreadPool& other) = delete; - ThreadPool(ThreadPool&& other) = delete; - ~ThreadPool(); - - ThreadPool& operator=(const ThreadPool& other) = delete; - ThreadPool& operator=(ThreadPool&& other) = delete; - - void enqueue(const std::function& task); - void wait(); - - private: - std::vector workers; - std::queue> tasks; - std::mutex tasks_mutex; - std::condition_variable condition; - std::condition_variable finished_condition; - std::atomic tasks_running; - bool stop = false; - - void worker_loop(size_t thread_index); -}; - -ThreadPool::ThreadPool(size_t num_threads) -{ - workers.reserve(num_threads); - for (size_t i = 0; i < num_threads; ++i) { - workers.emplace_back(&ThreadPool::worker_loop, this, i); - } -} - -ThreadPool::~ThreadPool() -{ - { - std::unique_lock lock(tasks_mutex); - stop = true; - } - condition.notify_all(); - for (auto& worker : workers) { - worker.join(); - } -} - -void ThreadPool::enqueue(const std::function& task) -{ - { - std::unique_lock lock(tasks_mutex); - tasks.push(task); - } - condition.notify_one(); -} - -void ThreadPool::wait() -{ - std::unique_lock lock(tasks_mutex); - finished_condition.wait(lock, [this] { return tasks.empty() && tasks_running == 0; }); -} - -void ThreadPool::worker_loop(size_t /*unused*/) -{ - // info("created worker ", worker_num); - while (true) { - std::function task; - { - std::unique_lock lock(tasks_mutex); - condition.wait(lock, [this] { return !tasks.empty() || stop; }); - - if (tasks.empty() && stop) { - break; - } - - task = tasks.front(); - tasks.pop(); - tasks_running++; - } - // info("worker ", worker_num, " processing a task!"); - task(); - // info("task done"); - { - std::unique_lock lock(tasks_mutex); - tasks_running--; - if (tasks.empty() && tasks_running == 0) { - // info("notifying main thread"); - finished_condition.notify_all(); - } - } - // info("worker ", worker_num, " done!"); - } - // info("worker exit ", worker_num); -} -} // namespace - namespace bb { /** * A thread pooled strategey that assumed that thread pools would be more efficient than spawning threads. diff --git a/barretenberg/cpp/src/barretenberg/common/thread_pool.cpp b/barretenberg/cpp/src/barretenberg/common/thread_pool.cpp new file mode 100644 index 00000000000..a046885e86b --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/common/thread_pool.cpp @@ -0,0 +1,73 @@ + +#include "thread_pool.hpp" +#include "barretenberg/common/log.hpp" +namespace bb { + +ThreadPool::ThreadPool(size_t num_threads) +{ + workers.reserve(num_threads); + for (size_t i = 0; i < num_threads; ++i) { + workers.emplace_back(&ThreadPool::worker_loop, this, i); + } +} + +ThreadPool::~ThreadPool() +{ + { + std::unique_lock lock(tasks_mutex); + stop = true; + } + condition.notify_all(); + for (auto& worker : workers) { + worker.join(); + } +} + +void ThreadPool::enqueue(const std::function& task) +{ + { + std::unique_lock lock(tasks_mutex); + tasks.push(task); + } + condition.notify_one(); +} + +void ThreadPool::wait() +{ + std::unique_lock lock(tasks_mutex); + finished_condition.wait(lock, [this] { return tasks.empty() && tasks_running == 0; }); +} + +void ThreadPool::worker_loop(size_t /*unused*/) +{ + // info("created worker ", worker_num); + while (true) { + std::function task; + { + std::unique_lock lock(tasks_mutex); + condition.wait(lock, [this] { return !tasks.empty() || stop; }); + + if (tasks.empty() && stop) { + break; + } + + task = tasks.front(); + tasks.pop(); + tasks_running++; + } + // info("worker ", worker_num, " processing a task!"); + task(); + // info("task done"); + { + std::unique_lock lock(tasks_mutex); + tasks_running--; + if (tasks.empty() && tasks_running == 0) { + // info("notifying main thread"); + finished_condition.notify_all(); + } + } + // info("worker ", worker_num, " done!"); + } + // info("worker exit ", worker_num); +} +} // namespace bb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp b/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp new file mode 100644 index 00000000000..fb6e7ca961c --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp @@ -0,0 +1,33 @@ + +#pragma once +#include +#include +#include +#include +#include +namespace bb { +class ThreadPool { + public: + ThreadPool(size_t num_threads); + ThreadPool(const ThreadPool& other) = delete; + ThreadPool(ThreadPool&& other) = delete; + ~ThreadPool(); + + ThreadPool& operator=(const ThreadPool& other) = delete; + ThreadPool& operator=(ThreadPool&& other) = delete; + + void enqueue(const std::function& task); + void wait(); + + private: + std::vector workers; + std::queue> tasks; + std::mutex tasks_mutex; + std::condition_variable condition; + std::condition_variable finished_condition; + std::atomic tasks_running; + bool stop = false; + + void worker_loop(size_t thread_index); +}; +} // namespace bb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 7170a9292ea..21df4746584 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -1,12 +1,15 @@ #pragma once #include "../hash_path.hpp" +#include "../types.hpp" +#include "barretenberg/common/thread_pool.hpp" +#include +#include +#include namespace bb::crypto::merkle_tree { using namespace bb; -typedef uint256_t index_t; - /** * @brief Implements a simple append-only merkle tree * Accepts template argument of the type of store backing the tree and the hashing policy @@ -14,20 +17,24 @@ typedef uint256_t index_t; */ template class AppendOnlyTree { public: - AppendOnlyTree(Store& store, size_t depth, uint8_t tree_id = 0); + typedef std::function append_completion_callback; + + AppendOnlyTree(Store& store, uint32_t depth, ThreadPool& workers, std::string name); AppendOnlyTree(AppendOnlyTree const& other) = delete; AppendOnlyTree(AppendOnlyTree&& other) = delete; - virtual ~AppendOnlyTree(); + AppendOnlyTree& operator=(AppendOnlyTree const& other) = delete; + AppendOnlyTree& operator=(AppendOnlyTree const&& other) = delete; + virtual ~AppendOnlyTree() = default; /** * @brief Adds a single value to the end of the tree */ - virtual fr add_value(const fr& value); + virtual void add_value(const fr& value, const append_completion_callback& on_completion); /** * @brief Adds the given set of values to the end of the tree */ - virtual fr add_values(const std::vector& values); + virtual void add_values(const std::vector& values, const append_completion_callback& on_completion); /** * @brief Returns the index of the right-most populated leaf in the tree @@ -42,7 +49,7 @@ template class AppendOnlyTree { /** * @brief Returns the depth of the tree */ - size_t depth() const; + uint32_t depth() const; /** * @brief Returns the hash path from the leaf at the given index to the root @@ -50,24 +57,29 @@ template class AppendOnlyTree { fr_hash_path get_hash_path(const index_t& index) const; protected: - fr get_element_or_zero(size_t level, const index_t& index) const; + fr get_element_or_zero(uint32_t level, const index_t& index) const; - void write_node(size_t level, const index_t& index, const fr& value); - std::pair read_node(size_t level, const index_t& index) const; + void write_node(uint32_t level, const index_t& index, const fr& value); + std::pair read_node(uint32_t level, const index_t& index) const; Store& store_; - size_t depth_; - uint8_t tree_id_; + const uint32_t depth_; + const std::string name_; std::vector zero_hashes_; fr root_; index_t size_; + ThreadPool& workers_; }; template -AppendOnlyTree::AppendOnlyTree(Store& store, size_t depth, uint8_t tree_id) +AppendOnlyTree::AppendOnlyTree(Store& store, + uint32_t depth, + ThreadPool& workers, + std::string name) : store_(store) , depth_(depth) - , tree_id_(tree_id) + , name_(std::move(name)) + , workers_(workers) { zero_hashes_.resize(depth + 1); @@ -79,10 +91,9 @@ AppendOnlyTree::AppendOnlyTree(Store& store, size_t depth, } zero_hashes_[0] = current; root_ = current; + size_ = 0; } -template AppendOnlyTree::~AppendOnlyTree() {} - template index_t AppendOnlyTree::size() const { return size_; @@ -93,7 +104,7 @@ template fr AppendOnlyTree size_t AppendOnlyTree::depth() const +template uint32_t AppendOnlyTree::depth() const { return depth_; } @@ -104,65 +115,73 @@ fr_hash_path AppendOnlyTree::get_hash_path(const index_t& fr_hash_path path; index_t current_index = index; - for (size_t level = depth_; level > 0; --level) { - bool is_right = bool(current_index & 0x01); + for (uint32_t level = depth_; level > 0; --level) { + bool is_right = static_cast(current_index & 0x01); fr right_value = is_right ? get_element_or_zero(level, current_index) : get_element_or_zero(level, current_index + 1); fr left_value = is_right ? get_element_or_zero(level, current_index - 1) : get_element_or_zero(level, current_index); - path.push_back(std::make_pair(left_value, right_value)); + path.emplace_back(left_value, right_value); current_index >>= 1; } return path; } -template fr AppendOnlyTree::add_value(const fr& value) +template +void AppendOnlyTree::add_value(const fr& value, const append_completion_callback& on_completion) { - return add_values(std::vector{ value }); + add_values(std::vector{ value }, on_completion); } template -fr AppendOnlyTree::add_values(const std::vector& values) +void AppendOnlyTree::add_values(const std::vector& values, + const append_completion_callback& on_completion) { - index_t index = size(); - size_t number_to_insert = values.size(); - size_t level = depth_; - std::vector hashes = values; - - // Add the values at the leaf nodes of the tree - for (size_t i = 0; i < number_to_insert; ++i) { - write_node(level, index + i, hashes[i]); - } + index_t start_size = size(); + uint32_t start_level = depth_; + std::shared_ptr> hashes = std::make_shared>(values); + + auto append_op = [=]() -> void { + index_t index = start_size; + uint32_t level = start_level; + uint32_t number_to_insert = static_cast(values.size()); + std::vector& hashes_local = *hashes; + // Add the values at the leaf nodes of the tree + for (uint32_t i = 0; i < number_to_insert; ++i) { + write_node(level, index + i, hashes_local[i]); + } - // Hash the values as a sub tree and insert them - while (number_to_insert > 1) { - number_to_insert >>= 1; - index >>= 1; - --level; - for (size_t i = 0; i < number_to_insert; ++i) { - hashes[i] = HashingPolicy::hash_pair(hashes[i * 2], hashes[i * 2 + 1]); - write_node(level, index + i, hashes[i]); + // Hash the values as a sub tree and insert them + while (number_to_insert > 1) { + number_to_insert >>= 1; + index >>= 1; + --level; + for (uint32_t i = 0; i < number_to_insert; ++i) { + hashes_local[i] = HashingPolicy::hash_pair(hashes_local[i * 2], hashes_local[i * 2 + 1]); + write_node(level, index + i, hashes_local[i]); + } } - } - // Hash from the root of the sub-tree to the root of the overall tree - fr new_hash = hashes[0]; - while (level > 0) { - bool is_right = bool(index & 0x01); - fr left_hash = is_right ? get_element_or_zero(level, index - 1) : new_hash; - fr right_hash = is_right ? new_hash : get_element_or_zero(level, index + 1); - new_hash = HashingPolicy::hash_pair(left_hash, right_hash); - index >>= 1; - --level; - write_node(level, index, new_hash); - } - size_ += values.size(); - root_ = new_hash; - return root_; + // Hash from the root of the sub-tree to the root of the overall tree + fr new_hash = hashes_local[0]; + while (level > 0) { + bool is_right = static_cast(index & 0x01); + fr left_hash = is_right ? get_element_or_zero(level, index - 1) : new_hash; + fr right_hash = is_right ? new_hash : get_element_or_zero(level, index + 1); + new_hash = HashingPolicy::hash_pair(left_hash, right_hash); + index >>= 1; + --level; + write_node(level, index, new_hash); + } + size_ += values.size(); + root_ = new_hash; + on_completion(root_, size_); + }; + workers_.enqueue(append_op); } template -fr AppendOnlyTree::get_element_or_zero(size_t level, const index_t& index) const +fr AppendOnlyTree::get_element_or_zero(uint32_t level, const index_t& index) const { const std::pair read_data = read_node(level, index); if (read_data.first) { @@ -173,18 +192,18 @@ fr AppendOnlyTree::get_element_or_zero(size_t level, const } template -void AppendOnlyTree::write_node(size_t level, const index_t& index, const fr& value) +void AppendOnlyTree::write_node(uint32_t level, const index_t& index, const fr& value) { std::vector buf; write(buf, value); - store_.put(level, size_t(index), buf); + store_.put(level, index, buf); } template -std::pair AppendOnlyTree::read_node(size_t level, const index_t& index) const +std::pair AppendOnlyTree::read_node(uint32_t level, const index_t& index) const { std::vector buf; - bool available = store_.get(level, size_t(index), buf); + bool available = store_.get(level, index, buf); if (!available) { return std::make_pair(false, fr::zero()); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp index e25f53ba1ca..8b872b29331 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -3,6 +3,8 @@ #include "../memory_tree.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" +#include "barretenberg/common/thread_pool.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" #include "barretenberg/numeric/random/engine.hpp" using namespace bb; @@ -13,92 +15,124 @@ auto& engine = numeric::get_debug_randomness(); auto& random_engine = numeric::get_randomness(); } // namespace -const size_t NUM_VALUES = 1024; +const uint32_t NUM_VALUES = 1024; static std::vector VALUES = []() { std::vector values(NUM_VALUES); - for (size_t i = 0; i < NUM_VALUES; ++i) { + for (uint32_t i = 0; i < NUM_VALUES; ++i) { values[i] = fr(random_engine.get_random_uint256()); } return values; }(); -inline void print_tree(const size_t depth, std::vector hashes, std::string const& msg) +inline void print_tree(const uint32_t depth, std::vector hashes, std::string const& msg) { info("\n", msg); - size_t offset = 0; - for (size_t i = 0; i < depth; i++) { + uint32_t offset = 0; + for (uint32_t i = 0; i < depth; i++) { info("i = ", i); - size_t layer_size = (1UL << (depth - i)); - for (size_t j = 0; j < layer_size; j++) { + uint32_t layer_size = (1U << (depth - i)); + for (uint32_t j = 0; j < layer_size; j++) { info("j = ", j, ": ", hashes[offset + j]); } offset += layer_size; } } -TEST(stdlib_append_only_tree, can_create) +TEST(crypto_append_only_tree, can_create) { constexpr size_t depth = 10; ArrayStore store(depth); - AppendOnlyTree tree(store, depth); + ThreadPool pool(1); + AppendOnlyTree tree(store, depth, pool, "Test Tree"); MemoryTree memdb(depth); EXPECT_EQ(tree.size(), 0); EXPECT_EQ(tree.root(), memdb.root()); } -TEST(stdlib_append_only_tree, can_add_value) +TEST(crypto_append_only_tree, can_add_value) { constexpr size_t depth = 10; ArrayStore store(depth); - AppendOnlyTree tree(store, depth); + ThreadPool pool(1); + AppendOnlyTree tree(store, depth, pool, "Test Tree"); MemoryTree memdb(depth); EXPECT_EQ(tree.size(), 0); EXPECT_EQ(tree.root(), memdb.root()); + LevelSignal signal(1); + auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + memdb.update_element(0, VALUES[0]); - tree.add_value(VALUES[0]); + tree.add_value(VALUES[0], completion); + signal.wait_for_level(0); EXPECT_EQ(tree.root(), memdb.root()); EXPECT_EQ(tree.get_hash_path(0), memdb.get_hash_path(0)); } -TEST(stdlib_append_only_tree, test_size) +TEST(crypto_append_only_tree, test_size) { constexpr size_t depth = 10; ArrayStore store(depth); - AppendOnlyTree tree(store, depth); + ThreadPool pool(1); + AppendOnlyTree tree(store, depth, pool, "Test Tree"); EXPECT_EQ(tree.size(), 0ULL); // Add a new non-zero leaf at index 0. - tree.add_value(30); - EXPECT_EQ(tree.size(), 1ULL); + { + LevelSignal signal(1); + auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + tree.add_value(30, completion); + signal.wait_for_level(0); + EXPECT_EQ(tree.size(), 1ULL); + } // Add second. - tree.add_value(10); - EXPECT_EQ(tree.size(), 2ULL); + { + LevelSignal signal(1); + auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + tree.add_value(10, completion); + signal.wait_for_level(0); + EXPECT_EQ(tree.size(), 2ULL); + } // Add third. - tree.add_value(20); - EXPECT_EQ(tree.size(), 3ULL); + { + LevelSignal signal(1); + auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + tree.add_value(20, completion); + signal.wait_for_level(0); + EXPECT_EQ(tree.size(), 3ULL); + } // Add forth but with same value. - tree.add_value(20); - EXPECT_EQ(tree.size(), 4ULL); + { + LevelSignal signal(1); + auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + tree.add_value(20, completion); + signal.wait_for_level(0); + EXPECT_EQ(tree.size(), 4ULL); + } } -TEST(stdlib_append_only_tree, can_add_multiple_values) +TEST(crypto_append_only_tree, can_add_multiple_values) { constexpr size_t depth = 10; ArrayStore store(depth); - AppendOnlyTree tree(store, depth); + ThreadPool pool(1); + AppendOnlyTree tree(store, depth, pool, "Test Tree"); MemoryTree memdb(depth); for (size_t i = 0; i < NUM_VALUES; ++i) { fr mock_root = memdb.update_element(i, VALUES[i]); - fr tree_root = tree.add_value(VALUES[i]); + LevelSignal signal(1); + auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + tree.add_value(VALUES[i], completion); + signal.wait_for_level(0); + fr tree_root = tree.root(); EXPECT_EQ(mock_root, tree_root); EXPECT_EQ(memdb.get_hash_path(0), tree.get_hash_path(0)); @@ -106,11 +140,12 @@ TEST(stdlib_append_only_tree, can_add_multiple_values) } } -TEST(stdlib_append_only_tree, can_be_filled) +TEST(crypto_append_only_tree, can_be_filled) { constexpr size_t depth = 3; ArrayStore store(depth); - AppendOnlyTree tree(store, depth); + ThreadPool pool(1); + AppendOnlyTree tree(store, depth, pool, "Test Tree"); MemoryTree memdb(depth); EXPECT_EQ(tree.size(), 0); @@ -118,7 +153,10 @@ TEST(stdlib_append_only_tree, can_be_filled) for (size_t i = 0; i < 8; i++) { memdb.update_element(i, VALUES[i]); - tree.add_value(VALUES[i]); + LevelSignal signal(1); + auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + tree.add_value(VALUES[i], completion); + signal.wait_for_level(0); } EXPECT_EQ(tree.root(), memdb.root()); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp index ade8472b1b9..cb5754947af 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp @@ -1,4 +1,5 @@ #pragma once +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" namespace bb::crypto::merkle_tree { @@ -10,19 +11,25 @@ namespace bb::crypto::merkle_tree { class ArrayStore { public: - ArrayStore(size_t levels, size_t indices = 1024) + ArrayStore(uint32_t levels, index_t indices = 1024) : map_(std::vector>>>( levels + 1, std::vector>>( indices, std::pair>(false, std::vector())))) {} - ~ArrayStore() {} + ~ArrayStore() = default; - void put(size_t level, size_t index, const std::vector& data) + ArrayStore() = delete; + ArrayStore(ArrayStore const& other) = delete; + ArrayStore(ArrayStore const&& other) = delete; + ArrayStore& operator=(ArrayStore const& other) = delete; + ArrayStore& operator=(ArrayStore const&& other) = delete; + + void put(uint32_t level, index_t index, const std::vector& data) { map_[level][index] = std::make_pair(true, data); } - bool get(size_t level, size_t index, std::vector& data) const + bool get(uint32_t level, index_t index, std::vector& data) const { const std::pair>& slot = map_[level][index]; if (slot.first) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp index bd5e9859cbe..e01622a972a 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp @@ -1,11 +1,10 @@ #pragma once +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" namespace bb::crypto::merkle_tree { -typedef uint256_t index_t; - struct indexed_leaf { fr value; index_t nextIndex; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index b7112be5447..c5715f5192c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -3,12 +3,16 @@ #include "../append_only_tree/append_only_tree.hpp" #include "../hash.hpp" #include "../hash_path.hpp" +#include "barretenberg/common/assert.hpp" +#include "barretenberg/common/thread_pool.hpp" #include "indexed_leaf.hpp" +#include +#include +#include +#include namespace bb::crypto::merkle_tree { -using index_t = uint256_t; - /** * @brief Used in parallel insertions in the the IndexedTree. Workers signal to other following workes as they move up * the level of the tree. @@ -16,7 +20,7 @@ using index_t = uint256_t; */ class LevelSignal { public: - LevelSignal(size_t initial_level) + LevelSignal(uint32_t initial_level) : signal_(initial_level){}; ~LevelSignal() = default; LevelSignal(const LevelSignal& other) @@ -31,9 +35,9 @@ class LevelSignal { * @param level The required level * */ - void wait_for_level(size_t level) + void wait_for_level(uint32_t level) { - size_t current_level = signal_.load(); + uint32_t current_level = signal_.load(); while (current_level > level) { signal_.wait(current_level); current_level = signal_.load(); @@ -45,14 +49,14 @@ class LevelSignal { * @param level The level to be signalled * */ - void signal_level(size_t level) + void signal_level(uint32_t level) { signal_.store(level); signal_.notify_all(); } private: - std::atomic signal_; + std::atomic signal_; }; /** @@ -64,17 +68,19 @@ class LevelSignal { template class IndexedTree : public AppendOnlyTree { public: - IndexedTree(Store& store, size_t depth, size_t initial_size = 1, uint8_t tree_id = 0); + typedef std::function&, fr&, index_t)> add_completion_callback; + + IndexedTree(Store& store, uint32_t depth, ThreadPool& workers, index_t initial_size, std::string name); IndexedTree(IndexedTree const& other) = delete; IndexedTree(IndexedTree&& other) = delete; - ~IndexedTree(); + ~IndexedTree() = default; /** * @brief Adds or updates a single values in the tree (updates not currently supported) * @param value The value to be added or updated * @returns The 'previous' hash paths of all updated values */ - fr_hash_path add_or_update_value(const fr& value); + void add_or_update_value(const fr& value, add_completion_callback completion); /** * @brief Adds or updates the given set of values in the tree (updates not currently supported) @@ -82,21 +88,7 @@ class IndexedTree : public AppendOnlyTree { * @param no_multithreading Performs single threaded insertion, just used whilst prototyping and benchmarking * @returns The 'previous' hash paths of all updated values */ - std::vector add_or_update_values(const std::vector& values, bool no_multithreading = false); - - /** - * @brief Adds or updates a single value without returning the previous hash path - * @param value The value to be added or updated - * @returns The new root of the tree - */ - fr add_value(const fr& value) override; - - /** - * @brief Adds or updates the given set of values without returning the previous hash paths - * @param values The values to be added or updated - * @returns The new root of the tree - */ - fr add_values(const std::vector& values) override; + void add_or_update_values(const std::vector& values, const add_completion_callback& completion); indexed_leaf get_leaf(const index_t& index); @@ -105,47 +97,52 @@ class IndexedTree : public AppendOnlyTree { using AppendOnlyTree::depth; private: + using typename AppendOnlyTree::append_completion_callback; + fr update_leaf_and_hash_to_root(const index_t& index, const indexed_leaf& leaf); fr update_leaf_and_hash_to_root(const index_t& index, const indexed_leaf& leaf, LevelSignal& leader, LevelSignal& follower, fr_hash_path& previous_hash_path); - fr append_subtree(const index_t& start_index); + void append_subtree(const index_t& start_index, append_completion_callback& completion); using AppendOnlyTree::get_element_or_zero; using AppendOnlyTree::write_node; using AppendOnlyTree::read_node; + using AppendOnlyTree::add_value; + using AppendOnlyTree::add_values; + private: using AppendOnlyTree::store_; using AppendOnlyTree::zero_hashes_; using AppendOnlyTree::depth_; - using AppendOnlyTree::tree_id_; + using AppendOnlyTree::name_; using AppendOnlyTree::root_; + using AppendOnlyTree::workers_; LeavesStore leaves_; }; template -IndexedTree::IndexedTree(Store& store, - size_t depth, - size_t initial_size, - uint8_t tree_id) - : AppendOnlyTree(store, depth, tree_id) +IndexedTree::IndexedTree( + Store& store, uint32_t depth, ThreadPool& workers, index_t initial_size, std::string name) + : AppendOnlyTree(store, depth, workers, name) { ASSERT(initial_size > 0); zero_hashes_.resize(depth + 1); // Create the zero hashes for the tree - indexed_leaf zero_leaf{ 0, 0, 0 }; - auto current = HashingPolicy::hash(zero_leaf.get_hash_inputs()); - for (size_t i = depth; i > 0; --i) { + // indexed_leaf zero_leaf{ 0, 0, 0 }; + auto current = fr::zero(); + for (uint32_t i = depth; i > 0; --i) { zero_hashes_[i] = current; current = HashingPolicy::hash_pair(current, current); } zero_hashes_[0] = current; + // Inserts the initial set of leaves as a chain in incrementing value order - for (size_t i = 0; i < initial_size; ++i) { + for (uint32_t i = 0; i < initial_size; ++i) { // Insert the zero leaf to the `leaves` and also to the tree at index 0. indexed_leaf initial_leaf = indexed_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }; leaves_.append_leaf(initial_leaf); @@ -156,12 +153,12 @@ IndexedTree::IndexedTree(Store& store, initial_size - 1, indexed_leaf{ .value = leaves_.get_leaf(initial_size - 1).value, .nextIndex = 0, .nextValue = 0 }, false); - append_subtree(0); -} -template -IndexedTree::~IndexedTree() -{} + LevelSignal signal(1); + append_completion_callback completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + append_subtree(0, completion); + signal.wait_for_level(0); +} template indexed_leaf IndexedTree::get_leaf(const index_t& index) @@ -170,27 +167,15 @@ indexed_leaf IndexedTree::get_leaf(const inde } template -fr IndexedTree::add_value(const fr& value) -{ - return add_values(std::vector{ value }); -} - -template -fr IndexedTree::add_values(const std::vector& values) +void IndexedTree::add_or_update_value(const fr& value, + add_completion_callback completion) { - add_or_update_values(values); - return root(); + add_or_update_values(std::vector{ value }, completion); } template -fr_hash_path IndexedTree::add_or_update_value(const fr& value) -{ - return add_or_update_values(std::vector{ value })[0]; -} - -template -std::vector IndexedTree::add_or_update_values( - const std::vector& values, bool no_multithreading) +void IndexedTree::add_or_update_values(const std::vector& values, + const add_completion_callback& completion) { // The first thing we do is sort the values into descending order but maintain knowledge of their orignal order struct { @@ -212,16 +197,16 @@ std::vector IndexedTree::add_or indexed_leaf low_leaf; }; - std::vector insertions(values.size()); + std::shared_ptr> insertions = + std::make_shared>(values.size()); index_t old_size = leaves_.get_size(); - for (size_t i = 0; i < values_sorted.size(); ++i) { fr value = values_sorted[i].first; index_t index_of_new_leaf = index_t(values_sorted[i].second) + old_size; // This gives us the leaf that need updating - index_t current; - bool is_already_present; + index_t current = 0; + bool is_already_present = false; std::tie(is_already_present, current) = leaves_.find_low_value(values_sorted[i].first); indexed_leaf current_leaf = leaves_.get_leaf(current); @@ -239,7 +224,7 @@ std::vector IndexedTree::add_or } // Capture the index and value of the updated 'low' leaf - leaf_insertion& insertion = insertions[i]; + leaf_insertion& insertion = (*insertions)[i]; insertion.low_leaf_index = current; insertion.low_leaf = indexed_leaf{ .value = current_leaf.value, .nextIndex = current_leaf.nextIndex, @@ -248,35 +233,30 @@ std::vector IndexedTree::add_or // We now kick off multiple workers to perform the low leaf updates // We create set of signals to coordinate the workers as the move up the tree - std::vector paths(insertions.size()); - std::vector signals; - // The first signal is set to 0. This ensure the first worker up the tree is not impeded - signals.emplace_back(size_t(0)); - // Workers will follow their leaders up the tree, being trigger by the signal in front of them - for (size_t i = 0; i < insertions.size(); ++i) { - signals.emplace_back(size_t(1 + depth_)); + std::shared_ptr> paths = std::make_shared>(insertions->size()); + std::shared_ptr> signals = std::make_shared>(); + // The first signal is set to 0. This ensures the first worker up the tree is not impeded + signals->emplace_back(0); + // Workers will follow their leaders up the tree, being triggered by the signal in front of them + for (size_t i = 0; i < insertions->size(); ++i) { + signals->emplace_back(uint32_t(1 + depth_)); } - if (no_multithreading) { - // Execute the jobs in series - for (size_t i = 0; i < insertions.size(); ++i) { - leaf_insertion& insertion = insertions[i]; - update_leaf_and_hash_to_root( - insertion.low_leaf_index, insertion.low_leaf, signals[i], signals[i + 1], paths[i]); - } - } else { - // Execute the jobs in parallel - parallel_for(insertions.size(), [&](size_t i) { - leaf_insertion& insertion = insertions[i]; + for (uint32_t i = 0; i < insertions->size(); ++i) { + auto op = [=]() { + leaf_insertion& insertion = (*insertions)[i]; update_leaf_and_hash_to_root( - insertion.low_leaf_index, insertion.low_leaf, signals[i], signals[i + 1], paths[i]); - }); + insertion.low_leaf_index, insertion.low_leaf, (*signals)[i], (*signals)[i + 1], (*paths)[i]); + if (i == insertions->size() - 1) { + // Now that we have updated all of the low leaves, we insert the new leaves as a subtree at the end + append_completion_callback append_completion = [=](fr root, index_t size) { + completion(*paths, root, size); + }; + append_subtree(old_size, append_completion); + } + }; + workers_.enqueue(op); } - - // Now that we have updated all of the low leaves, we insert the new leaves as a subtree at the end - root_ = append_subtree(old_size); - - return paths; } template @@ -302,20 +282,20 @@ fr IndexedTree::update_leaf_and_hash_to_root( // 2. Read the node and it's sibling // 3. Write the new node value index_t index = leaf_index; - size_t level = depth_; + uint32_t level = depth_; fr new_hash = HashingPolicy::hash(leaf.get_hash_inputs()); // Wait until we see that our leader has cleared 'depth_ - 1' (i.e. the level above the leaves that we are about to // write into) this ensures that our leader is not still reading the leaves - size_t leader_level = depth_ - 1; + uint32_t leader_level = depth_ - 1; leader.wait_for_level(leader_level); // Extract the value of the leaf node and it's sibling - bool is_right = bool(index & 0x01); + bool is_right = static_cast(index & 0x01); // extract the current leaf hash values for the previous hash path fr current_right_value = get_element_or_zero(level, index + (is_right ? 0 : 1)); fr current_left_value = get_element_or_zero(level, is_right ? (index - 1) : index); - previous_hash_path.push_back(std::make_pair(current_left_value, current_right_value)); + previous_hash_path.emplace_back(current_left_value, current_right_value); // Write the new leaf hash in place write_node(level, index, new_hash); @@ -326,24 +306,23 @@ fr IndexedTree::update_leaf_and_hash_to_root( if (level > 1) { // Level is > 1. Therefore we need to wait for our leader to have written to the level above meaning we can // read from it - size_t level_to_read = level - 1; + uint32_t level_to_read = level - 1; leader_level = level_to_read; - leader.wait_for_level(leader_level); // Now read the node and it's sibling index_t index_of_node_above = index >> 1; - bool node_above_is_right = bool(index_of_node_above & 0x01); + bool node_above_is_right = static_cast(index_of_node_above & 0x01); fr above_right_value = get_element_or_zero(level_to_read, index_of_node_above + (node_above_is_right ? 0 : 1)); fr above_left_value = get_element_or_zero( level_to_read, node_above_is_right ? (index_of_node_above - 1) : index_of_node_above); - previous_hash_path.push_back(std::make_pair(above_left_value, above_right_value)); + previous_hash_path.emplace_back(above_left_value, above_right_value); } // Now that we have extracted the hash path from the row above // we can compute the new hash at that level and write it - is_right = bool(index & 0x01); + is_right = static_cast(index & 0x01); fr new_right_value = is_right ? new_hash : get_element_or_zero(level, index + 1); fr new_left_value = is_right ? get_element_or_zero(level, index - 1) : new_hash; new_hash = HashingPolicy::hash_pair(new_left_value, new_right_value); @@ -364,7 +343,8 @@ fr IndexedTree::update_leaf_and_hash_to_root( } template -fr IndexedTree::append_subtree(const index_t& start_index) +void IndexedTree::append_subtree(const index_t& start_index, + append_completion_callback& completion) { index_t index = start_index; size_t number_to_insert = size_t(index_t(leaves_.get_size()) - index); @@ -375,7 +355,7 @@ fr IndexedTree::append_subtree(const index_t& hashes_to_append[i] = HashingPolicy::hash(leaves_.get_leaf(size_t(index_to_insert)).get_hash_inputs()); } - return AppendOnlyTree::add_values(hashes_to_append); + AppendOnlyTree::add_values(hashes_to_append, completion); } } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index d1dc13d95e5..32a2adfd136 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -4,6 +4,8 @@ #include "../nullifier_tree/nullifier_memory_tree.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" +#include "barretenberg/common/thread_pool.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/numeric/random/engine.hpp" #include "leaves_cache.hpp" @@ -17,84 +19,100 @@ auto& engine = numeric::get_debug_randomness(); auto& random_engine = numeric::get_randomness(); } // namespace -const size_t NUM_VALUES = 1024; +const uint32_t NUM_VALUES = 1024; static std::vector VALUES = []() { std::vector values(NUM_VALUES); - for (size_t i = 0; i < NUM_VALUES; ++i) { + for (uint32_t i = 0; i < NUM_VALUES; ++i) { values[i] = fr(random_engine.get_random_uint256()); } return values; }(); -TEST(stdlib_indexed_tree, can_create) +using CompletionCallback = IndexedTree::add_completion_callback; + +template +void add_value(IndexedTree& tree, fr value) +{ + LevelSignal signal(1); + CompletionCallback completion = [&](std::vector&, fr&, index_t) { signal.signal_level(0); }; + tree.add_or_update_value(value, completion); + signal.wait_for_level(0); +} + +TEST(crypto_indexed_tree, can_create) { + ThreadPool workers(1); ArrayStore store(10); - IndexedTree tree = IndexedTree(store, 10); + IndexedTree tree = + IndexedTree(store, 10, workers, 1, "Test Tree"); EXPECT_EQ(tree.size(), 1ULL); NullifierMemoryTree memdb(10); EXPECT_EQ(memdb.root(), tree.root()); } -TEST(stdlib_indexed_tree, test_size) +TEST(crypto_indexed_tree, test_size) { + ThreadPool workers(1); ArrayStore store(32); - auto db = IndexedTree(store, 32); + auto db = IndexedTree(store, 32, workers, 1, "Test Tree"); // We assume that the first leaf is already filled with (0, 0, 0). EXPECT_EQ(db.size(), 1ULL); // Add a new non-zero leaf at index 1. - db.add_value(30); + add_value(db, 30); EXPECT_EQ(db.size(), 2ULL); // Add third. - db.add_value(10); + add_value(db, 20); EXPECT_EQ(db.size(), 3ULL); - // Add forth. - db.add_value(20); + add_value(db, 10); EXPECT_EQ(db.size(), 4ULL); } -TEST(stdlib_indexed_tree, test_get_hash_path) +TEST(crypto_indexed_tree, test_get_hash_path) { NullifierMemoryTree memdb(10); + ThreadPool workers(1); ArrayStore store(10); - auto db = IndexedTree(store, 10); + auto db = IndexedTree(store, 10, workers, 1, "Test Tree"); EXPECT_EQ(memdb.root(), db.root()); EXPECT_EQ(memdb.get_hash_path(0), db.get_hash_path(0)); memdb.update_element(VALUES[512]); - db.add_value(VALUES[512]); + add_value(db, VALUES[512]); EXPECT_EQ(db.get_hash_path(0), memdb.get_hash_path(0)); - for (size_t i = 0; i < 512; ++i) { + for (uint32_t i = 0; i < 512; ++i) { memdb.update_element(VALUES[i]); - db.add_value(VALUES[i]); + add_value(db, VALUES[i]); } EXPECT_EQ(db.get_hash_path(512), memdb.get_hash_path(512)); } -TEST(stdlib_indexed_tree, test_batch_insert) +TEST(crypto_indexed_tree, test_batch_insert) { - const size_t batch_size = 16; - const size_t num_batches = 16; - size_t depth = 10; + const uint32_t batch_size = 16; + const uint32_t num_batches = 16; + uint32_t depth = 10; + ThreadPool workers(1); + ThreadPool multi_workers(8); NullifierMemoryTree memdb(depth, batch_size); ArrayStore store1(depth); IndexedTree tree1 = - IndexedTree(store1, depth, batch_size); + IndexedTree(store1, depth, workers, batch_size, "Test Tree 1"); ArrayStore store2(depth); IndexedTree tree2 = - IndexedTree(store2, depth, batch_size); + IndexedTree(store2, depth, workers, batch_size, "Test Tree 2"); EXPECT_EQ(memdb.root(), tree1.root()); EXPECT_EQ(tree1.root(), tree2.root()); @@ -105,16 +123,34 @@ TEST(stdlib_indexed_tree, test_batch_insert) EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); - for (size_t i = 0; i < num_batches; i++) { + for (uint32_t i = 0; i < num_batches; i++) { std::vector batch; std::vector memory_tree_hash_paths; - for (size_t j = 0; j < batch_size; j++) { + for (uint32_t j = 0; j < batch_size; j++) { batch.push_back(fr(random_engine.get_random_uint256())); fr_hash_path path = memdb.update_element(batch[j]); memory_tree_hash_paths.push_back(path); } - std::vector tree1_hash_paths = tree1.add_or_update_values(batch, true); - std::vector tree2_hash_paths = tree2.add_or_update_values(batch); + std::vector tree1_hash_paths; + std::vector tree2_hash_paths; + { + LevelSignal signal(1); + CompletionCallback completion = [&](std::vector& hash_paths, fr&, index_t) { + tree1_hash_paths = hash_paths; + signal.signal_level(0); + }; + tree1.add_or_update_values(batch, completion); + signal.wait_for_level(0); + } + { + LevelSignal signal(1); + CompletionCallback completion = [&](std::vector& hash_paths, fr&, index_t) { + tree2_hash_paths = hash_paths; + signal.signal_level(0); + }; + tree2.add_or_update_values(batch, completion); + signal.wait_for_level(0); + } EXPECT_EQ(memdb.root(), tree1.root()); EXPECT_EQ(tree1.root(), tree2.root()); @@ -124,7 +160,7 @@ TEST(stdlib_indexed_tree, test_batch_insert) EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); - for (size_t j = 0; j < batch_size; j++) { + for (uint32_t j = 0; j < batch_size; j++) { EXPECT_EQ(tree1_hash_paths[j], tree2_hash_paths[j]); // EXPECT_EQ(tree1_hash_paths[j], memory_tree_hash_paths[j]); } @@ -136,12 +172,12 @@ fr hash_leaf(const indexed_leaf& leaf) return HashPolicy::hash(leaf.get_hash_inputs()); } -bool check_hash_path(const fr& root, const fr_hash_path& path, const indexed_leaf& leaf_value, const size_t idx) +bool check_hash_path(const fr& root, const fr_hash_path& path, const indexed_leaf& leaf_value, const uint32_t idx) { auto current = hash_leaf(leaf_value); - size_t depth_ = path.size(); - size_t index = idx; - for (size_t i = 0; i < depth_; ++i) { + uint32_t depth_ = static_cast(path.size()); + uint32_t index = idx; + for (uint32_t i = 0; i < depth_; ++i) { fr left = (index & 1) ? path[i].first : current; fr right = (index & 1) ? current : path[i].second; current = HashPolicy::hash_pair(left, right); @@ -150,13 +186,14 @@ bool check_hash_path(const fr& root, const fr_hash_path& path, const indexed_lea return current == root; } -TEST(stdlib_indexed_tree, test_indexed_memory) +TEST(crypto_indexed_tree, test_indexed_memory) { + ThreadPool workers(8); // Create a depth-3 indexed merkle tree - constexpr size_t depth = 3; + constexpr uint32_t depth = 3; ArrayStore store(depth); IndexedTree tree = - IndexedTree(store, depth, 1); + IndexedTree(store, depth, workers, 1, "Test Tree 1"); /** * Intial state: @@ -180,7 +217,7 @@ TEST(stdlib_indexed_tree, test_indexed_memory) * nextIdx 1 0 0 0 0 0 0 0 * nextVal 30 0 0 0 0 0 0 0 */ - tree.add_value(30); + add_value(tree, 30); EXPECT_EQ(tree.size(), 2); EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 1, 30 })); EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); @@ -194,7 +231,7 @@ TEST(stdlib_indexed_tree, test_indexed_memory) * nextIdx 2 0 1 0 0 0 0 0 * nextVal 10 0 30 0 0 0 0 0 */ - tree.add_value(10); + add_value(tree, 10); EXPECT_EQ(tree.size(), 3); EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); @@ -209,7 +246,7 @@ TEST(stdlib_indexed_tree, test_indexed_memory) * nextIdx 2 0 3 1 0 0 0 0 * nextVal 10 0 20 30 0 0 0 0 */ - tree.add_value(20); + add_value(tree, 20); EXPECT_EQ(tree.size(), 4); EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); @@ -233,7 +270,7 @@ TEST(stdlib_indexed_tree, test_indexed_memory) * nextIdx 2 4 3 1 0 0 0 0 * nextVal 10 50 20 30 0 0 0 0 */ - tree.add_value(50); + add_value(tree, 50); EXPECT_EQ(tree.size(), 5); EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 4, 50 })); @@ -247,9 +284,9 @@ TEST(stdlib_indexed_tree, test_indexed_memory) auto e010 = hash_leaf(tree.get_leaf(2)); auto e011 = hash_leaf(tree.get_leaf(3)); auto e100 = hash_leaf(tree.get_leaf(4)); - auto e101 = hash_leaf({ 0, 0, 0 }); - auto e110 = hash_leaf({ 0, 0, 0 }); - auto e111 = hash_leaf({ 0, 0, 0 }); + auto e101 = fr::zero(); + auto e110 = fr::zero(); + auto e111 = fr::zero(); auto e00 = HashPolicy::hash_pair(e000, e001); auto e01 = HashPolicy::hash_pair(e010, e011); @@ -288,44 +325,44 @@ TEST(stdlib_indexed_tree, test_indexed_memory) EXPECT_EQ(tree.get_hash_path(7), expected); } -TEST(stdlib_indexed_tree, test_indexed_tree) -{ - // Create a depth-8 indexed merkle tree - constexpr size_t depth = 8; - ArrayStore store(depth); - IndexedTree tree = - IndexedTree(store, depth, 1); - - indexed_leaf zero_leaf = { 0, 0, 0 }; - EXPECT_EQ(tree.size(), 1); - EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf(zero_leaf)); - - // Add 20 random values to the tree - for (size_t i = 0; i < 20; i++) { - auto value = fr::random_element(); - tree.add_value(value); - } - - auto abs_diff = [](uint256_t a, uint256_t b) { - if (a > b) { - return (a - b); - } else { - return (b - a); - } - }; - - // Check if a new random value is not a member of this tree. - fr new_member = fr::random_element(); - std::vector differences; - for (size_t i = 0; i < size_t(tree.size()); i++) { - uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); - uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); - differences.push_back(diff_hi + diff_lo); - } - auto it = std::min_element(differences.begin(), differences.end()); - auto index = static_cast(it - differences.begin()); - - // Merkle proof at `index` proves non-membership of `new_member` - auto hash_path = tree.get_hash_path(index); - EXPECT_TRUE(check_hash_path(tree.root(), hash_path, tree.get_leaf(index), index)); -} \ No newline at end of file +// TEST(crypto_indexed_tree, test_indexed_tree) +// { +// // Create a depth-8 indexed merkle tree +// constexpr uint32_t depth = 8; +// ArrayStore store(depth); +// IndexedTree tree = +// IndexedTree(store, depth, 1); + +// indexed_leaf zero_leaf = { 0, 0, 0 }; +// EXPECT_EQ(tree.size(), 1); +// EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf(zero_leaf)); + +// // Add 20 random values to the tree +// for (uint32_t i = 0; i < 20; i++) { +// auto value = fr::random_element(); +// tree.add_value(value); +// } + +// auto abs_diff = [](uint256_t a, uint256_t b) { +// if (a > b) { +// return (a - b); +// } else { +// return (b - a); +// } +// }; + +// // Check if a new random value is not a member of this tree. +// fr new_member = fr::random_element(); +// std::vector differences; +// for (uint32_t i = 0; i < uint32_t(tree.size()); i++) { +// uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); +// uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); +// differences.push_back(diff_hi + diff_lo); +// } +// auto it = std::min_element(differences.begin(), differences.end()); +// auto index = static_cast(it - differences.begin()); + +// // Merkle proof at `index` proves non-membership of `new_member` +// auto hash_path = tree.get_hash_path(index); +// EXPECT_TRUE(check_hash_path(tree.root(), hash_path, tree.get_leaf(index), index)); +// } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp index b38f556807e..9bf01c4015b 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp @@ -4,8 +4,6 @@ namespace bb::crypto::merkle_tree { -typedef uint256_t index_t; - /** * @brief Used to facilitate testing of the IndexedTree. Stores leaves in memory with an index for O(logN) retrieval of * 'low leaves' diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp index 1ac402ac310..e85e3d2af4b 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp @@ -1,12 +1,11 @@ #pragma once +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/stdlib/hash/poseidon2/poseidon2.hpp" namespace bb::crypto::merkle_tree { -typedef uint256_t index_t; - struct nullifier_leaf { fr value; index_t nextIndex; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp index 5b9fdb94ee2..550f4132663 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -99,7 +99,8 @@ NullifierMemoryTree::NullifierMemoryTree(size_t depth, size_t ini hashes_.resize(total_size_ * 2 - 2); // Build the entire tree and fill with 0 hashes. - auto current = WrappedNullifierLeaf(nullifier_leaf::zero()).hash(); + // auto current = WrappedNullifierLeaf(nullifier_leaf::zero()).hash(); + auto current = fr::zero(); size_t layer_size = total_size_; for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { for (size_t i = 0; i < layer_size; ++i) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp new file mode 100644 index 00000000000..0c93cfd2954 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -0,0 +1,8 @@ +#pragma once + +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include +#include +namespace bb::crypto::merkle_tree { +using index_t = uint64_t; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt deleted file mode 100644 index cf4d754e97b..00000000000 --- a/barretenberg/cpp/src/barretenberg/messaging/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -barretenberg_module(messaging) - diff --git a/barretenberg/cpp/src/barretenberg/messaging/header.hpp b/barretenberg/cpp/src/barretenberg/messaging/header.hpp index 028a1b57ea5..62bb3d9c5a1 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/header.hpp +++ b/barretenberg/cpp/src/barretenberg/messaging/header.hpp @@ -1,39 +1,62 @@ #pragma once +#include "barretenberg/serialize/msgpack.hpp" #include #include namespace bb::messaging { -const uint32_t MAGIC_STRING_LENGTH = 8; -const uint8_t MAGIC_STRING[] = "AZTEC!!!"; // The null terminator is not included enum SystemMsgTypes { TERMINATE = 0, PING = 1, PONG = 2 }; const uint32_t FIRST_APP_MSG_TYPE = 100; -#pragma pack(push, 1) +// #pragma pack(push, 1) struct MsgHeader { - uint8_t msgStart[MAGIC_STRING_LENGTH]; // MAGIC_STRING - uint32_t msgLength; // Length including the header - uint32_t msgType; // The type of message - uint32_t msgId; // Unique Id for the message - uint32_t requestId; // Id of the message this is responding too (may not be used) - - MsgHeader(uint32_t type, uint32_t reqId, uint32_t lengthWithoutHeader = 0) - : msgLength(sizeof(MsgHeader) + lengthWithoutHeader) - , msgType(type) + uint32_t messageId; // Unique Id for the message + uint32_t requestId; // Id of the message this is responding too (may not be used) + + MSGPACK_FIELDS(messageId, requestId); + + MsgHeader() = default; + + MsgHeader(uint32_t reqId) + : requestId(reqId) + {} + + MsgHeader(uint32_t msgId, uint32_t reqId) + : messageId(msgId) , requestId(reqId) - { - std::memcpy(msgStart, MAGIC_STRING, MAGIC_STRING_LENGTH); - } - - // MsgHeader(uint32_t type, uint32_t reqId, uint32_t msgId, uint32_t totalLength) - // : MsgHeader(type, reqId) - // { - // this->msgId = msgId; - // this->msgLength = totalLength; - // } + {} +}; + +struct HeaderOnlyMessage { + uint32_t msgType; + MsgHeader header; + + HeaderOnlyMessage(uint32_t type, MsgHeader& hdr) + : msgType(type) + , header(hdr) + {} + + HeaderOnlyMessage() = default; + + MSGPACK_FIELDS(msgType, header); +}; + +template struct TypedMessage { + uint32_t msgType; + MsgHeader header; + T value; + + TypedMessage(uint32_t type, MsgHeader& hdr, const T& val) + : msgType(type) + , header(hdr) + , value(val) + {} + + TypedMessage() = default; + + MSGPACK_FIELDS(msgType, header, value); }; -#pragma pack(pop) -const uint32_t HEADER_SIZE = sizeof(MsgHeader); +// #pragma pack(pop) } // namespace bb::messaging diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp deleted file mode 100644 index ab595ce3709..00000000000 --- a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include "stream_parser.hpp" -#include "barretenberg/messaging/header.hpp" -#include -#include -#include - -namespace bb::messaging { -bool StreamParser::onNewData(char* data, uint32_t length) -{ - uint32_t newSize = bufferLength + length; - if (newSize > buffer.size()) { - buffer.resize(buffer.size() + length); - } - std::memcpy(&buffer[bufferLength], data, length); - bufferLength += length; - - while (true) { - uint32_t dataRemaining = bufferLength - readPointer; - if (dataRemaining < MAGIC_STRING_LENGTH) { - // not even enough to query the magic string - shrinkBuffer(); - break; - } - if (std::memcmp(&buffer[readPointer], MAGIC_STRING, MAGIC_STRING_LENGTH) != 0) { - ++readPointer; - continue; - } - // We have found the magic string, now see if there is a full header - if (dataRemaining < HEADER_SIZE) { - // Less than header bytes available, wait for more data - shrinkBuffer(); - break; - } - // We have a full header at least - const char* messageStart = &buffer[readPointer]; - const MsgHeader* header = reinterpret_cast(messageStart); - // Now see if we have a full message - if (header->msgLength > dataRemaining) { - // We don't, wait for more data - shrinkBuffer(); - break; - } - - // We have a full message! - readPointer += header->msgLength; - bool continueStream = messageHandler(header, messageStart + HEADER_SIZE); - // If we are told the stream has ended, propagate that information up - if (!continueStream) { - return false; - } - } - return true; -} - -void StreamParser::shrinkBuffer() -{ - uint32_t dataRemaining = bufferLength - readPointer; - if (dataRemaining > 0 && readPointer > 0) { - // use memove as ranges could overlap - std::memmove(&buffer[0], &buffer[readPointer], dataRemaining); - } - readPointer = 0; - bufferLength = dataRemaining; -} -} // namespace bb::messaging diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp index 5ed3443868c..9f5006dced1 100644 --- a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp +++ b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.hpp @@ -1,25 +1,69 @@ #pragma once #include "barretenberg/messaging/header.hpp" +#include "barretenberg/serialize/cbind.hpp" +#include "msgpack/v3/object_fwd_decl.hpp" #include #include +#include #include #include namespace bb::messaging { -class StreamParser { +template class StreamDispatcher { private: - std::vector buffer; - uint32_t bufferLength = 0; - uint32_t readPointer = 0; - std::function messageHandler; - - void shrinkBuffer(); + OutputStream& outputStream; + std::unordered_map> messageHandlers; + void sendPong(uint32_t pingId); + bool handleSystemMessage(msgpack::object& obj); public: - StreamParser(std::function handler) - : messageHandler(std::move(handler)) + StreamDispatcher(OutputStream& out) + : outputStream(out) {} - bool onNewData(char* data, uint32_t length); + bool onNewData(msgpack::object& obj); + void registerTarget(uint32_t msgType, std::function& handler); }; + +template bool StreamDispatcher::onNewData(msgpack::object& obj) +{ + bb::messaging::HeaderOnlyMessage header; + obj.convert(header); + + if (header.msgType < FIRST_APP_MSG_TYPE) { + return handleSystemMessage(obj); + } + auto iter = messageHandlers.find(header.msgType); + if (iter == messageHandlers.end()) { + std::cerr << "No registered handler for message of type " << header.msgType << std::endl; + return true; + } + return (iter->second)(obj); +} + +template +void StreamDispatcher::registerTarget(uint32_t msgType, std::function& handler) +{ + messageHandlers.insert({ msgType, handler }); +} + +template bool StreamDispatcher::handleSystemMessage(msgpack::object& obj) +{ + bb::messaging::HeaderOnlyMessage header; + obj.convert(header); + if (header.msgType == SystemMsgTypes::TERMINATE) { + return false; + } + if (header.msgType == SystemMsgTypes::PING) { + sendPong(header.header.messageId); + } + return true; +} + +template void StreamDispatcher::sendPong(uint32_t pingId) +{ + MsgHeader header(pingId); + HeaderOnlyMessage packedHeader(SystemMsgTypes::PONG, header); + outputStream.send(packedHeader); +} } // namespace bb::messaging diff --git a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt index 8e659628ed5..27372850e8c 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt @@ -1,3 +1,3 @@ -barretenberg_module(world_state lmdb) +barretenberg_module(world_state lmdb crypto_merkle_tree cryto_pedersen_hash) diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/message.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/message.hpp index 6b2da95df0a..4ae6bad5f5d 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/service/message.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/service/message.hpp @@ -1,32 +1,74 @@ #pragma once +#include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/serialize/msgpack_impl/struct_map_impl.hpp" +#include #include namespace bb::world_state { using namespace bb::messaging; + enum WorldStateMsgTypes { START_TREE_REQUEST = FIRST_APP_MSG_TYPE, START_TREE_RESPONSE, GET_TREE_INFO_REQUEST, - GET_TREE_INFO_RESPONSE + GET_TREE_INFO_RESPONSE, + INSERT_LEAVES_REQUEST, + INSERT_LEAVES_RESPONSE, }; struct StartTreeRequest { std::string name; uint32_t depth; + uint32_t preFilledSize; - MSGPACK_FIELDS(name, depth); + MSGPACK_FIELDS(name, depth, preFilledSize); }; struct StartTreeResponse { std::string name; uint32_t depth; bool success; - std::string reason; + std::string message; + + MSGPACK_FIELDS(name, depth, success, message); +}; + +struct GetTreeInfoRequest { + std::string name; + + MSGPACK_FIELDS(name); +}; + +struct GetTreeInfoResponse { + std::string name; + uint32_t depth; + bb::fr root; + uint64_t size; + bool success; + std::string message; + + MSGPACK_FIELDS(name, depth, root, size, success, message); +}; + +struct InsertLeavesRequest { + std::string name; + std::vector leaves; + + MSGPACK_FIELDS(name, leaves); +}; + +struct InsertLeavesResponse { + bb::fr root; + uint64_t size; + bool success; + std::string message; - MSGPACK_FIELDS(name, depth, success, reason); + MSGPACK_FIELDS(root, size, success, message); }; } // namespace bb::world_state + +MSGPACK_ADD_ENUM(bb::world_state::WorldStateMsgTypes) diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp index 5795c34e01c..3a5e3901a69 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp @@ -1,4 +1,11 @@ #pragma once +#include "barretenberg/common/thread_pool.hpp" +#include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp" +#include "barretenberg/crypto/merkle_tree/array_store.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/cbind.hpp" #include "barretenberg/world_state/service/message.hpp" @@ -6,63 +13,130 @@ #include #include #include +#include #include +namespace bb::world_state { using namespace bb::messaging; +using namespace bb::crypto::merkle_tree; +using HashPolicy = PedersenHashPolicy; -namespace bb::world_state { template class WorldStateService { private: OutputStream& outputStream; - std::unordered_map> messagesHandlers; - void sendPong(uint32_t pingId); - - void startTree(uint32_t requestId, const char* data, uint32_t length); + std::unordered_map>> trees; + std::vector> stores; + ThreadPool workers; public: - WorldStateService(OutputStream& out) + WorldStateService(OutputStream& out, size_t numThreads) : outputStream(out) + , workers(numThreads) {} - bool processMessage(const MsgHeader* header, const char* data); + bool startTree(msgpack::object& obj); + bool getTreeInfo(msgpack::object& obj); + bool insertLeaves(msgpack::object& obj); }; -template -bool WorldStateService::processMessage(const MsgHeader* header, const char*) +template bool WorldStateService::startTree(msgpack::object& obj) { - if (header->msgType == SystemMsgTypes::TERMINATE) { - std::cerr << "Terminating!!" << std::endl; - return false; - } - if (header->msgType == SystemMsgTypes::PING) { - sendPong(header->msgId); + TypedMessage startTreeRequest; + obj.convert(startTreeRequest); + + uint32_t actualDepth = startTreeRequest.value.depth; + + auto it = trees.find(startTreeRequest.value.name); + if (it == trees.end()) { + auto store = std::make_shared(startTreeRequest.value.depth, 1024 * 16); + stores.push_back(store); + auto tree = + std::make_shared>(*store, + startTreeRequest.value.depth, + workers, + startTreeRequest.value.preFilledSize, + startTreeRequest.value.name); + std::unordered_map>>::value_type + v = { startTreeRequest.value.name, tree }; + trees.insert(v); + } else { + actualDepth = static_cast(it->second->depth()); } + + StartTreeResponse response; + response.depth = actualDepth; + response.name = startTreeRequest.value.name; + response.success = it == trees.end(); + response.message = response.success ? "" : "Tree already exists"; + + MsgHeader header(startTreeRequest.header.messageId); + TypedMessage startTreeResponse(WorldStateMsgTypes::START_TREE_RESPONSE, header, response); + outputStream.sendPackedObject(startTreeResponse); return true; } -template void WorldStateService::sendPong(uint32_t pingId) +template bool WorldStateService::getTreeInfo(msgpack::object& obj) { - MsgHeader header(SystemMsgTypes::PONG, pingId); - outputStream.send(&header); + TypedMessage treeInfoRequest; + obj.convert(treeInfoRequest); + + GetTreeInfoResponse treeInfoResponse; + treeInfoResponse.name = treeInfoRequest.value.name; + + auto it = trees.find(treeInfoRequest.value.name); + if (it == trees.end()) { + treeInfoResponse.success = false; + treeInfoResponse.message = "Tree not found"; + } else { + treeInfoResponse.message = ""; + treeInfoResponse.success = true; + treeInfoResponse.depth = static_cast(it->second->depth()); + treeInfoResponse.root = it->second->root(); + treeInfoResponse.size = static_cast(it->second->size()); + } + + MsgHeader header(treeInfoRequest.header.messageId); + TypedMessage getTreeInfoResponse( + WorldStateMsgTypes::GET_TREE_INFO_RESPONSE, header, treeInfoResponse); + outputStream.sendPackedObject(getTreeInfoResponse); + return true; } -template -void WorldStateService::startTree(uint32_t requestId, const char* data, uint32_t length) +template bool WorldStateService::insertLeaves(msgpack::object& obj) { - StartTreeRequest startTreeRequest; - msgpack::unpack(data, length).get().convert(startTreeRequest); - std::cout << "Received Start Tree Request " << startTreeRequest.name << " " << startTreeRequest.depth << std::endl; + TypedMessage insertLeavesRequest; + obj.convert(insertLeavesRequest); - StartTreeResponse response; - response.depth = startTreeRequest.depth; - response.name = startTreeRequest.name; - response.reason = "Hello!"; - response.success = true; + uint32_t requestId = insertLeavesRequest.header.messageId; - msgpack::sbuffer buffer; - msgpack::pack(buffer, response); + auto it = trees.find(insertLeavesRequest.value.name); + if (it == trees.end()) { + InsertLeavesResponse response; + response.success = false; + response.message = "Tree not found"; - MsgHeader header(WorldStateMsgTypes::START_TREE_RESPONSE, requestId, buffer.size()); + // Send the response immediately + MsgHeader header(requestId); + TypedMessage insertLeavesResponse( + WorldStateMsgTypes::INSERT_LEAVES_RESPONSE, header, response); + outputStream.sendPackedObject(insertLeavesResponse); - outputStream.send(&header, buffer.data()); + } else { + + // Send the response async + auto completion = [=](std::vector&, fr& root, index_t size) -> void { + MsgHeader header(requestId); + InsertLeavesResponse response; + response.message = ""; + response.success = true; + response.root = root; + response.size = size; + TypedMessage insertLeavesResponse( + WorldStateMsgTypes::INSERT_LEAVES_RESPONSE, header, response); + outputStream.sendPackedObject(insertLeavesResponse); + }; + it->second->add_or_update_values(insertLeavesRequest.value.leaves, completion); + } + + return true; } } // namespace bb::world_state \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt index 2dbeaa768c7..7df0ff6f290 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt @@ -1,6 +1,5 @@ if (NOT(FUZZING)) add_executable(world_state main.cpp) - - target_link_libraries(world_state PRIVATE messaging) + target_link_libraries(world_state lmdb crypto_merkle_tree crypto_pedersen_hash common) endif() diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp b/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp index d64b895b9c4..2eeaffa83cd 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp @@ -1,21 +1,45 @@ #include "barretenberg/messaging/header.hpp" #include "barretenberg/messaging/stream_parser.hpp" +#include "barretenberg/world_state/service/message.hpp" #include "barretenberg/world_state/service/world_state_service.hpp" #include "std_io.hpp" +#include #include #include using namespace bb::world_state; +template void waitForStream(std::basic_istream& inputStream, StreamDispatcher& dispatcher) +{ + msgpack::unpacker unp; + bool moreDataExpected = true; + while (moreDataExpected) { + unp.reserve_buffer(1); + inputStream.read(unp.buffer(), 1); + unp.buffer_consumed(1); + + msgpack::object_handle result; + while (moreDataExpected && unp.next(result)) { + msgpack::object obj(result.get()); + moreDataExpected = dispatcher.onNewData(obj); + } + } +} + int main(int, char**) { - std::cerr << "Hello from the world state" << std::endl; SynchronisedStdOutput outputStream(std::cout); - WorldStateService service(outputStream); + WorldStateService service(outputStream, 16); + StreamDispatcher dispatcher(outputStream); + + std::function f1 = [&service](msgpack::object& obj) { return service.startTree(obj); }; + dispatcher.registerTarget(WorldStateMsgTypes::START_TREE_REQUEST, f1); + + std::function f2 = [&service](msgpack::object& obj) { return service.getTreeInfo(obj); }; + dispatcher.registerTarget(WorldStateMsgTypes::GET_TREE_INFO_REQUEST, f2); + + std::function f3 = [&service](msgpack::object& obj) { return service.insertLeaves(obj); }; + dispatcher.registerTarget(WorldStateMsgTypes::INSERT_LEAVES_REQUEST, f3); - std::function handler = [&service](const MsgHeader* header, const char* data) { - return service.processMessage(header, data); - }; - StreamParser parser(handler); - waitForInput(std::cin, parser); + waitForStream(std::cin, dispatcher); } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp b/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp index df6b5eca24a..239e0c6f824 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp @@ -1,6 +1,11 @@ #pragma once +#include "barretenberg/messaging/header.hpp" #include "barretenberg/messaging/stream_parser.hpp" +#include "barretenberg/serialize/cbind.hpp" +#include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/world_state/service/message.hpp" + #include #include #include @@ -10,16 +15,20 @@ #include #include -void waitForInput(std::basic_istream& inputStream, bb::messaging::StreamParser& parser) -{ - std::vector buffer(1); - while (inputStream.read(&buffer[0], 1)) { - bool moreDataExpected = parser.onNewData(buffer.data(), 1); - if (!moreDataExpected) { - break; - } - } -}; +// struct MsgPackedHeader { +// uint32_t msgType; +// msgpack::object header; + +// MSGPACK_FIELDS(msgType, header); +// }; + +// struct MsgPackedType { +// uint32_t msgType; +// msgpack::object header; +// msgpack::object value; + +// MSGPACK_FIELDS(msgType, header, value); +// }; class SynchronisedStdOutput { private: @@ -31,12 +40,22 @@ class SynchronisedStdOutput { SynchronisedStdOutput(std::basic_ostream& str) : stream(str) {} - void send(bb::messaging::MsgHeader* header) + void send(bb::messaging::HeaderOnlyMessage& header) { std::unique_lock lock(mutex); - header->msgId = msgId++; - const char* data = reinterpret_cast(header); - stream.write(data, header->msgLength); + header.header.messageId = msgId++; + msgpack::sbuffer buffer; + msgpack::pack(buffer, header); + stream.write(buffer.data(), static_cast(buffer.size())); stream.flush(); } -}; \ No newline at end of file + template void sendPackedObject(bb::messaging::TypedMessage& header) + { + std::unique_lock lock(mutex); + header.header.messageId = msgId++; + msgpack::sbuffer buffer; + msgpack::pack(buffer, header); + stream.write(buffer.data(), static_cast(buffer.size())); + stream.flush(); + } +}; diff --git a/yarn-project/merkle-tree/package.json b/yarn-project/merkle-tree/package.json index 8d19e74c5a5..bf30b5e10e4 100644 --- a/yarn-project/merkle-tree/package.json +++ b/yarn-project/merkle-tree/package.json @@ -51,6 +51,7 @@ "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", "@aztec/types": "workspace:^", + "@msgpack/msgpack": "^3.0.0-beta2", "sha256": "^0.2.0", "tslib": "^2.4.0" }, diff --git a/yarn-project/merkle-tree/src/factory/tree_factory.ts b/yarn-project/merkle-tree/src/factory/js_tree_factory.ts similarity index 100% rename from yarn-project/merkle-tree/src/factory/tree_factory.ts rename to yarn-project/merkle-tree/src/factory/js_tree_factory.ts diff --git a/yarn-project/merkle-tree/src/index.ts b/yarn-project/merkle-tree/src/index.ts index 89b52580867..9ac6ec71186 100644 --- a/yarn-project/merkle-tree/src/index.ts +++ b/yarn-project/merkle-tree/src/index.ts @@ -10,7 +10,7 @@ export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tr export { StandardIndexedTreeWithAppend } from './standard_indexed_tree/test/standard_indexed_tree_with_append.js'; export * from './standard_tree/standard_tree.js'; export { INITIAL_LEAF, getTreeMeta } from './tree_base.js'; -export * from './factory/tree_factory.js'; +export * from './factory/js_tree_factory.js'; export * from './snapshots/snapshot_builder.js'; export * from './snapshots/full_snapshot.js'; export * from './snapshots/append_only_snapshot.js'; diff --git a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts index 44fd8c4d4d7..8a755a65b65 100644 --- a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts @@ -68,3 +68,11 @@ export interface MerkleTree extends SiblingPathSo */ findLeafIndexAfter(leaf: T, startIndex: bigint, includeUncommitted: boolean): bigint | undefined; } + +export interface SnapshottedMerkleTree extends MerkleTree { + getName(): string; + + getNode(level: number, index: bigint): Promise; + + getZeroHash(level: number): Buffer; +} diff --git a/yarn-project/merkle-tree/src/native/message.ts b/yarn-project/merkle-tree/src/native/message.ts new file mode 100644 index 00000000000..e59315db75c --- /dev/null +++ b/yarn-project/merkle-tree/src/native/message.ts @@ -0,0 +1,69 @@ +import { Fr } from "@aztec/circuits.js"; + +export enum SystemMsgTypes { TERMINATE = 0, PING = 1, PONG = 2 }; + +const FIRST_APP_MSG_TYPE = 100; + +export class MsgHeader { + constructor( + public messageId: number, + public requestId: number, + ) { + + } +} + +export enum WorldStateMsgTypes { + START_TREE_REQUEST = FIRST_APP_MSG_TYPE, + START_TREE_RESPONSE, + GET_TREE_INFO_REQUEST, + GET_TREE_INFO_RESPONSE, + INSERT_LEAVES_REQUEST, + INSERT_LEAVES_RESPONSE, +}; + +export type StartTreeRequest = { + name: string; + depth: number; + preFilledSize: number; +}; + +export type StartTreeResponse = { + name: string; + depth: number; + success: boolean; + message: string; +}; + +export type GetTreeInfoRequest = { + name: string; +} + +export type GetTreeInfoResponse = { + name: string; + depth: number; + success: boolean; + message: string; + root: Buffer; + size: bigint; +} + +export type InsertLeavesRequest = { + name: string; + leaves: Buffer[]; +} + +export type InsertLeavesResponse = { + success: boolean; + message: string; + root: Buffer; + size: bigint; +} + +export type MsgPackedType = { + msgType: number; + header: MsgHeader; + value: T; +} + +export type MsgPackedHeader = MsgPackedType<{}>; \ No newline at end of file diff --git a/yarn-project/merkle-tree/src/native/native_client.ts b/yarn-project/merkle-tree/src/native/native_client.ts index 5f2b4568328..049bd921b2d 100644 --- a/yarn-project/merkle-tree/src/native/native_client.ts +++ b/yarn-project/merkle-tree/src/native/native_client.ts @@ -1,21 +1,105 @@ -// import { createDebugLogger } from "@aztec/foundation/log"; -// import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; - -// export type NativeWorldStateConfig = { -// worldStateBinaryPath: string; -// } - -// class NativeTrees { -// constructor(private childProcess: ChildProcessWithoutNullStreams) { - -// } -// public static init(config: NativeWorldStateConfig, logger = createDebugLogger('aztec:native_client')) { - -// const child = spawn(config.worldStateBinaryPath); -// console.log(`${new Date()} : CHILD STARTED`); -// child.stdout.on("data", (d) => console.log(`${new Date()} : STDOUT => ${d}`)); -// child.stderr.on("data", (d) => console.log(`${new Date()} : STDERR => ${d}`)); -// child.on("close", () => console.log(`${new Date()} : CHILD ENDED`)); -// return new NativeTrees(child); -// } -// } +import { DebugLogger, createDebugLogger } from "@aztec/foundation/log"; +import { RunningPromise, promiseWithResolvers } from "@aztec/foundation/promise"; +import { decodeMultiStream, encode } from "@msgpack/msgpack"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { GetTreeInfoRequest, GetTreeInfoResponse, InsertLeavesRequest, InsertLeavesResponse, MsgHeader, MsgPackedHeader, MsgPackedType, StartTreeRequest, StartTreeResponse, SystemMsgTypes, WorldStateMsgTypes } from './message.js'; +import { Fr } from "@aztec/circuits.js"; + +export type NativeWorldStateConfig = { + worldStateBinaryPath: string; +} + +type Request = { + requestId: number; + responseHandler: (responseData: object) => void; +} + +export class NativeTreesClient { + private msgId = 1; + private requests = new Map(); + private terminateResolve?: (data: Buffer) => void; + private runningPromise: RunningPromise; + + constructor(private childProcess: ChildProcessWithoutNullStreams, private logger: DebugLogger) { + this.runningPromise = new RunningPromise(() => this.handleData(), 10); + this.runningPromise.start(); + childProcess.stderr.on("data", (d) => this.logger.debug(`World State Out: ${d}`)); + childProcess.on("close", () => this.onRemoteClose()); + } + public static init(config: NativeWorldStateConfig, logger = createDebugLogger('aztec:native_client')) { + logger.debug(`Starting world state at: ${config.worldStateBinaryPath}`); + const child = spawn(config.worldStateBinaryPath, [], { stdio: ['pipe', 'pipe', 'pipe']}); + return new NativeTreesClient(child, logger); + } + + public async terminate() { + const encoded = encode({header: new MsgHeader(this.msgId++, 0), msgType: SystemMsgTypes.TERMINATE, value: {}} satisfies MsgPackedHeader); + const { promise, resolve } = promiseWithResolvers(); + this.terminateResolve = resolve; + this.childProcess.stdin.write(encoded); + await promise; + this.runningPromise.stop(); + } + + public async sendPing() { + const { promise } = this.sendMessage(SystemMsgTypes.PING); + await promise; + } + + public async sendStartTree(name: string, depth: number, preFilledSize = 1) { + const request: StartTreeRequest = { + name, + depth, + preFilledSize, + }; + const { promise } = this.sendMessage(WorldStateMsgTypes.START_TREE_REQUEST, request); + const response = (await promise) as StartTreeResponse; + return response; + } + + public async getTreeInfo(name: string) { + const request: GetTreeInfoRequest = { + name, + }; + const { promise } = this.sendMessage(WorldStateMsgTypes.GET_TREE_INFO_REQUEST, request); + const response = (await promise) as GetTreeInfoResponse; + return response; + } + + public async insertLeaves(name: string, leaves: Fr[]) { + const request: InsertLeavesRequest = { + name, + leaves: leaves.map(x => x.toBuffer()), + }; + const { promise } = this.sendMessage(104, request); + const response = (await promise) as InsertLeavesResponse; + return response; + } + + private onRemoteClose() { + this.terminateResolve?.(Buffer.alloc(0)); + } + + private sendMessage(msgType: number, msg?: T) { + const msgId = this.msgId++; + const header = new MsgHeader(msgId, 0); + const encoded = msg ? encode({header, msgType, value: msg} satisfies MsgPackedType) : encode({header, msgType, value: {}} satisfies MsgPackedHeader); + const { promise, resolve } = promiseWithResolvers(); + + const request: Request = { + requestId: msgId, + responseHandler: resolve, + } + this.requests.set(msgId, request); + this.childProcess.stdin.write(encoded); + return { promise, resolve }; + } + + private async handleData() { + for await (const item of decodeMultiStream(this.childProcess.stdout)) { + const itemAsHeader: MsgPackedHeader = item as MsgPackedHeader; + const request = this.requests.get(itemAsHeader.header.requestId); + request?.responseHandler(itemAsHeader.value); + } + } +} diff --git a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts index eca6d258f36..fa2355fea1f 100644 --- a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts @@ -1,10 +1,9 @@ import { SiblingPath } from '@aztec/circuit-types'; -import { type Bufferable, type FromBuffer, serializeToBuffer } from '@aztec/foundation/serialize'; +import { serializeToBuffer, type Bufferable, type FromBuffer } from '@aztec/foundation/serialize'; import { type AztecKVStore, type AztecMap } from '@aztec/kv-store'; import { type Hasher } from '@aztec/types/interfaces'; -import { type AppendOnlyTree } from '../interfaces/append_only_tree.js'; -import { type TreeBase } from '../tree_base.js'; +import { SnapshottedMerkleTree } from '../interfaces/merkle_tree.js'; import { type TreeSnapshot, type TreeSnapshotBuilder } from './snapshot_builder.js'; // stores the last block that modified this node @@ -45,7 +44,7 @@ export class AppendOnlySnapshotBuilder implements TreeSnap constructor( private db: AztecKVStore, - private tree: TreeBase & AppendOnlyTree, + private tree: SnapshottedMerkleTree, private hasher: Hasher, private deserializer: FromBuffer, ) { @@ -164,7 +163,7 @@ class AppendOnlySnapshot implements TreeSnapshot { private block: number, private leafCount: bigint, private historicalRoot: Buffer, - private tree: TreeBase & AppendOnlyTree, + private tree: SnapshottedMerkleTree, private hasher: Hasher, private deserializer: FromBuffer, ) {} diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_native.test.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_native.test.ts new file mode 100644 index 00000000000..836c6142f1e --- /dev/null +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_native.test.ts @@ -0,0 +1,130 @@ +import { Fr } from '@aztec/foundation/fields'; + +import { MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NullifierLeafPreimage } from '@aztec/circuits.js'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { NativeTreesClient } from '../../native/native_client.js'; +import { Pedersen } from '../../pedersen.js'; +import { getWorldStateConfig } from '../../test/get_native_config.js'; +import { INITIAL_LEAF } from '../../tree_base.js'; +import { SiblingPath } from '@aztec/circuit-types'; + +const logger = createDebugLogger('aztec:standard_native_tree_test'); + +// const noopDeserializer: FromBuffer = { +// fromBuffer: (buffer: Buffer) => buffer, +// }; + +// const createDb = async (store: AztecKVStore, hasher: Hasher, name: string, depth: number) => { +// return await newTree(StandardTree, store, hasher, name, noopDeserializer, depth); +// }; + +// const createFromName = async (store: AztecKVStore, hasher: Hasher, name: string) => { +// return await loadTree(StandardTree, store, hasher, name, noopDeserializer); +// }; + +// treeTestSuite('StandardTree', createDb, createFromName); +// standardBasedTreeTestSuite('StandardTree', createDb); + +const TEST_TREE_DEPTH = 3; + +const createNullifierTreeLeafHashInputs = (value: number, nextIndex: number, nextValue: number) => { + return new NullifierLeafPreimage(new Fr(value), new Fr(nextValue), BigInt(nextIndex)).toHashInputs(); +}; + +describe('Native Indexed Tree', () => { + let nativeClient: NativeTreesClient; + let pedersen: Pedersen; + + beforeAll(async () => { + const config = await getWorldStateConfig(logger); + if (!config?.worldStateBinaryPath) { + throw new Error(`Native world state required!`); + } + nativeClient = NativeTreesClient.init(config); + pedersen = new Pedersen(); + }); + + afterAll(async () => { + await nativeClient.terminate(); + }); + + it('creates a new indexed tree', async () => { + const treeName = "Public Data Tree"; + const response = await nativeClient.sendStartTree(treeName, 32); + expect(response.success).toBeTruthy(); + + const response2 = await nativeClient.sendStartTree(treeName, 33); + expect(response2.success).toBeFalsy(); + expect(response2.message).toEqual("Tree already exists"); + expect(response2.depth).toBe(32); + }); + + it('returns the tree information', async () => { + const treeName = "Test Tree 1"; + const response = await nativeClient.sendStartTree(treeName, TEST_TREE_DEPTH); + expect(response.success).toBeTruthy(); + + const getInfoResponse = await nativeClient.getTreeInfo(treeName); + expect(getInfoResponse.success).toBeTruthy(); + expect(getInfoResponse.depth).toBe(TEST_TREE_DEPTH); + expect(getInfoResponse.size).toBe(1); + + /** + * Initial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0. + */ + + const initialLeafHash = pedersen.hashInputs(createNullifierTreeLeafHashInputs(0, 0, 0)); + const level1ZeroHash = pedersen.hash(INITIAL_LEAF, INITIAL_LEAF); + const level2ZeroHash = pedersen.hash(level1ZeroHash, level1ZeroHash); + + let index0Hash = initialLeafHash; + // Each element is named by the level followed by the index on that level. E.g. e10 -> level 1, index 0, e21 -> level 2, index 1 + let e10 = pedersen.hash(index0Hash, INITIAL_LEAF); + let e20 = pedersen.hash(e10, level1ZeroHash); + + let root = pedersen.hash(e20, level2ZeroHash); + + expect(getInfoResponse.root).toEqual(root); + }); + + it('inserts new leaves', async () => { + const treeName = "Test Tree 2"; + const treeDepth = 32; + const initialSize = 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX; + const response = await nativeClient.sendStartTree(treeName, treeDepth, initialSize); + expect(response.success).toBeTruthy(); + + const getInfoResponse = await nativeClient.getTreeInfo(treeName); + expect(getInfoResponse.success).toBeTruthy(); + expect(getInfoResponse.depth).toBe(treeDepth); + expect(getInfoResponse.size).toBe(initialSize); + + const numLeavesToInsert = 12 * 1024; + const leaves = Array.from({length: numLeavesToInsert}).map(Fr.random); + const insertLeavesResponse = await nativeClient.insertLeaves(treeName, leaves); + expect(insertLeavesResponse.success).toBeTruthy(); + expect(insertLeavesResponse.size).toBe(initialSize + leaves.length); + }, 30_000); + + // it('should be able to find indexes of leaves', async () => { + // const db = openTmpStore(); + // const tree = await createDb(db, pedersen, 'test', 3); + // const values = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; + + // await tree.appendLeaves([values[0]]); + + // expect(tree.findLeafIndex(values[0], true)).toBeDefined(); + // expect(tree.findLeafIndex(values[0], false)).toBe(undefined); + // expect(tree.findLeafIndex(values[1], true)).toBe(undefined); + + // await tree.commit(); + + // expect(tree.findLeafIndex(values[0], false)).toBeDefined(); + // }); +}); diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts index e3a4285620a..712bdb6e2f0 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts @@ -4,12 +4,12 @@ import { type AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type Hasher } from '@aztec/types/interfaces'; -import { loadTree, newTree } from '../factory/tree_factory.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; import { PedersenWithCounter } from '../test/utils/pedersen_with_counter.js'; import { INITIAL_LEAF } from '../tree_base.js'; import { StandardTree } from './standard_tree.js'; +import { loadTree, newTree } from '../factory/js_tree_factory.js'; const noopDeserializer: FromBuffer = { fromBuffer: (buffer: Buffer) => buffer, diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts new file mode 100644 index 00000000000..0e208215d6f --- /dev/null +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts @@ -0,0 +1,109 @@ +import { type TreeInsertionStats } from '@aztec/circuit-types/stats'; +import { type Bufferable, serializeToBuffer, FromBuffer } from '@aztec/foundation/serialize'; +import { Timer } from '@aztec/foundation/timer'; + +import { AppendOnlySnapshotBuilder } from '../snapshots/append_only_snapshot.js'; +import { type TreeSnapshot } from '../snapshots/snapshot_builder.js'; +import { Hasher } from '@aztec/types/interfaces'; +import { SnapshottedMerkleTree } from '../interfaces/merkle_tree.js'; +import { SiblingPath } from '@aztec/circuit-types'; +import { AztecKVStore } from '@aztec/kv-store'; +import { INITIAL_LEAF, MAX_DEPTH } from '../tree_base.js'; +import { DebugLogger, createDebugLogger } from '@aztec/foundation/log'; + +/** + * A Merkle tree implementation that uses a LevelDB database to store the tree. + */ +export class StandardTreeNative implements SnapshottedMerkleTree { + #snapshotBuilder = new AppendOnlySnapshotBuilder(this.store, this, this.hasher, this.deserializer); + private zeroHashes: Buffer[] = []; + protected readonly maxIndex: bigint; + protected log: DebugLogger; + + public constructor( + protected store: AztecKVStore, + protected hasher: Hasher, + private name: string, + private depth: number, + protected size: bigint = 0n, + protected deserializer: FromBuffer, + root?: Buffer, + ) { + if (!(depth >= 1 && depth <= MAX_DEPTH)) { + throw Error('Invalid depth'); + } + + // Compute the zero values at each layer. + let current = INITIAL_LEAF; + for (let i = depth - 1; i >= 0; --i) { + this.zeroHashes[i] = current; + current = hasher.hash(current, current); + } + this.maxIndex = 2n ** BigInt(depth) - 1n; + + this.log = createDebugLogger(`aztec:merkle-tree:${name.toLowerCase()}`); + } + getName(): string { + return this.name; + } + getNode(level: number, index: bigint): Promise { + throw new Error('Method not implemented.'); + } + getZeroHash(level: number): Buffer { + return this.zeroHashes[level]; + } + getRoot(includeUncommitted: boolean): Buffer { + throw new Error('Method not implemented.'); + } + getNumLeaves(includeUncommitted: boolean): bigint { + throw new Error('Method not implemented.'); + } + commit(): Promise { + throw new Error('Method not implemented.'); + } + getDepth(): number { + throw new Error('Method not implemented.'); + } + rollback(): Promise { + throw new Error('Method not implemented.'); + } + getLeafValue(index: bigint, includeUncommitted: boolean): T | undefined { + throw new Error('Method not implemented.'); + } + getSiblingPath(index: bigint, includeUncommitted: boolean): Promise> { + throw new Error('Method not implemented.'); + } + + /** + * Appends the given leaves to the tree. + * @param leaves - The leaves to append. + * @returns Empty promise. + */ + public appendLeaves(leaves: T[]): Promise { + + return Promise.resolve(); + } + + public snapshot(blockNumber: number): Promise> { + return this.#snapshotBuilder.snapshot(blockNumber); + } + + public getSnapshot(blockNumber: number): Promise> { + return this.#snapshotBuilder.getSnapshot(blockNumber); + } + + public findLeafIndex(value: T, includeUncommitted: boolean): bigint | undefined { + return this.findLeafIndexAfter(value, 0n, includeUncommitted); + } + + public findLeafIndexAfter(value: T, startIndex: bigint, includeUncommitted: boolean): bigint | undefined { + const buffer = serializeToBuffer(value); + for (let i = startIndex; i < this.getNumLeaves(includeUncommitted); i++) { + const currentValue = this.getLeafValue(i, includeUncommitted); + if (currentValue && serializeToBuffer(currentValue).equals(buffer)) { + return i; + } + } + return undefined; + } +} diff --git a/yarn-project/merkle-tree/src/test/get_native_config.ts b/yarn-project/merkle-tree/src/test/get_native_config.ts new file mode 100644 index 00000000000..158001515ce --- /dev/null +++ b/yarn-project/merkle-tree/src/test/get_native_config.ts @@ -0,0 +1,28 @@ +import { type DebugLogger, fileURLToPath } from '@aztec/aztec.js'; + +import fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'path'; + +const { + BB_RELEASE_DIR = 'barretenberg/cpp/build/bin', + WORLD_STATE_BINARY_PATH, + TEMP_DIR = tmpdir(), + DATA_DIRECTORY = '', +} = process.env; + +export const getWorldStateConfig = async ( + logger: DebugLogger, +): Promise<{ worldStateBinaryPath: string; } | undefined> => { + try { + const worldStateBinaryPath = + WORLD_STATE_BINARY_PATH ?? + path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../', BB_RELEASE_DIR, 'world_state'); + await fs.access(worldStateBinaryPath, fs.constants.R_OK); + + return { worldStateBinaryPath }; + } catch (err) { + logger.error(`Native BB not available, error: ${err}`); + return undefined; + } +}; diff --git a/yarn-project/merkle-tree/src/tree_base.ts b/yarn-project/merkle-tree/src/tree_base.ts index cd20e8e0f2d..cc5edcbad2c 100644 --- a/yarn-project/merkle-tree/src/tree_base.ts +++ b/yarn-project/merkle-tree/src/tree_base.ts @@ -8,7 +8,7 @@ import { type Hasher } from '@aztec/types/interfaces'; import { HasherWithStats } from './hasher_with_stats.js'; import { type MerkleTree } from './interfaces/merkle_tree.js'; -const MAX_DEPTH = 254; +export const MAX_DEPTH = 254; const indexToKeyHash = (name: string, level: number, index: bigint) => `${name}:${level}:${index}`; const encodeMeta = (root: Buffer, depth: number, size: bigint) => { @@ -194,7 +194,7 @@ export abstract class TreeBase implements MerkleTree { return this.getLatestValueAtIndex(this.depth, index, includeUncommitted); } - public getNode(level: number, index: bigint): Buffer | undefined { + public getNode(level: number, index: bigint): Promise { if (level < 0 || level > this.depth) { throw Error('Invalid level: ' + level); } @@ -203,7 +203,7 @@ export abstract class TreeBase implements MerkleTree { throw Error('Invalid index: ' + index); } - return this.dbGet(indexToKeyHash(this.name, level, index)); + return Promise.resolve(this.dbGet(indexToKeyHash(this.name, level, index))); } public getZeroHash(level: number): Buffer { diff --git a/yarn-project/prover-client/tsconfig.json b/yarn-project/prover-client/tsconfig.json index 0cbffc97b3e..5f748569b8c 100644 --- a/yarn-project/prover-client/tsconfig.json +++ b/yarn-project/prover-client/tsconfig.json @@ -34,5 +34,5 @@ "path": "../world-state" } ], - "include": ["src", "../bb-prover/src/test/test_circuit_prover.ts"] + "include": ["src"] } diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 5e715fd4690..98fb2afd255 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -624,6 +624,7 @@ __metadata: "@aztec/kv-store": "workspace:^" "@aztec/types": "workspace:^" "@jest/globals": ^29.5.0 + "@msgpack/msgpack": ^3.0.0-beta2 "@types/jest": ^29.5.0 "@types/node": ^18.15.3 "@types/sha256": ^0.2.0 @@ -2746,6 +2747,13 @@ __metadata: languageName: node linkType: hard +"@msgpack/msgpack@npm:^3.0.0-beta2": + version: 3.0.0-beta2 + resolution: "@msgpack/msgpack@npm:3.0.0-beta2" + checksum: d86e5d48146051952d6bea35a6cf733a401cf65ad5614d79689aa48c7076021737ca2c782978dd1b6c0c9c45888b246e379e45ae906179e3a0e8ef4ee6f221c1 + languageName: node + linkType: hard + "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.2": version: 3.0.2 resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.2" From d4650c562979739df8f5eec0715cf49655e4ada7 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 25 Jun 2024 17:28:11 +0100 Subject: [PATCH 21/63] WIP --- barretenberg/cpp/src/CMakeLists.txt | 1 - .../append_only_tree.bench.cpp | 48 +- .../indexed_tree_bench/indexed_tree.bench.cpp | 54 +- .../crypto/merkle_tree/CMakeLists.txt | 1 + .../append_only_tree/append_only_tree.hpp | 234 +++++--- .../append_only_tree.test.cpp | 369 +++++++++--- .../crypto/merkle_tree/array_store.hpp | 44 -- .../crypto/merkle_tree/fixtures.hpp | 50 ++ .../merkle_tree/indexed_tree/indexed_leaf.hpp | 13 +- .../merkle_tree/indexed_tree/indexed_tree.hpp | 374 +++++++----- .../indexed_tree/indexed_tree.test.cpp | 549 +++++++++++++----- .../merkle_tree/lmdb_store/lmdb_store.cpp | 127 ++++ .../merkle_tree/lmdb_store/lmdb_store.hpp | 340 +++++++++++ .../lmdb_store/lmdb_store.test.cpp | 476 +++++++++++++++ .../merkle_tree/node_store/array_store.hpp | 94 +++ .../node_store/cached_leaves_store.hpp | 126 ++++ .../node_store/cached_tree_store.hpp | 319 ++++++++++ .../merkle_tree/node_store/tree_meta.hpp | 24 + .../barretenberg/lmdb_store/CMakeLists.txt | 2 - .../barretenberg/lmdb_store/lmdb_store.cpp | 85 --- .../barretenberg/lmdb_store/lmdb_store.hpp | 182 ------ .../lmdb_store/lmdb_store.test.cpp | 273 --------- .../messaging/stream_parser.test.cpp | 184 ------ .../service/world_state_service.hpp | 59 +- 24 files changed, 2728 insertions(+), 1300 deletions(-) delete mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_leaves_store.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt delete mode 100644 barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp delete mode 100644 barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp delete mode 100644 barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 6ba950613b4..f506da7bd5d 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -89,7 +89,6 @@ add_subdirectory(barretenberg/vm) add_subdirectory(barretenberg/wasi) add_subdirectory(barretenberg/world_state) add_subdirectory(barretenberg/world_state_service) -add_subdirectory(barretenberg/lmdb_store) if(SMT) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp index bde03dcb4ff..8e0d4176de8 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp @@ -1,25 +1,43 @@ #include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/array_store.hpp" +#include "barretenberg/common/thread_pool.hpp" +#include "barretenberg/crypto/merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/numeric/random/engine.hpp" #include +#include using namespace benchmark; using namespace bb::crypto::merkle_tree; -using Pedersen = AppendOnlyTree; -using Poseidon2 = AppendOnlyTree; +using StoreType = CachedTreeStore; + +using Pedersen = AppendOnlyTree; +using Poseidon2 = AppendOnlyTree; const size_t TREE_DEPTH = 32; const size_t MAX_BATCH_SIZE = 128; -namespace { -auto& random_engine = bb::numeric::get_randomness(); -} // namespace - template void perform_batch_insert(TreeType& tree, const std::vector& values) { - tree.add_values(values); + Signal signal(1); + auto completion = [&](const fr&, index_t) -> void { signal.signal_level(0); }; + + tree.add_values(values, completion); + signal.wait_for_level(0); +} + +template void commit_tree(TreeType& tree) +{ + Signal signal(1); + auto completion = [&]() -> void { signal.signal_level(0); }; + tree.commit(completion); + signal.wait_for_level(0); } template void append_only_tree_bench(State& state) noexcept @@ -27,8 +45,16 @@ template void append_only_tree_bench(State& state) noexcept const size_t batch_size = size_t(state.range(0)); const size_t depth = TREE_DEPTH; - ArrayStore store(depth, 1024 * 1024); - TreeType tree = TreeType(store, depth); + std::string directory = randomTempDirectory(); + std::string name = randomString(); + std::filesystem::create_directories(directory); + uint32_t num_threads = 16; + LMDBEnvironment environment = LMDBEnvironment(directory, 1024, 2, num_threads); + + LMDBStore db(environment, name, false, false, IntegerKeyCmp); + StoreType store(name, depth, db); + ThreadPool workers(num_threads); + TreeType tree = TreeType(store, workers); for (auto _ : state) { state.PauseTiming(); @@ -39,6 +65,8 @@ template void append_only_tree_bench(State& state) noexcept state.ResumeTiming(); perform_batch_insert(tree, values); } + + std::filesystem::remove_all(directory); } BENCHMARK(append_only_tree_bench) ->Unit(benchmark::kMillisecond) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp index 5e7c0f10abb..33afba6bd1f 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp @@ -1,27 +1,31 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" -#include "barretenberg/crypto/merkle_tree/array_store.hpp" +#include "barretenberg/crypto/merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" #include "barretenberg/numeric/random/engine.hpp" #include +#include using namespace benchmark; using namespace bb::crypto::merkle_tree; -using Poseidon2 = IndexedTree; -using Pedersen = IndexedTree; +using StoreType = CachedTreeStore; -const size_t TREE_DEPTH = 32; -const size_t MAX_BATCH_SIZE = 128; +using Poseidon2 = IndexedTree; +using Pedersen = IndexedTree; -namespace { -auto& random_engine = bb::numeric::get_randomness(); -} // namespace +const size_t TREE_DEPTH = 40; +const size_t MAX_BATCH_SIZE = 128; -template -void perform_batch_insert(TreeType& tree, const std::vector& values, bool single_threaded) +template void add_values(TreeType& tree, const std::vector& values) { - tree.add_or_update_values(values, single_threaded); + Signal signal(1); + auto completion = [&](const std::vector&, fr&, index_t) -> void { signal.signal_level(0); }; + + tree.add_or_update_values(values, completion); + signal.wait_for_level(0); } template void multi_thread_indexed_tree_bench(State& state) noexcept @@ -29,8 +33,16 @@ template void multi_thread_indexed_tree_bench(State& state) const size_t batch_size = size_t(state.range(0)); const size_t depth = TREE_DEPTH; - ArrayStore store(depth, 1024 * 1024); - TreeType tree = TreeType(store, depth, batch_size); + std::string directory = randomTempDirectory(); + std::string name = randomString(); + std::filesystem::create_directories(directory); + uint32_t num_threads = 16; + LMDBEnvironment environment = LMDBEnvironment(directory, 1024, 2, num_threads); + + LMDBStore db(environment, name, false, false, IntegerKeyCmp); + StoreType store(name, depth, db); + ThreadPool workers(num_threads); + TreeType tree = TreeType(store, workers, batch_size); for (auto _ : state) { state.PauseTiming(); @@ -39,7 +51,7 @@ template void multi_thread_indexed_tree_bench(State& state) values[i] = fr(random_engine.get_random_uint256()); } state.ResumeTiming(); - perform_batch_insert(tree, values, false); + add_values(tree, values); } } @@ -48,8 +60,16 @@ template void single_thread_indexed_tree_bench(State& state) const size_t batch_size = size_t(state.range(0)); const size_t depth = TREE_DEPTH; - ArrayStore store(depth, 1024 * 1024); - TreeType tree = TreeType(store, depth, batch_size); + std::string directory = randomTempDirectory(); + std::string name = randomString(); + std::filesystem::create_directories(directory); + uint32_t num_threads = 1; + LMDBEnvironment environment = LMDBEnvironment(directory, 1024, 2, num_threads); + + LMDBStore db(environment, name, false, false, IntegerKeyCmp); + StoreType store(name, depth, db); + ThreadPool workers(num_threads); + TreeType tree = TreeType(store, workers, batch_size); for (auto _ : state) { state.PauseTiming(); @@ -58,7 +78,7 @@ template void single_thread_indexed_tree_bench(State& state) values[i] = fr(random_engine.get_random_uint256()); } state.ResumeTiming(); - perform_batch_insert(tree, values, true); + add_values(tree, values); } } BENCHMARK(single_thread_indexed_tree_bench) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt index 475b8ae9829..24205228325 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt @@ -1,5 +1,6 @@ barretenberg_module( crypto_merkle_tree + lmdb stdlib_primitives stdlib_blake3s stdlib_pedersen_hash diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 21df4746584..773859f6661 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -1,7 +1,9 @@ #pragma once #include "../hash_path.hpp" +#include "../node_store//tree_meta.hpp" #include "../types.hpp" #include "barretenberg/common/thread_pool.hpp" +#include #include #include #include @@ -17,9 +19,13 @@ using namespace bb; */ template class AppendOnlyTree { public: - typedef std::function append_completion_callback; + using append_completion_callback = std::function; + using meta_data_callback = std::function; + using hash_path_callback = std::function; + using commit_callback = std::function; + using rollback_callback = std::function; - AppendOnlyTree(Store& store, uint32_t depth, ThreadPool& workers, std::string name); + AppendOnlyTree(Store& store, ThreadPool& workers); AppendOnlyTree(AppendOnlyTree const& other) = delete; AppendOnlyTree(AppendOnlyTree&& other) = delete; AppendOnlyTree& operator=(AppendOnlyTree const& other) = delete; @@ -37,94 +43,100 @@ template class AppendOnlyTree { virtual void add_values(const std::vector& values, const append_completion_callback& on_completion); /** - * @brief Returns the index of the right-most populated leaf in the tree - */ - index_t size() const; - - /** - * @brief Returns the root of the tree + * @brief Returns the hash path from the leaf at the given index to the root */ - fr root() const; + void get_hash_path(const index_t& index, const hash_path_callback& on_completion, bool includeUncommitted) const; - /** - * @brief Returns the depth of the tree - */ - uint32_t depth() const; + void get_meta_data(bool includeUncommitted, const meta_data_callback& on_completion); - /** - * @brief Returns the hash path from the leaf at the given index to the root - */ - fr_hash_path get_hash_path(const index_t& index) const; + void commit(const commit_callback& on_completion); + void rollback(const rollback_callback& on_completion); protected: - fr get_element_or_zero(uint32_t level, const index_t& index) const; + using ReadTransaction = typename Store::ReadTransaction; + using ReadTransactionPtr = typename Store::ReadTransactionPtr; + fr get_element_or_zero(uint32_t level, const index_t& index, ReadTransaction& tx, bool includeUncommitted) const; void write_node(uint32_t level, const index_t& index, const fr& value); - std::pair read_node(uint32_t level, const index_t& index) const; + std::pair read_node(uint32_t level, + const index_t& index, + ReadTransaction& tx, + bool includeUncommitted) const; Store& store_; - const uint32_t depth_; - const std::string name_; + uint32_t depth_; + std::string name_; std::vector zero_hashes_; - fr root_; - index_t size_; ThreadPool& workers_; }; template -AppendOnlyTree::AppendOnlyTree(Store& store, - uint32_t depth, - ThreadPool& workers, - std::string name) +AppendOnlyTree::AppendOnlyTree(Store& store, ThreadPool& workers) : store_(store) - , depth_(depth) - , name_(std::move(name)) , workers_(workers) { - zero_hashes_.resize(depth + 1); + index_t stored_size = 0; + bb::fr stored_root = fr::zero(); + { + ReadTransactionPtr tx = store_.createReadTransaction(); + store_.get_full_meta(stored_size, stored_root, name_, depth_, *tx, false); + } + zero_hashes_.resize(depth_ + 1); // Create the zero hashes for the tree auto current = HashingPolicy::zero_hash(); - for (size_t i = depth; i > 0; --i) { + for (size_t i = depth_; i > 0; --i) { zero_hashes_[i] = current; current = HashingPolicy::hash_pair(current, current); } zero_hashes_[0] = current; - root_ = current; - size_ = 0; -} -template index_t AppendOnlyTree::size() const -{ - return size_; -} - -template fr AppendOnlyTree::root() const -{ - return root_; + if (stored_size == 0) { + store_.put_meta(0, current); + store_.commit(); + } } -template uint32_t AppendOnlyTree::depth() const +template +void AppendOnlyTree::get_meta_data(bool includeUncommitted, + const meta_data_callback& on_completion) { - return depth_; + auto job = [=]() { + index_t size = 0; + bb::fr root = fr::zero(); + { + ReadTransactionPtr tx = store_.createReadTransaction(); + store_.get_meta(size, root, *tx, includeUncommitted); + } + on_completion(name_, depth_, size, root); + }; + workers_.enqueue(job); } template -fr_hash_path AppendOnlyTree::get_hash_path(const index_t& index) const +void AppendOnlyTree::get_hash_path(const index_t& index, + const hash_path_callback& on_completion, + bool includeUncommitted) const { - fr_hash_path path; - index_t current_index = index; - - for (uint32_t level = depth_; level > 0; --level) { - bool is_right = static_cast(current_index & 0x01); - fr right_value = - is_right ? get_element_or_zero(level, current_index) : get_element_or_zero(level, current_index + 1); - fr left_value = - is_right ? get_element_or_zero(level, current_index - 1) : get_element_or_zero(level, current_index); - path.emplace_back(left_value, right_value); - current_index >>= 1; - } - return path; + auto job = [=]() { + fr_hash_path path; + index_t current_index = index; + { + ReadTransactionPtr tx = store_.createReadTransaction(); + + for (uint32_t level = depth_; level > 0; --level) { + bool is_right = static_cast(current_index & 0x01); + fr right_value = is_right ? get_element_or_zero(level, current_index, *tx, includeUncommitted) + : get_element_or_zero(level, current_index + 1, *tx, includeUncommitted); + fr left_value = is_right ? get_element_or_zero(level, current_index - 1, *tx, includeUncommitted) + : get_element_or_zero(level, current_index, *tx, includeUncommitted); + path.emplace_back(left_value, right_value); + current_index >>= 1; + } + } + on_completion(path); + }; + workers_.enqueue(job); } template @@ -137,53 +149,66 @@ template void AppendOnlyTree::add_values(const std::vector& values, const append_completion_callback& on_completion) { - index_t start_size = size(); uint32_t start_level = depth_; std::shared_ptr> hashes = std::make_shared>(values); auto append_op = [=]() -> void { - index_t index = start_size; - uint32_t level = start_level; - uint32_t number_to_insert = static_cast(values.size()); - std::vector& hashes_local = *hashes; - // Add the values at the leaf nodes of the tree - for (uint32_t i = 0; i < number_to_insert; ++i) { - write_node(level, index + i, hashes_local[i]); - } - - // Hash the values as a sub tree and insert them - while (number_to_insert > 1) { - number_to_insert >>= 1; - index >>= 1; - --level; + bb::fr new_root = fr::zero(); + index_t new_size = 0; + { + typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); + index_t start_size; + bb::fr root; + store_.get_meta(start_size, root, *tx, true); + index_t index = start_size; + uint32_t level = start_level; + uint32_t number_to_insert = static_cast(values.size()); + std::vector& hashes_local = *hashes; + // Add the values at the leaf nodes of the tree for (uint32_t i = 0; i < number_to_insert; ++i) { - hashes_local[i] = HashingPolicy::hash_pair(hashes_local[i * 2], hashes_local[i * 2 + 1]); write_node(level, index + i, hashes_local[i]); } - } - // Hash from the root of the sub-tree to the root of the overall tree - fr new_hash = hashes_local[0]; - while (level > 0) { - bool is_right = static_cast(index & 0x01); - fr left_hash = is_right ? get_element_or_zero(level, index - 1) : new_hash; - fr right_hash = is_right ? new_hash : get_element_or_zero(level, index + 1); - new_hash = HashingPolicy::hash_pair(left_hash, right_hash); - index >>= 1; - --level; - write_node(level, index, new_hash); + // Hash the values as a sub tree and insert them + while (number_to_insert > 1) { + number_to_insert >>= 1; + index >>= 1; + --level; + for (uint32_t i = 0; i < number_to_insert; ++i) { + hashes_local[i] = HashingPolicy::hash_pair(hashes_local[i * 2], hashes_local[i * 2 + 1]); + write_node(level, index + i, hashes_local[i]); + } + } + + // Hash from the root of the sub-tree to the root of the overall tree + fr new_hash = hashes_local[0]; + while (level > 0) { + bool is_right = static_cast(index & 0x01); + fr left_hash = is_right ? get_element_or_zero(level, index - 1, *tx, true) : new_hash; + fr right_hash = is_right ? new_hash : get_element_or_zero(level, index + 1, *tx, true); + new_hash = HashingPolicy::hash_pair(left_hash, right_hash); + index >>= 1; + --level; + if (level > 0) { + write_node(level, index, new_hash); + } + } + new_size = start_size + values.size(); + new_root = new_hash; + store_.put_meta(new_size, new_root); } - size_ += values.size(); - root_ = new_hash; - on_completion(root_, size_); + on_completion(new_root, new_size); }; workers_.enqueue(append_op); } template -fr AppendOnlyTree::get_element_or_zero(uint32_t level, const index_t& index) const +fr AppendOnlyTree::get_element_or_zero(uint32_t level, + const index_t& index, + ReadTransaction& tx, + bool includeUncommitted) const { - const std::pair read_data = read_node(level, index); + const std::pair read_data = read_node(level, index, tx, includeUncommitted); if (read_data.first) { return read_data.second; } @@ -196,14 +221,17 @@ void AppendOnlyTree::write_node(uint32_t level, const inde { std::vector buf; write(buf, value); - store_.put(level, index, buf); + store_.put_node(level, index, buf); } template -std::pair AppendOnlyTree::read_node(uint32_t level, const index_t& index) const +std::pair AppendOnlyTree::read_node(uint32_t level, + const index_t& index, + ReadTransaction& tx, + bool includeUncommitted) const { std::vector buf; - bool available = store_.get(level, index, buf); + bool available = store_.get_node(level, index, buf, tx, includeUncommitted); if (!available) { return std::make_pair(false, fr::zero()); } @@ -211,4 +239,24 @@ std::pair AppendOnlyTree::read_node(uint32_t lev return std::make_pair(true, value); } +template +void AppendOnlyTree::commit(const commit_callback& on_completion) +{ + auto job = [=]() { + store_.commit(); + on_completion(); + }; + workers_.enqueue(job); +} + +template +void AppendOnlyTree::rollback(const rollback_callback& on_completion) +{ + auto job = [=]() { + store_.rollback(); + on_completion(); + }; + workers_.enqueue(job); +} + } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp index 8b872b29331..3191d71dac0 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -1,165 +1,352 @@ #include "append_only_tree.hpp" -#include "../array_store.hpp" +#include "../fixtures.hpp" #include "../memory_tree.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" #include "barretenberg/common/thread_pool.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" -#include "barretenberg/numeric/random/engine.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "gtest/gtest.h" +#include +#include +#include +#include using namespace bb; using namespace bb::crypto::merkle_tree; -namespace { -auto& engine = numeric::get_debug_randomness(); -auto& random_engine = numeric::get_randomness(); -} // namespace +using Store = CachedTreeStore; +using TreeType = AppendOnlyTree; -const uint32_t NUM_VALUES = 1024; -static std::vector VALUES = []() { - std::vector values(NUM_VALUES); - for (uint32_t i = 0; i < NUM_VALUES; ++i) { - values[i] = fr(random_engine.get_random_uint256()); +class PersistedAppendOnlyTreeTest : public testing::Test { + protected: + void SetUp() override + { + // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers + _directory = randomTempDirectory(); + std::filesystem::create_directories(_directory); + _environment = std::make_unique(_directory, 1, 2, 2); } - return values; -}(); -inline void print_tree(const uint32_t depth, std::vector hashes, std::string const& msg) + void TearDown() override { std::filesystem::remove_all(_directory); } + + static std::string _directory; + + std::unique_ptr _environment; +}; + +std::string PersistedAppendOnlyTreeTest::_directory; + +void check_size(TreeType& tree, index_t expected_size, bool includeUncommitted = true) { - info("\n", msg); - uint32_t offset = 0; - for (uint32_t i = 0; i < depth; i++) { - info("i = ", i); - uint32_t layer_size = (1U << (depth - i)); - for (uint32_t j = 0; j < layer_size; j++) { - info("j = ", j, ": ", hashes[offset + j]); - } - offset += layer_size; - } + Signal signal(1); + auto completion = [&](const std::string&, uint32_t, const index_t& size, const fr&) -> void { + EXPECT_EQ(size, expected_size); + signal.signal_level(0); + }; + tree.get_meta_data(includeUncommitted, completion); + signal.wait_for_level(0); +} + +void check_root(TreeType& tree, fr expected_root, bool includeUncommitted = true) +{ + Signal signal(1); + auto completion = [&](const std::string&, uint32_t, const index_t&, const fr& root) -> void { + EXPECT_EQ(root, expected_root); + signal.signal_level(0); + }; + tree.get_meta_data(includeUncommitted, completion); + signal.wait_for_level(0); +} + +void check_hash_path(TreeType& tree, index_t index, fr_hash_path expected_hash_path, bool includeUncommitted = true) +{ + Signal signal(1); + auto completion = [&](const fr_hash_path& path) -> void { + EXPECT_EQ(path, expected_hash_path); + signal.signal_level(0); + }; + tree.get_hash_path(index, completion, includeUncommitted); + signal.wait_for_level(0); +} + +void commit_tree(TreeType& tree) +{ + Signal signal(1); + auto completion = [&]() -> void { signal.signal_level(0); }; + tree.commit(completion); + signal.wait_for_level(0); +} + +void add_value(TreeType& tree, const fr& value) +{ + Signal signal(1); + auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + + tree.add_value(value, completion); + signal.wait_for_level(0); +} + +void add_values(TreeType& tree, const std::vector& values) +{ + Signal signal(1); + auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + + tree.add_values(values, completion); + signal.wait_for_level(0); } -TEST(crypto_append_only_tree, can_create) +TEST_F(PersistedAppendOnlyTreeTest, can_create) { constexpr size_t depth = 10; - ArrayStore store(depth); + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + EXPECT_NO_THROW(Store store(name, depth, db)); + Store store(name, depth, db); + ThreadPool pool(1); - AppendOnlyTree tree(store, depth, pool, "Test Tree"); + TreeType tree(store, pool); MemoryTree memdb(depth); - EXPECT_EQ(tree.size(), 0); - EXPECT_EQ(tree.root(), memdb.root()); + check_size(tree, 0); + check_root(tree, memdb.root()); +} + +TEST_F(PersistedAppendOnlyTreeTest, can_only_recreate_with_same_name_and_depth) +{ + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + + EXPECT_ANY_THROW(Store store_wrong_name("Wrong name", depth, db)); + EXPECT_ANY_THROW(Store store_wrong_depth(name, depth + 1, db)); } -TEST(crypto_append_only_tree, can_add_value) +TEST_F(PersistedAppendOnlyTreeTest, can_add_value_and_get_hash_path) { constexpr size_t depth = 10; - ArrayStore store(depth); + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + ThreadPool pool(1); - AppendOnlyTree tree(store, depth, pool, "Test Tree"); + TreeType tree(store, pool); MemoryTree memdb(depth); - EXPECT_EQ(tree.size(), 0); - EXPECT_EQ(tree.root(), memdb.root()); + check_size(tree, 0); + check_root(tree, memdb.root()); - LevelSignal signal(1); + Signal signal(1); auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; memdb.update_element(0, VALUES[0]); tree.add_value(VALUES[0], completion); signal.wait_for_level(0); - EXPECT_EQ(tree.root(), memdb.root()); - EXPECT_EQ(tree.get_hash_path(0), memdb.get_hash_path(0)); + check_size(tree, 1); + check_root(tree, memdb.root()); + check_hash_path(tree, 0, memdb.get_hash_path(0)); } -TEST(crypto_append_only_tree, test_size) +TEST_F(PersistedAppendOnlyTreeTest, can_commit_and_restore) { constexpr size_t depth = 10; - ArrayStore store(depth); - ThreadPool pool(1); - AppendOnlyTree tree(store, depth, pool, "Test Tree"); + std::string name = randomString(); + MemoryTree memdb(depth); + { + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); - EXPECT_EQ(tree.size(), 0ULL); + ThreadPool pool(1); + TreeType tree(store, pool); - // Add a new non-zero leaf at index 0. - { - LevelSignal signal(1); - auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; - tree.add_value(30, completion); - signal.wait_for_level(0); - EXPECT_EQ(tree.size(), 1ULL); + check_size(tree, 0); + check_root(tree, memdb.root()); + check_hash_path(tree, 0, memdb.get_hash_path(0)); + + bb::fr initial_root = memdb.root(); + fr_hash_path initial_hash_path = memdb.get_hash_path(0); + memdb.update_element(0, VALUES[0]); + add_value(tree, VALUES[0]); + + // check uncommitted state + check_size(tree, 1); + check_root(tree, memdb.root()); + check_hash_path(tree, 0, memdb.get_hash_path(0)); + + // check committed state + check_size(tree, 0, false); + check_root(tree, initial_root, false); + check_hash_path(tree, 0, initial_hash_path, false); + + // commit the changes + commit_tree(tree); + // now committed and uncommitted should be the same + + // check uncommitted state + check_size(tree, 1); + check_root(tree, memdb.root()); + check_hash_path(tree, 0, memdb.get_hash_path(0)); + + // check committed state + check_size(tree, 1, false); + check_root(tree, memdb.root(), false); + check_hash_path(tree, 0, memdb.get_hash_path(0), false); } - // Add second. + // Re-create the store and tree, it should be the same as how we left it { - LevelSignal signal(1); - auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; - tree.add_value(10, completion); - signal.wait_for_level(0); - EXPECT_EQ(tree.size(), 2ULL); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + + ThreadPool pool(1); + TreeType tree(store, pool); + + // check uncommitted state + check_size(tree, 1); + check_root(tree, memdb.root()); + check_hash_path(tree, 0, memdb.get_hash_path(0)); + + // check committed state + check_size(tree, 1, false); + check_root(tree, memdb.root(), false); + check_hash_path(tree, 0, memdb.get_hash_path(0), false); } +} + +TEST_F(PersistedAppendOnlyTreeTest, test_size) +{ + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + ThreadPool pool(1); + TreeType tree(store, pool); + + check_size(tree, 0); + + // Add a new non-zero leaf at index 0. + add_value(tree, 30); + check_size(tree, 1); + + // Add second. + add_value(tree, 10); + check_size(tree, 2); // Add third. - { - LevelSignal signal(1); - auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; - tree.add_value(20, completion); - signal.wait_for_level(0); - EXPECT_EQ(tree.size(), 3ULL); - } + add_value(tree, 20); + check_size(tree, 3); // Add forth but with same value. - { - LevelSignal signal(1); - auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; - tree.add_value(20, completion); - signal.wait_for_level(0); - EXPECT_EQ(tree.size(), 4ULL); - } + add_value(tree, 40); + check_size(tree, 4); } -TEST(crypto_append_only_tree, can_add_multiple_values) +TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values) { constexpr size_t depth = 10; - ArrayStore store(depth); + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); ThreadPool pool(1); - AppendOnlyTree tree(store, depth, pool, "Test Tree"); + TreeType tree(store, pool); MemoryTree memdb(depth); for (size_t i = 0; i < NUM_VALUES; ++i) { fr mock_root = memdb.update_element(i, VALUES[i]); - LevelSignal signal(1); - auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; - tree.add_value(VALUES[i], completion); - signal.wait_for_level(0); - fr tree_root = tree.root(); - EXPECT_EQ(mock_root, tree_root); + add_value(tree, VALUES[i]); + check_root(tree, mock_root); + + check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_hash_path(tree, i, memdb.get_hash_path(i)); + } +} - EXPECT_EQ(memdb.get_hash_path(0), tree.get_hash_path(0)); - EXPECT_EQ(memdb.get_hash_path(i), tree.get_hash_path(i)); +TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values_in_a_batch) +{ + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + ThreadPool pool(1); + TreeType tree(store, pool); + MemoryTree memdb(depth); + + for (size_t i = 0; i < NUM_VALUES; ++i) { + memdb.update_element(i, VALUES[i]); } + add_values(tree, VALUES); + check_size(tree, NUM_VALUES); + check_root(tree, memdb.root()); + check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_hash_path(tree, NUM_VALUES - 1, memdb.get_hash_path(NUM_VALUES - 1)); } -TEST(crypto_append_only_tree, can_be_filled) +TEST_F(PersistedAppendOnlyTreeTest, can_be_filled) { constexpr size_t depth = 3; - ArrayStore store(depth); + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); ThreadPool pool(1); - AppendOnlyTree tree(store, depth, pool, "Test Tree"); + TreeType tree(store, pool); MemoryTree memdb(depth); - EXPECT_EQ(tree.size(), 0); - EXPECT_EQ(tree.root(), memdb.root()); + check_size(tree, 0); + check_root(tree, memdb.root()); for (size_t i = 0; i < 8; i++) { memdb.update_element(i, VALUES[i]); - LevelSignal signal(1); - auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; - tree.add_value(VALUES[i], completion); + add_value(tree, VALUES[i]); + } + + check_root(tree, memdb.root()); + check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_hash_path(tree, 7, memdb.get_hash_path(7)); +} + +TEST_F(PersistedAppendOnlyTreeTest, can_add_single_whilst_reading) +{ + constexpr size_t depth = 10; + MemoryTree memdb(depth); + fr_hash_path initial_path = memdb.get_hash_path(0); + memdb.update_element(0, VALUES[0]); + fr_hash_path final_hash_path = memdb.get_hash_path(0); + + uint32_t num_reads = 16 * 1024; + std::vector paths(num_reads); + + { + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + ThreadPool pool(8); + TreeType tree(store, pool); + + check_size(tree, 0); + + Signal signal(2); + + auto add_completion = [&](const fr&, index_t) { + signal.signal_level(1); + auto commit_completion = [&]() { signal.signal_level(0); }; + tree.commit(commit_completion); + }; + tree.add_value(VALUES[0], add_completion); + + for (size_t i = 0; i < num_reads; i++) { + auto completion = [&, i](const fr_hash_path& path) { paths[i] = path; }; + tree.get_hash_path(0, completion, false); + } signal.wait_for_level(0); } - EXPECT_EQ(tree.root(), memdb.root()); - EXPECT_EQ(tree.get_hash_path(0), memdb.get_hash_path(0)); - EXPECT_EQ(tree.get_hash_path(7), memdb.get_hash_path(7)); + for (auto& path : paths) { + EXPECT_TRUE(path == initial_path || path == final_hash_path); + } } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp deleted file mode 100644 index cb5754947af..00000000000 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/stdlib/primitives/field/field.hpp" - -namespace bb::crypto::merkle_tree { - -/** - * @brief A very basic 2-d array for use as a backing store for merkle trees. - * Can store up to 'indices' nodes per row and 'levels' rows. - */ -class ArrayStore { - - public: - ArrayStore(uint32_t levels, index_t indices = 1024) - : map_(std::vector>>>( - levels + 1, - std::vector>>( - indices, std::pair>(false, std::vector())))) - {} - ~ArrayStore() = default; - - ArrayStore() = delete; - ArrayStore(ArrayStore const& other) = delete; - ArrayStore(ArrayStore const&& other) = delete; - ArrayStore& operator=(ArrayStore const& other) = delete; - ArrayStore& operator=(ArrayStore const&& other) = delete; - - void put(uint32_t level, index_t index, const std::vector& data) - { - map_[level][index] = std::make_pair(true, data); - } - bool get(uint32_t level, index_t index, std::vector& data) const - { - const std::pair>& slot = map_[level][index]; - if (slot.first) { - data = slot.second; - } - return slot.first; - } - - private: - std::vector>>> map_; -}; -} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp new file mode 100644 index 00000000000..52c651c9924 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include +#include +#include + +namespace bb::crypto::merkle_tree { + +const uint32_t NUM_VALUES = 1024; +inline auto& engine = numeric::get_debug_randomness(); +inline auto& random_engine = numeric::get_randomness(); + +static std::vector VALUES = []() { + std::vector values(NUM_VALUES); + for (uint32_t i = 0; i < NUM_VALUES; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + return values; +}(); + +inline std::string randomString() +{ + std::stringstream ss; + ss << random_engine.get_random_uint256(); + return ss.str(); +} + +inline std::string randomTempDirectory() +{ + std::stringstream ss; + ss << "/tmp/lmdb/" << randomString(); + return ss.str(); +} + +inline void print_tree(const uint32_t depth, std::vector hashes, std::string const& msg) +{ + info("\n", msg); + uint32_t offset = 0; + for (uint32_t i = 0; i < depth; i++) { + info("i = ", i); + uint32_t layer_size = (1U << (depth - i)); + for (uint32_t j = 0; j < layer_size; j++) { + info("j = ", j, ": ", hashes[offset + j]); + } + offset += layer_size; + } +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp index e01622a972a..cb67ddd4592 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp @@ -1,6 +1,7 @@ #pragma once #include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" namespace bb::crypto::merkle_tree { @@ -10,7 +11,17 @@ struct indexed_leaf { index_t nextIndex; fr nextValue; - bool operator==(indexed_leaf const&) const = default; + MSGPACK_FIELDS(value, nextIndex, nextValue) + + bool operator==(indexed_leaf const& other) const + { + return value == other.value && nextValue == other.nextValue && nextIndex == other.nextIndex; + } + + // indexed_leaf operator=(indexed_leaf const& other) const + // { + // return indexed_leaf{ .value = other.value, .nextIndex = other.nextIndex, .nextValue = other.nextValue }; + // } std::ostream& operator<<(std::ostream& os) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index c5715f5192c..40b2ace0f0c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -6,10 +6,12 @@ #include "barretenberg/common/assert.hpp" #include "barretenberg/common/thread_pool.hpp" #include "indexed_leaf.hpp" +#include #include #include #include #include +#include namespace bb::crypto::merkle_tree { @@ -18,17 +20,19 @@ namespace bb::crypto::merkle_tree { * the level of the tree. * */ -class LevelSignal { +class Signal { public: - LevelSignal(uint32_t initial_level) + Signal(uint32_t initial_level) : signal_(initial_level){}; - ~LevelSignal() = default; - LevelSignal(const LevelSignal& other) + ~Signal() = default; + Signal(const Signal& other) : signal_(other.signal_.load()) {} - LevelSignal(const LevelSignal&& other) noexcept + Signal(const Signal&& other) noexcept : signal_(other.signal_.load()) {} + Signal& operator=(const Signal& other) = delete; + Signal& operator=(const Signal&& other) = delete; /** * @brief Causes the thread to wait until the required level has been signalled @@ -57,20 +61,18 @@ class LevelSignal { private: std::atomic signal_; -}; - -/** - * @brief Implements a parallelised batch insertion indexed tree - * Accepts template argument of the type of store backing the tree, the type of store containing the leaves and the - * hashing policy - * - */ -template -class IndexedTree : public AppendOnlyTree { +}; /** + * @brief Implements a parallelised batch insertion indexed tree + * Accepts template argument of the type of store backing the tree, the type of store containing the leaves and the + * hashing policy + * + */ +template class IndexedTree : public AppendOnlyTree { public: - typedef std::function&, fr&, index_t)> add_completion_callback; + using add_completion_callback = std::function&, fr&, index_t)>; + using leaf_callback = std::function; - IndexedTree(Store& store, uint32_t depth, ThreadPool& workers, index_t initial_size, std::string name); + IndexedTree(Store& store, ThreadPool& workers, index_t initial_size); IndexedTree(IndexedTree const& other) = delete; IndexedTree(IndexedTree&& other) = delete; ~IndexedTree() = default; @@ -80,7 +82,7 @@ class IndexedTree : public AppendOnlyTree { * @param value The value to be added or updated * @returns The 'previous' hash paths of all updated values */ - void add_or_update_value(const fr& value, add_completion_callback completion); + void add_or_update_value(const fr& value, const add_completion_callback& completion); /** * @brief Adds or updates the given set of values in the tree (updates not currently supported) @@ -90,22 +92,40 @@ class IndexedTree : public AppendOnlyTree { */ void add_or_update_values(const std::vector& values, const add_completion_callback& completion); - indexed_leaf get_leaf(const index_t& index); + void get_leaf(const index_t& index, bool includeUncommitted, const leaf_callback& completion); using AppendOnlyTree::get_hash_path; - using AppendOnlyTree::root; - using AppendOnlyTree::depth; private: using typename AppendOnlyTree::append_completion_callback; + using ReadTransaction = typename Store::ReadTransaction; + using ReadTransactionPtr = typename Store::ReadTransactionPtr; + + struct leaf_insertion { + index_t low_leaf_index; + indexed_leaf low_leaf; + }; - fr update_leaf_and_hash_to_root(const index_t& index, const indexed_leaf& leaf); - fr update_leaf_and_hash_to_root(const index_t& index, - const indexed_leaf& leaf, - LevelSignal& leader, - LevelSignal& follower, - fr_hash_path& previous_hash_path); - void append_subtree(const index_t& start_index, append_completion_callback& completion); + // fr update_leaf_and_hash_to_root(const index_t& index, const indexed_leaf& leaf); + void update_leaf_and_hash_to_root(const index_t& index, + const indexed_leaf& leaf, + Signal& leader, + Signal& follower, + fr_hash_path& previous_hash_path, + ReadTransaction& tx); + + using insertion_generation_callback = + std::function>, std::shared_ptr>)>; + void generate_insertions(std::shared_ptr>> values_to_be_sorted, + const insertion_generation_callback& completion); + + using insertion_completion_callback = std::function> paths)>; + void perform_insertions(std::shared_ptr> insertions, + const insertion_completion_callback& completion); + + using hash_generation_callback = std::function> hashes)>; + void generate_hashes_for_appending(std::shared_ptr> leaves_to_hash, + const hash_generation_callback& completion); using AppendOnlyTree::get_element_or_zero; using AppendOnlyTree::write_node; @@ -119,163 +139,239 @@ class IndexedTree : public AppendOnlyTree { using AppendOnlyTree::zero_hashes_; using AppendOnlyTree::depth_; using AppendOnlyTree::name_; - using AppendOnlyTree::root_; using AppendOnlyTree::workers_; - LeavesStore leaves_; }; -template -IndexedTree::IndexedTree( - Store& store, uint32_t depth, ThreadPool& workers, index_t initial_size, std::string name) - : AppendOnlyTree(store, depth, workers, name) +template +IndexedTree::IndexedTree(Store& store, ThreadPool& workers, index_t initial_size) + : AppendOnlyTree(store, workers) { ASSERT(initial_size > 0); - zero_hashes_.resize(depth + 1); + zero_hashes_.resize(depth_ + 1); // Create the zero hashes for the tree // indexed_leaf zero_leaf{ 0, 0, 0 }; auto current = fr::zero(); - for (uint32_t i = depth; i > 0; --i) { + for (uint32_t i = depth_; i > 0; --i) { zero_hashes_[i] = current; current = HashingPolicy::hash_pair(current, current); } zero_hashes_[0] = current; + index_t stored_size = 0; + bb::fr stored_root = fr::zero(); + { + ReadTransactionPtr tx = store_.createReadTransaction(); + std::string name; + uint32_t depth = 0; + store_.get_full_meta(stored_size, stored_root, name, depth, *tx, false); + } + + if (stored_size > 0) { + return; + } + + std::vector appended_leaves; + std::vector appended_hashes; // Inserts the initial set of leaves as a chain in incrementing value order for (uint32_t i = 0; i < initial_size; ++i) { // Insert the zero leaf to the `leaves` and also to the tree at index 0. - indexed_leaf initial_leaf = indexed_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }; - leaves_.append_leaf(initial_leaf); + bool last = i == (initial_size - 1); + indexed_leaf initial_leaf = + indexed_leaf{ .value = i, .nextIndex = last ? 0 : i + 1, .nextValue = last ? 0 : i + 1 }; + appended_leaves.push_back(initial_leaf); + appended_hashes.push_back(HashingPolicy::hash(initial_leaf.get_hash_inputs())); + store_.set_at_index(i, initial_leaf, true); } - // Points the last leaf back to the first - leaves_.set_at_index( - initial_size - 1, - indexed_leaf{ .value = leaves_.get_leaf(initial_size - 1).value, .nextIndex = 0, .nextValue = 0 }, - false); - - LevelSignal signal(1); + Signal signal(1); append_completion_callback completion = [&](fr, index_t) -> void { signal.signal_level(0); }; - append_subtree(0, completion); + AppendOnlyTree::add_values(appended_hashes, completion); signal.wait_for_level(0); + store_.commit(); } -template -indexed_leaf IndexedTree::get_leaf(const index_t& index) +template +void IndexedTree::get_leaf(const index_t& index, + bool includeUncommitted, + const leaf_callback& completion) { - return leaves_.get_leaf(index); + auto job = [=]() { + ReadTransactionPtr tx = store_.createReadTransaction(); + indexed_leaf leaf = store_.get_leaf(index, *tx, includeUncommitted); + completion(leaf); + }; + workers_.enqueue(job); } -template -void IndexedTree::add_or_update_value(const fr& value, - add_completion_callback completion) +template +void IndexedTree::add_or_update_value(const fr& value, const add_completion_callback& completion) { add_or_update_values(std::vector{ value }, completion); } -template -void IndexedTree::add_or_update_values(const std::vector& values, - const add_completion_callback& completion) +template +void IndexedTree::add_or_update_values(const std::vector& values, + const add_completion_callback& completion) { - // The first thing we do is sort the values into descending order but maintain knowledge of their orignal order - struct { - bool operator()(const std::pair& a, const std::pair& b) const - { - return uint256_t(a.first) > uint256_t(b.first); - } - } comp; - std::vector> values_sorted(values.size()); + std::shared_ptr>> values_to_be_sorted = + std::make_shared>>(values.size()); for (size_t i = 0; i < values.size(); ++i) { - values_sorted[i] = std::make_pair(values[i], i); + (*values_to_be_sorted)[i] = std::make_pair(values[i], i); } - std::sort(values_sorted.begin(), values_sorted.end(), comp); - // Now that we have the sorted values we need to identify the leaves that need updating. - // This is performed sequentially and is stored in this 'leaf_insertion' struct - struct leaf_insertion { - index_t low_leaf_index; - indexed_leaf low_leaf; + struct IntermediateResults { + std::shared_ptr> hashes_to_append; + std::shared_ptr> paths; + std::atomic count; + + IntermediateResults() + : count(2){}; }; + std::shared_ptr results = std::make_shared(); - std::shared_ptr> insertions = - std::make_shared>(values.size()); - index_t old_size = leaves_.get_size(); - for (size_t i = 0; i < values_sorted.size(); ++i) { - fr value = values_sorted[i].first; - index_t index_of_new_leaf = index_t(values_sorted[i].second) + old_size; - - // This gives us the leaf that need updating - index_t current = 0; - bool is_already_present = false; - std::tie(is_already_present, current) = leaves_.find_low_value(values_sorted[i].first); - indexed_leaf current_leaf = leaves_.get_leaf(current); - - indexed_leaf new_leaf = - indexed_leaf{ .value = value, .nextIndex = current_leaf.nextIndex, .nextValue = current_leaf.nextValue }; - - // We only handle new values being added. We don't yet handle values being updated - if (!is_already_present) { - // Update the current leaf to point it to the new leaf - current_leaf.nextIndex = index_of_new_leaf; - current_leaf.nextValue = value; - - leaves_.set_at_index(current, current_leaf, false); - leaves_.set_at_index(index_of_new_leaf, new_leaf, true); + append_completion_callback final_completion = [=](fr root, index_t size) { + completion(*results->paths, root, size); + }; + + hash_generation_callback hash_completion = [=](std::shared_ptr> hashes_to_append) { + results->hashes_to_append = hashes_to_append; + if (results->count.fetch_sub(1) == 1) { + AppendOnlyTree::add_values(*hashes_to_append, final_completion); } + }; - // Capture the index and value of the updated 'low' leaf - leaf_insertion& insertion = (*insertions)[i]; - insertion.low_leaf_index = current; - insertion.low_leaf = indexed_leaf{ .value = current_leaf.value, - .nextIndex = current_leaf.nextIndex, - .nextValue = current_leaf.nextValue }; - } + insertion_completion_callback insertion_completion = [=](std::shared_ptr> paths) { + results->paths = paths; + if (results->count.fetch_sub(1) == 1) { + AppendOnlyTree::add_values(*results->hashes_to_append, final_completion); + } + }; + + insertion_generation_callback insertion_generation_completed = + [=](std::shared_ptr> insertions, + std::shared_ptr> leaves_to_append) { + workers_.enqueue([=]() { generate_hashes_for_appending(leaves_to_append, hash_completion); }); + perform_insertions(insertions, insertion_completion); + }; + workers_.enqueue([=]() { generate_insertions(values_to_be_sorted, insertion_generation_completed); }); +} +template +void IndexedTree::perform_insertions(std::shared_ptr> insertions, + const insertion_completion_callback& completion) +{ // We now kick off multiple workers to perform the low leaf updates // We create set of signals to coordinate the workers as the move up the tree std::shared_ptr> paths = std::make_shared>(insertions->size()); - std::shared_ptr> signals = std::make_shared>(); + std::shared_ptr> signals = std::make_shared>(); // The first signal is set to 0. This ensures the first worker up the tree is not impeded signals->emplace_back(0); // Workers will follow their leaders up the tree, being triggered by the signal in front of them for (size_t i = 0; i < insertions->size(); ++i) { signals->emplace_back(uint32_t(1 + depth_)); } - for (uint32_t i = 0; i < insertions->size(); ++i) { auto op = [=]() { leaf_insertion& insertion = (*insertions)[i]; - update_leaf_and_hash_to_root( - insertion.low_leaf_index, insertion.low_leaf, (*signals)[i], (*signals)[i + 1], (*paths)[i]); + { + ReadTransactionPtr tx = store_.createReadTransaction(); + update_leaf_and_hash_to_root( + insertion.low_leaf_index, insertion.low_leaf, (*signals)[i], (*signals)[i + 1], (*paths)[i], *tx); + } if (i == insertions->size() - 1) { - // Now that we have updated all of the low leaves, we insert the new leaves as a subtree at the end - append_completion_callback append_completion = [=](fr root, index_t size) { - completion(*paths, root, size); - }; - append_subtree(old_size, append_completion); + completion(paths); } }; workers_.enqueue(op); } } -template -fr IndexedTree::update_leaf_and_hash_to_root(const index_t& leaf_index, - const indexed_leaf& leaf) +template +void IndexedTree::generate_hashes_for_appending( + std::shared_ptr> leaves_to_hash, const hash_generation_callback& completion) { - LevelSignal leader(0); - LevelSignal follower(0); - fr_hash_path hash_path; - return update_leaf_and_hash_to_root(leaf_index, leaf, leader, follower, hash_path); + std::shared_ptr> hashed = std::make_shared>(leaves_to_hash->size()); + std::vector& leaves = *leaves_to_hash; + for (uint32_t i = 0; i < leaves.size(); ++i) { + (*hashed)[i] = HashingPolicy::hash(leaves[i].get_hash_inputs()); + } + completion(hashed); +} + +template +void IndexedTree::generate_insertions( + std::shared_ptr>> values_to_be_sorted, + const insertion_generation_callback& on_completion) +{ + // The first thing we do is sort the values into descending order but maintain knowledge of their orignal order + struct { + bool operator()(const std::pair& a, const std::pair& b) const + { + return uint256_t(a.first) > uint256_t(b.first); + } + } comp; + std::sort(values_to_be_sorted->begin(), values_to_be_sorted->end(), comp); + + std::vector>& values = *values_to_be_sorted; + + // Now that we have the sorted values we need to identify the leaves that need updating. + // This is performed sequentially and is stored in this 'leaf_insertion' struct + std::shared_ptr> insertions = + std::make_shared>(values.size()); + std::shared_ptr> leaves_to_append = + std::make_shared>(values.size()); + index_t old_size = 0; + { + ReadTransactionPtr tx = store_.createReadTransaction(); + bb::fr old_root = fr::zero(); + store_.get_meta(old_size, old_root, *tx, true); + for (size_t i = 0; i < values.size(); ++i) { + fr value = values[i].first; + size_t index_into_appended_leaves = values[i].second; + index_t index_of_new_leaf = index_t(index_into_appended_leaves) + old_size; + + // This gives us the leaf that need updating + index_t current = 0; + bool is_already_present = false; + std::tie(is_already_present, current) = store_.find_low_value(values[i].first, true, *tx); + indexed_leaf current_leaf = store_.get_leaf(current, *tx, true); + + indexed_leaf new_leaf = indexed_leaf{ .value = value, + .nextIndex = current_leaf.nextIndex, + .nextValue = current_leaf.nextValue }; + + // We only handle new values being added. We don't yet handle values being updated + if (!is_already_present) { + // Update the current leaf to point it to the new leaf + current_leaf.nextIndex = index_of_new_leaf; + current_leaf.nextValue = value; + store_.set_at_index(current, current_leaf, false); + store_.set_at_index(index_of_new_leaf, new_leaf, true); + } + + (*leaves_to_append)[index_into_appended_leaves] = new_leaf; + + // Capture the index and value of the updated 'low' leaf as well as the new leaf to be appended + leaf_insertion& insertion = (*insertions)[i]; + insertion.low_leaf_index = current; + insertion.low_leaf = indexed_leaf{ .value = current_leaf.value, + .nextIndex = current_leaf.nextIndex, + .nextValue = current_leaf.nextValue }; + } + } + on_completion(insertions, leaves_to_append); } -template -fr IndexedTree::update_leaf_and_hash_to_root(const index_t& leaf_index, - const indexed_leaf& leaf, - LevelSignal& leader, - LevelSignal& follower, - fr_hash_path& previous_hash_path) +template +void IndexedTree::update_leaf_and_hash_to_root(const index_t& leaf_index, + const indexed_leaf& leaf, + Signal& leader, + Signal& follower, + fr_hash_path& previous_hash_path, + ReadTransaction& tx) { + auto get_node = [&](uint32_t level, index_t index) -> fr { return get_element_or_zero(level, index, tx, true); }; // We are a worker at a specific leaf index. // We are going to move up the tree and at each node/level: // 1. Wait for the level above to become 'signalled' as clear for us to write into @@ -293,8 +389,8 @@ fr IndexedTree::update_leaf_and_hash_to_root( // Extract the value of the leaf node and it's sibling bool is_right = static_cast(index & 0x01); // extract the current leaf hash values for the previous hash path - fr current_right_value = get_element_or_zero(level, index + (is_right ? 0 : 1)); - fr current_left_value = get_element_or_zero(level, is_right ? (index - 1) : index); + fr current_right_value = get_node(level, index + (is_right ? 0 : 1)); + fr current_left_value = get_node(level, is_right ? (index - 1) : index); previous_hash_path.emplace_back(current_left_value, current_right_value); // Write the new leaf hash in place @@ -313,18 +409,17 @@ fr IndexedTree::update_leaf_and_hash_to_root( // Now read the node and it's sibling index_t index_of_node_above = index >> 1; bool node_above_is_right = static_cast(index_of_node_above & 0x01); - fr above_right_value = - get_element_or_zero(level_to_read, index_of_node_above + (node_above_is_right ? 0 : 1)); - fr above_left_value = get_element_or_zero( - level_to_read, node_above_is_right ? (index_of_node_above - 1) : index_of_node_above); + fr above_right_value = get_node(level_to_read, index_of_node_above + (node_above_is_right ? 0 : 1)); + fr above_left_value = + get_node(level_to_read, node_above_is_right ? (index_of_node_above - 1) : index_of_node_above); previous_hash_path.emplace_back(above_left_value, above_right_value); } // Now that we have extracted the hash path from the row above // we can compute the new hash at that level and write it is_right = static_cast(index & 0x01); - fr new_right_value = is_right ? new_hash : get_element_or_zero(level, index + 1); - fr new_left_value = is_right ? get_element_or_zero(level, index - 1) : new_hash; + fr new_right_value = is_right ? new_hash : get_node(level, index + 1); + fr new_left_value = is_right ? get_node(level, index - 1) : new_hash; new_hash = HashingPolicy::hash_pair(new_left_value, new_right_value); index >>= 1; --level; @@ -339,23 +434,6 @@ fr IndexedTree::update_leaf_and_hash_to_root( write_node(level, index, new_hash); follower.signal_level(level); } - return new_hash; -} - -template -void IndexedTree::append_subtree(const index_t& start_index, - append_completion_callback& completion) -{ - index_t index = start_index; - size_t number_to_insert = size_t(index_t(leaves_.get_size()) - index); - std::vector hashes_to_append = std::vector(number_to_insert); - - for (size_t i = 0; i < number_to_insert; ++i) { - index_t index_to_insert = index + i; - hashes_to_append[i] = HashingPolicy::hash(leaves_.get_leaf(size_t(index_to_insert)).get_hash_inputs()); - } - - AppendOnlyTree::add_values(hashes_to_append, completion); } } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index 32a2adfd136..f1557e7b67f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -1,104 +1,284 @@ #include "indexed_tree.hpp" -#include "../array_store.hpp" +#include "../fixtures.hpp" #include "../hash.hpp" +#include "../node_store/array_store.hpp" #include "../nullifier_tree/nullifier_memory_tree.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" #include "barretenberg/common/thread_pool.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/cached_leaves_store.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" #include "barretenberg/numeric/random/engine.hpp" #include "leaves_cache.hpp" +#include +#include +#include +#include using namespace bb; using namespace bb::crypto::merkle_tree; using HashPolicy = Poseidon2HashPolicy; -namespace { -auto& engine = numeric::get_debug_randomness(); -auto& random_engine = numeric::get_randomness(); -} // namespace +using Store = CachedTreeStore; +using TreeType = IndexedTree; -const uint32_t NUM_VALUES = 1024; -static std::vector VALUES = []() { - std::vector values(NUM_VALUES); - for (uint32_t i = 0; i < NUM_VALUES; ++i) { - values[i] = fr(random_engine.get_random_uint256()); +using CompletionCallback = TreeType::add_completion_callback; + +class PersistedIndexedTreeTest : public testing::Test { + protected: + void SetUp() override + { + // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers + _directory = randomTempDirectory(); + std::filesystem::create_directories(_directory); + _environment = std::make_unique(_directory, 1, 2, 2); } - return values; -}(); -using CompletionCallback = IndexedTree::add_completion_callback; + void TearDown() override { std::filesystem::remove_all(_directory); } + + static std::string _directory; + + std::unique_ptr _environment; +}; + +std::string PersistedIndexedTreeTest::_directory; + +void check_size(TreeType& tree, index_t expected_size, bool includeUncommitted = true) +{ + Signal signal(1); + auto completion = [&](const std::string&, uint32_t, const index_t& size, const fr&) -> void { + EXPECT_EQ(size, expected_size); + signal.signal_level(0); + }; + tree.get_meta_data(includeUncommitted, completion); + signal.wait_for_level(0); +} + +fr get_root(TreeType& tree, bool includeUncommitted = true) +{ + fr r; + Signal signal(1); + auto completion = [&](const std::string&, uint32_t, const index_t&, const fr& root) -> void { + r = root; + signal.signal_level(0); + }; + tree.get_meta_data(includeUncommitted, completion); + signal.wait_for_level(0); + return r; +} + +void check_root(TreeType& tree, fr expected_root, bool includeUncommitted = true) +{ + fr root = get_root(tree, includeUncommitted); + EXPECT_EQ(root, expected_root); +} + +fr_hash_path get_hash_path(TreeType& tree, index_t index, bool includeUncommitted = true) +{ + fr_hash_path h; + Signal signal(1); + auto completion = [&](const fr_hash_path& path) -> void { + h = path; + signal.signal_level(0); + }; + tree.get_hash_path(index, completion, includeUncommitted); + signal.wait_for_level(0); + return h; +} -template -void add_value(IndexedTree& tree, fr value) +indexed_leaf get_leaf(TreeType& tree, index_t index, bool includeUncommitted = true) { - LevelSignal signal(1); - CompletionCallback completion = [&](std::vector&, fr&, index_t) { signal.signal_level(0); }; + indexed_leaf l; + Signal signal(1); + auto completion = [&](const indexed_leaf& leaf) -> void { + l = leaf; + signal.signal_level(0); + }; + tree.get_leaf(index, includeUncommitted, completion); + signal.wait_for_level(0); + return l; +} + +void check_hash_path(TreeType& tree, index_t index, fr_hash_path expected_hash_path, bool includeUncommitted = true) +{ + fr_hash_path path = get_hash_path(tree, index, includeUncommitted); + EXPECT_EQ(path, expected_hash_path); +} + +void commit_tree(TreeType& tree) +{ + Signal signal(1); + auto completion = [&]() -> void { signal.signal_level(0); }; + tree.commit(completion); + signal.wait_for_level(0); +} + +void add_value(TreeType& tree, const fr& value) +{ + Signal signal(1); + auto completion = [&](const std::vector&, fr&, index_t) -> void { signal.signal_level(0); }; + tree.add_or_update_value(value, completion); signal.wait_for_level(0); } -TEST(crypto_indexed_tree, can_create) +void add_values(TreeType& tree, const std::vector& values) +{ + Signal signal(1); + auto completion = [&](const std::vector&, fr&, index_t) -> void { signal.signal_level(0); }; + + tree.add_or_update_values(values, completion); + signal.wait_for_level(0); +} + +TEST_F(PersistedIndexedTreeTest, can_create) { + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + EXPECT_NO_THROW(Store store(name, depth, db)); + Store store(name, depth, db); ThreadPool workers(1); - ArrayStore store(10); - IndexedTree tree = - IndexedTree(store, 10, workers, 1, "Test Tree"); - EXPECT_EQ(tree.size(), 1ULL); + TreeType tree = TreeType(store, workers, 1); + check_size(tree, 1); NullifierMemoryTree memdb(10); - EXPECT_EQ(memdb.root(), tree.root()); + check_root(tree, memdb.root()); } -TEST(crypto_indexed_tree, test_size) +TEST_F(PersistedIndexedTreeTest, can_only_recreate_with_same_name_and_depth) { - ThreadPool workers(1); - ArrayStore store(32); - auto db = IndexedTree(store, 32, workers, 1, "Test Tree"); + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); - // We assume that the first leaf is already filled with (0, 0, 0). - EXPECT_EQ(db.size(), 1ULL); + EXPECT_ANY_THROW(Store store_wrong_name("Wrong name", depth, db)); + EXPECT_ANY_THROW(Store store_wrong_depth(name, depth + 1, db)); +} - // Add a new non-zero leaf at index 1. - add_value(db, 30); - EXPECT_EQ(db.size(), 2ULL); +TEST_F(PersistedIndexedTreeTest, test_size) +{ + ThreadPool workers(1); + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, 1); - // Add third. - add_value(db, 20); - EXPECT_EQ(db.size(), 3ULL); + check_size(tree, 1); - add_value(db, 10); - EXPECT_EQ(db.size(), 4ULL); + // We assume that the first leaf is already filled with (0, 0, 0). + for (uint32_t i = 0; i < 4; i++) { + add_value(tree, VALUES[i]); + check_size(tree, i + 2); + } } -TEST(crypto_indexed_tree, test_get_hash_path) +TEST_F(PersistedIndexedTreeTest, test_get_hash_path) { NullifierMemoryTree memdb(10); ThreadPool workers(1); - ArrayStore store(10); - auto db = IndexedTree(store, 10, workers, 1, "Test Tree"); - - EXPECT_EQ(memdb.root(), db.root()); + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, 1); - EXPECT_EQ(memdb.get_hash_path(0), db.get_hash_path(0)); + check_size(tree, 1); + check_root(tree, memdb.root()); + check_hash_path(tree, 0, memdb.get_hash_path(0)); memdb.update_element(VALUES[512]); - add_value(db, VALUES[512]); + add_value(tree, VALUES[512]); + + check_size(tree, 2); + check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_hash_path(tree, 1, memdb.get_hash_path(1)); - EXPECT_EQ(db.get_hash_path(0), memdb.get_hash_path(0)); + uint32_t num_to_append = 512; - for (uint32_t i = 0; i < 512; ++i) { + for (uint32_t i = 0; i < num_to_append; ++i) { + std::cout << i + 1 << std::endl; memdb.update_element(VALUES[i]); - add_value(db, VALUES[i]); + add_value(tree, VALUES[i]); } - EXPECT_EQ(db.get_hash_path(512), memdb.get_hash_path(512)); + std::cout << "Append " << num_to_append + 2 << std::endl; + check_size(tree, num_to_append + 2); + check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_hash_path(tree, 512, memdb.get_hash_path(512)); } -TEST(crypto_indexed_tree, test_batch_insert) +TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) { + NullifierMemoryTree memdb(10); + + ThreadPool workers(1); + constexpr size_t depth = 10; + std::string name = randomString(); + + { + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, 1); + + check_size(tree, 1); + check_root(tree, memdb.root()); + check_hash_path(tree, 0, memdb.get_hash_path(0)); + + add_value(tree, VALUES[512]); + + // Committed data should not have changed + check_size(tree, 1, false); + check_root(tree, memdb.root(), false); + check_hash_path(tree, 0, memdb.get_hash_path(0), false); + check_hash_path(tree, 1, memdb.get_hash_path(1), false); + + memdb.update_element(VALUES[512]); + + // Uncommitted data should have changed + check_size(tree, 2, true); + check_root(tree, memdb.root(), true); + check_hash_path(tree, 0, memdb.get_hash_path(0), true); + check_hash_path(tree, 1, memdb.get_hash_path(1), true); + + // Now commit + commit_tree(tree); + + // Now committed data should have changed + check_size(tree, 2, false); + check_root(tree, memdb.root(), false); + check_hash_path(tree, 0, memdb.get_hash_path(0), false); + check_hash_path(tree, 1, memdb.get_hash_path(1), false); + } + + // Now restore and it should continue from where we left off + { + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, 1); + + // check uncommitted state + check_size(tree, 2); + check_root(tree, memdb.root()); + check_hash_path(tree, 0, memdb.get_hash_path(0)); + + // check committed state + check_size(tree, 2, false); + check_root(tree, memdb.root(), false); + check_hash_path(tree, 0, memdb.get_hash_path(0), false); + } +} + +TEST_F(PersistedIndexedTreeTest, test_batch_insert) +{ + auto& random_engine = numeric::get_randomness(); const uint32_t batch_size = 16; const uint32_t num_batches = 16; uint32_t depth = 10; @@ -106,35 +286,37 @@ TEST(crypto_indexed_tree, test_batch_insert) ThreadPool multi_workers(8); NullifierMemoryTree memdb(depth, batch_size); - ArrayStore store1(depth); - IndexedTree tree1 = - IndexedTree(store1, depth, workers, batch_size, "Test Tree 1"); + std::string name1 = randomString(); + LMDBStore db1(*_environment, name1, false, false, IntegerKeyCmp); + Store store1(name1, depth, db1); + auto tree1 = TreeType(store1, workers, batch_size); - ArrayStore store2(depth); - IndexedTree tree2 = - IndexedTree(store2, depth, workers, batch_size, "Test Tree 2"); + std::string name2 = randomString(); + LMDBStore db2(*_environment, name2, false, false, IntegerKeyCmp); + Store store2(name2, depth, db2); + auto tree2 = TreeType(store2, workers, batch_size); - EXPECT_EQ(memdb.root(), tree1.root()); - EXPECT_EQ(tree1.root(), tree2.root()); + check_root(tree1, memdb.root()); + check_root(tree2, memdb.root()); - EXPECT_EQ(memdb.get_hash_path(0), tree1.get_hash_path(0)); - EXPECT_EQ(tree1.get_hash_path(0), tree2.get_hash_path(0)); + check_hash_path(tree1, 0, memdb.get_hash_path(0)); + check_hash_path(tree2, 0, memdb.get_hash_path(0)); - EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); - EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); + check_hash_path(tree1, 512, memdb.get_hash_path(512)); + check_hash_path(tree2, 512, memdb.get_hash_path(512)); for (uint32_t i = 0; i < num_batches; i++) { std::vector batch; std::vector memory_tree_hash_paths; for (uint32_t j = 0; j < batch_size; j++) { - batch.push_back(fr(random_engine.get_random_uint256())); + batch.emplace_back(random_engine.get_random_uint256()); fr_hash_path path = memdb.update_element(batch[j]); memory_tree_hash_paths.push_back(path); } std::vector tree1_hash_paths; std::vector tree2_hash_paths; { - LevelSignal signal(1); + Signal signal(1); CompletionCallback completion = [&](std::vector& hash_paths, fr&, index_t) { tree1_hash_paths = hash_paths; signal.signal_level(0); @@ -143,7 +325,7 @@ TEST(crypto_indexed_tree, test_batch_insert) signal.wait_for_level(0); } { - LevelSignal signal(1); + Signal signal(1); CompletionCallback completion = [&](std::vector& hash_paths, fr&, index_t) { tree2_hash_paths = hash_paths; signal.signal_level(0); @@ -151,18 +333,17 @@ TEST(crypto_indexed_tree, test_batch_insert) tree2.add_or_update_values(batch, completion); signal.wait_for_level(0); } - EXPECT_EQ(memdb.root(), tree1.root()); - EXPECT_EQ(tree1.root(), tree2.root()); + check_root(tree1, memdb.root()); + check_root(tree2, memdb.root()); - EXPECT_EQ(memdb.get_hash_path(0), tree1.get_hash_path(0)); - EXPECT_EQ(tree1.get_hash_path(0), tree2.get_hash_path(0)); + check_hash_path(tree1, 0, memdb.get_hash_path(0)); + check_hash_path(tree2, 0, memdb.get_hash_path(0)); - EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); - EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); + check_hash_path(tree1, 512, memdb.get_hash_path(512)); + check_hash_path(tree2, 512, memdb.get_hash_path(512)); for (uint32_t j = 0; j < batch_size; j++) { EXPECT_EQ(tree1_hash_paths[j], tree2_hash_paths[j]); - // EXPECT_EQ(tree1_hash_paths[j], memory_tree_hash_paths[j]); } } } @@ -172,8 +353,10 @@ fr hash_leaf(const indexed_leaf& leaf) return HashPolicy::hash(leaf.get_hash_inputs()); } -bool check_hash_path(const fr& root, const fr_hash_path& path, const indexed_leaf& leaf_value, const uint32_t idx) +bool verify_hash_path(TreeType& tree, const indexed_leaf& leaf_value, const uint32_t idx) { + fr root = get_root(tree, true); + fr_hash_path path = get_hash_path(tree, idx, true); auto current = hash_leaf(leaf_value); uint32_t depth_ = static_cast(path.size()); uint32_t index = idx; @@ -186,14 +369,15 @@ bool check_hash_path(const fr& root, const fr_hash_path& path, const indexed_lea return current == root; } -TEST(crypto_indexed_tree, test_indexed_memory) +TEST_F(PersistedIndexedTreeTest, test_indexed_memory) { ThreadPool workers(8); // Create a depth-3 indexed merkle tree - constexpr uint32_t depth = 3; - ArrayStore store(depth); - IndexedTree tree = - IndexedTree(store, depth, workers, 1, "Test Tree 1"); + constexpr size_t depth = 3; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, 1); /** * Intial state: @@ -205,8 +389,10 @@ TEST(crypto_indexed_tree, test_indexed_memory) * nextVal 0 0 0 0 0 0 0 0 */ indexed_leaf zero_leaf = { 0, 0, 0 }; - EXPECT_EQ(tree.size(), 1); - EXPECT_EQ(tree.get_leaf(0), zero_leaf); + check_size(tree, 1); + EXPECT_EQ(get_leaf(tree, 0), zero_leaf); + + std::cout << "1" << std::endl; /** * Add new value 30: @@ -218,9 +404,16 @@ TEST(crypto_indexed_tree, test_indexed_memory) * nextVal 30 0 0 0 0 0 0 0 */ add_value(tree, 30); - EXPECT_EQ(tree.size(), 2); - EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 1, 30 })); - EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + std::cout << "After add" << std::endl; + check_size(tree, 2); + indexed_leaf test = { 0, 1, 30 }; + EXPECT_EQ(get_leaf(tree, 0), test); + test = { 30, 0, 0 }; + EXPECT_EQ(get_leaf(tree, 1), test); + EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 1, 30 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 0, 0 })); + + std::cout << "2" << std::endl; /** * Add new value 10: @@ -232,10 +425,14 @@ TEST(crypto_indexed_tree, test_indexed_memory) * nextVal 10 0 30 0 0 0 0 0 */ add_value(tree, 10); - EXPECT_EQ(tree.size(), 3); - EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); - EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); - EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 1, 30 })); + check_size(tree, 3); + test = { 0, 2, 10 }; + EXPECT_EQ(get_leaf(tree, 0), test); + EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf({ 10, 1, 30 })); + + std::cout << "3" << std::endl; /** * Add new value 20: @@ -247,11 +444,15 @@ TEST(crypto_indexed_tree, test_indexed_memory) * nextVal 10 0 20 30 0 0 0 0 */ add_value(tree, 20); - EXPECT_EQ(tree.size(), 4); - EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); - EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); - EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 3, 20 })); - EXPECT_EQ(hash_leaf(tree.get_leaf(3)), hash_leaf({ 20, 1, 30 })); + check_size(tree, 4); + test = { 0, 2, 10 }; + EXPECT_EQ(get_leaf(tree, 0), test); + EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf({ 10, 3, 20 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 3)), hash_leaf({ 20, 1, 30 })); + + std::cout << "4" << std::endl; // Adding the same value must not affect anything // tree.update_element(20); @@ -271,19 +472,21 @@ TEST(crypto_indexed_tree, test_indexed_memory) * nextVal 10 50 20 30 0 0 0 0 */ add_value(tree, 50); - EXPECT_EQ(tree.size(), 5); - EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); - EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 4, 50 })); - EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 3, 20 })); - EXPECT_EQ(hash_leaf(tree.get_leaf(3)), hash_leaf({ 20, 1, 30 })); - EXPECT_EQ(hash_leaf(tree.get_leaf(4)), hash_leaf({ 50, 0, 0 })); + check_size(tree, 5); + EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 4, 50 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf({ 10, 3, 20 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 3)), hash_leaf({ 20, 1, 30 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 4)), hash_leaf({ 50, 0, 0 })); + + std::cout << "5" << std::endl; // Manually compute the node values - auto e000 = hash_leaf(tree.get_leaf(0)); - auto e001 = hash_leaf(tree.get_leaf(1)); - auto e010 = hash_leaf(tree.get_leaf(2)); - auto e011 = hash_leaf(tree.get_leaf(3)); - auto e100 = hash_leaf(tree.get_leaf(4)); + auto e000 = hash_leaf(get_leaf(tree, 0)); + auto e001 = hash_leaf(get_leaf(tree, 1)); + auto e010 = hash_leaf(get_leaf(tree, 2)); + auto e011 = hash_leaf(get_leaf(tree, 3)); + auto e100 = hash_leaf(get_leaf(tree, 4)); auto e101 = fr::zero(); auto e110 = fr::zero(); auto e111 = fr::zero(); @@ -304,16 +507,16 @@ TEST(crypto_indexed_tree, test_indexed_memory) std::make_pair(e00, e01), std::make_pair(e0, e1), }; - EXPECT_EQ(tree.get_hash_path(0), expected); - EXPECT_EQ(tree.get_hash_path(1), expected); - fr_hash_path expected2 = { + check_hash_path(tree, 0, expected); + check_hash_path(tree, 1, expected); + expected = { std::make_pair(e010, e011), std::make_pair(e00, e01), std::make_pair(e0, e1), }; - EXPECT_EQ(tree.get_hash_path(2), expected2); - EXPECT_EQ(tree.get_hash_path(3), expected2); - EXPECT_EQ(tree.root(), root); + check_hash_path(tree, 2, expected); + check_hash_path(tree, 3, expected); + check_root(tree, root); // Check the hash path at index 6 and 7 expected = { @@ -321,48 +524,92 @@ TEST(crypto_indexed_tree, test_indexed_memory) std::make_pair(e10, e11), std::make_pair(e0, e1), }; - EXPECT_EQ(tree.get_hash_path(6), expected); - EXPECT_EQ(tree.get_hash_path(7), expected); + check_hash_path(tree, 6, expected); + check_hash_path(tree, 7, expected); } -// TEST(crypto_indexed_tree, test_indexed_tree) -// { -// // Create a depth-8 indexed merkle tree -// constexpr uint32_t depth = 8; -// ArrayStore store(depth); -// IndexedTree tree = -// IndexedTree(store, depth, 1); - -// indexed_leaf zero_leaf = { 0, 0, 0 }; -// EXPECT_EQ(tree.size(), 1); -// EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf(zero_leaf)); - -// // Add 20 random values to the tree -// for (uint32_t i = 0; i < 20; i++) { -// auto value = fr::random_element(); -// tree.add_value(value); -// } - -// auto abs_diff = [](uint256_t a, uint256_t b) { -// if (a > b) { -// return (a - b); -// } else { -// return (b - a); -// } -// }; - -// // Check if a new random value is not a member of this tree. -// fr new_member = fr::random_element(); -// std::vector differences; -// for (uint32_t i = 0; i < uint32_t(tree.size()); i++) { -// uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); -// uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); -// differences.push_back(diff_hi + diff_lo); -// } -// auto it = std::min_element(differences.begin(), differences.end()); -// auto index = static_cast(it - differences.begin()); - -// // Merkle proof at `index` proves non-membership of `new_member` -// auto hash_path = tree.get_hash_path(index); -// EXPECT_TRUE(check_hash_path(tree.root(), hash_path, tree.get_leaf(index), index)); -// } \ No newline at end of file +TEST_F(PersistedIndexedTreeTest, test_indexed_tree) +{ + ThreadPool workers(1); + // Create a depth-8 indexed merkle tree + constexpr uint32_t depth = 8; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, 1); + + indexed_leaf zero_leaf = { 0, 0, 0 }; + check_size(tree, 1); + EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(zero_leaf)); + + // Add 20 random values to the tree + for (uint32_t i = 0; i < 20; i++) { + auto value = fr::random_element(); + add_value(tree, value); + } + + auto abs_diff = [](uint256_t a, uint256_t b) { + if (a > b) { + return (a - b); + } else { + return (b - a); + } + }; + + check_size(tree, 21); + + // Check if a new random value is not a member of this tree. + fr new_member = fr::random_element(); + std::vector differences; + for (uint32_t i = 0; i < uint32_t(21); i++) { + uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(get_leaf(tree, i).value)); + uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(get_leaf(tree, i).value)); + differences.push_back(diff_hi + diff_lo); + } + auto it = std::min_element(differences.begin(), differences.end()); + auto index = static_cast(it - differences.begin()); + + // Merkle proof at `index` proves non-membership of `new_member` + EXPECT_TRUE(verify_hash_path(tree, get_leaf(tree, index), index)); +} + +TEST_F(PersistedIndexedTreeTest, can_add_single_whilst_reading) +{ + constexpr size_t depth = 10; + NullifierMemoryTree memdb(10); + fr_hash_path initial_path = memdb.get_hash_path(0); + memdb.update_element(VALUES[0]); + fr_hash_path final_hash_path = memdb.get_hash_path(0); + + uint32_t num_reads = 16 * 1024; + std::vector paths(num_reads); + + { + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + ThreadPool pool(8); + TreeType tree(store, pool, 1); + + check_size(tree, 1); + + Signal signal(2); + + auto add_completion = [&](const std::vector&, fr&, index_t) { + signal.signal_level(1); + auto commit_completion = [&]() { signal.signal_level(0); }; + tree.commit(commit_completion); + }; + tree.add_or_update_value(VALUES[0], add_completion); + + for (size_t i = 0; i < num_reads; i++) { + auto completion = [&, i](const fr_hash_path& path) { paths[i] = path; }; + tree.get_hash_path(0, completion, false); + } + signal.wait_for_level(0); + } + + for (auto& path : paths) { + EXPECT_TRUE(path == initial_path || path == final_hash_path); + } +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp new file mode 100644 index 00000000000..a9a6ea3615d --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp @@ -0,0 +1,127 @@ +#include "lmdb_store.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/numeric/uint128/uint128.hpp" +#include "barretenberg/numeric/uint256/uint256.hpp" +#include +#include +#include +#include +#include + +namespace bb::crypto::merkle_tree { + +int SizeCmp(const MDB_val* a, const MDB_val* b) +{ + if (a->mv_size < b->mv_size) { + return -1; + } + if (a->mv_size > b->mv_size) { + return 1; + } + return 0; +} + +int Invalid(const MDB_val*, const MDB_val*) +{ + throw std::runtime_error("Invalid comparison"); +} + +int IntegerKeyCmp(const MDB_val* a, const MDB_val* b) +{ + // std::cout << "A size " << a->mv_size << " B size " << b->mv_size << std::endl; + if (a->mv_size != b->mv_size) { + return SizeCmp(a, b); + } + + using f_type = std::function; + static std::vector functions{ + ValueCmp, ValueCmp, ValueCmp, Invalid, ValueCmp + }; + return functions[a->mv_size / 8](a, b); +} + +LMDBEnvironment::LMDBEnvironment(const std::string& directory, + uint64_t mapSizeMB, + uint32_t maxNumDBs, + uint32_t maxNumReaders) + : _maxReaders(maxNumReaders) + , _numReaders(0) +{ + if (!call_lmdb_func(mdb_env_create, &_mdbEnv)) { + // throw here + } + uint64_t kb = 1024; + uint64_t totalMapSize = kb * kb * mapSizeMB; + call_lmdb_func(mdb_env_set_mapsize, _mdbEnv, totalMapSize); + call_lmdb_func(mdb_env_set_maxdbs, _mdbEnv, static_cast(maxNumDBs)); + call_lmdb_func(mdb_env_set_maxreaders, _mdbEnv, maxNumReaders); + uint32_t flags = MDB_NOTLS; + if (!call_lmdb_func( + mdb_env_open, _mdbEnv, directory.c_str(), flags, static_cast(S_IRWXU | S_IRWXG | S_IRWXO))) { + call_lmdb_func(mdb_env_close, _mdbEnv); + // throw here + } +} + +void LMDBEnvironment::waitForReader() +{ + std::unique_lock lock(_readersLock); + if (_numReaders >= _maxReaders) { + _readersCondition.wait(lock, [&] { return _numReaders < _maxReaders; }); + } + ++_numReaders; +} + +void LMDBEnvironment::releaseReader() +{ + std::unique_lock lock(_readersLock); + --_numReaders; + _readersCondition.notify_one(); +} + +void LMDBWriteTransaction::put_node(uint32_t level, index_t index, std::vector& data) +{ + NodeKeyType key = (static_cast(1 << level) + static_cast(index)) - 1; + put_value_by_integer(key, data); +} + +void LMDBWriteTransaction::put_value(std::vector& key, std::vector& data) +{ + MDB_val dbKey; + dbKey.mv_size = key.size(); + dbKey.mv_data = (void*)key.data(); + + MDB_val dbVal; + dbVal.mv_size = data.size(); + dbVal.mv_data = (void*)data.data(); + call_lmdb_func(mdb_put, underlying(), _database.underlying(), &dbKey, &dbVal, 0U); +} + +bool LMDBReadTransaction::get_value(std::vector& key, std::vector& data) const +{ + MDB_val dbKey; + dbKey.mv_size = key.size(); + dbKey.mv_data = (void*)key.data(); + + MDB_val dbVal; + if (!call_lmdb_func(mdb_get, underlying(), _database.underlying(), &dbKey, &dbVal)) { + return false; + } + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + return true; +} + +bool LMDBReadTransaction::get_node(uint32_t level, index_t index, std::vector& data) const +{ + NodeKeyType key = (static_cast(1 << level) + static_cast(index)) - 1; + return get_value_by_integer(key, data); +} + +LMDBReadTransaction::Ptr LMDBStore::createReadTransaction() +{ + _environment.waitForReader(); + return std::make_unique(_environment, _database); +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp new file mode 100644 index 00000000000..cc0a1f04d9b --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp @@ -0,0 +1,340 @@ +#pragma once +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/numeric/uint256/uint256.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bb::crypto::merkle_tree { + +using NodeKeyType = uint128_t; +using LeafIndexKeyType = uint64_t; +using FrKeyType = uint256_t; +using MetaKeyType = uint8_t; + +int SizeCmp(const MDB_val* a, const MDB_val* b); + +int Invalid(const MDB_val*, const MDB_val*); + +template int ValueCmp(const MDB_val* a, const MDB_val* b) +{ + const T* lhs = static_cast(a->mv_data); + const T* rhs = static_cast(b->mv_data); + if (*lhs < *rhs) { + return -1; + } + if (*lhs > *rhs) { + return 1; + } + return 0; +} + +int IntegerKeyCmp(const MDB_val* a, const MDB_val* b); + +template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) +{ + int error = f(args...); + if (error != 0 && error != MDB_NOTFOUND) { + std::stringstream ss; + ss << "ERROR: " << mdb_strerror(error) << std::endl; + std::cout << ss.str(); + } + return error == 0; +} + +template void call_lmdb_func(void (*f)(TArgs...), TArgs... args) +{ + f(args...); +} + +class LMDBEnvironment { + public: + LMDBEnvironment(const std::string& directory, uint64_t mapSizeMB, uint32_t maxNumDBs, uint32_t maxNumReaders); + + ~LMDBEnvironment() { call_lmdb_func(mdb_env_close, _mdbEnv); } + + MDB_env* underlying() const { return _mdbEnv; } + + void waitForReader(); + + void releaseReader(); + + private: + MDB_env* _mdbEnv; + uint32_t _maxReaders; + uint32_t _numReaders; + std::mutex _readersLock; + std::condition_variable _readersCondition; +}; + +class LMDBDatabase; + +class LMDBTransaction { + public: + LMDBTransaction(LMDBEnvironment& env, bool readOnly = false) + : _environment(env) + { + MDB_txn* p = nullptr; + if (!call_lmdb_func(mdb_txn_begin, _environment.underlying(), p, readOnly ? MDB_RDONLY : 0U, &_transaction)) { + // throw here + } + } + + virtual ~LMDBTransaction() = default; + + MDB_txn* underlying() const { return _transaction; } + + protected: + LMDBEnvironment& _environment; + MDB_txn* _transaction; +}; + +class LMDBDatabase { + public: + LMDBDatabase(const LMDBEnvironment& env, + const LMDBTransaction& transaction, + const std::string& name, + bool integerKeys = false, + bool reverseKeys = false, + MDB_cmp_func* cmp = nullptr) + : _environment(env) + { + unsigned int flags = MDB_CREATE; + if (integerKeys) { + flags |= MDB_INTEGERKEY; + } + if (reverseKeys) { + flags |= MDB_REVERSEKEY; + } + if (!call_lmdb_func(mdb_dbi_open, transaction.underlying(), name.c_str(), flags, &_dbi)) { + // throw here + } + if (cmp != nullptr) { + if (!call_lmdb_func(mdb_set_compare, transaction.underlying(), _dbi, cmp)) { + // throw here + } + } + } + + ~LMDBDatabase() { call_lmdb_func(mdb_dbi_close, _environment.underlying(), _dbi); } + + const MDB_dbi& underlying() const { return _dbi; } + + private: + MDB_dbi _dbi; + const LMDBEnvironment& _environment; +}; + +class LMDBReadTransaction : public LMDBTransaction { + public: + using Ptr = std::unique_ptr; + + LMDBReadTransaction(LMDBEnvironment& env, const LMDBDatabase& database) + : LMDBTransaction(env, true) + , _database(database) + {} + + ~LMDBReadTransaction() override { abort(); } + + template bool get_value_or_previous(T& key, std::vector& data) const; + + bool get_node(uint32_t level, index_t index, std::vector& data) const; + + template bool get_value_by_integer(T& key, std::vector& data) const; + + bool get_value(std::vector& key, std::vector& data) const; + + protected: + const LMDBDatabase& _database; + + void abort() + { + call_lmdb_func(mdb_txn_abort, _transaction); + _environment.releaseReader(); + } +}; + +template bool LMDBReadTransaction::get_value_by_integer(T& key, std::vector& data) const +{ + MDB_val dbKey; + dbKey.mv_size = sizeof(T); + dbKey.mv_data = (void*)&key; + + MDB_val dbVal; + if (!call_lmdb_func(mdb_get, underlying(), _database.underlying(), &dbKey, &dbVal)) { + return false; + } + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + return true; +} + +template bool LMDBReadTransaction::get_value_or_previous(T& key, std::vector& data) const +{ + T keyCopy = key; + MDB_cursor* cursor; + call_lmdb_func(mdb_cursor_open, underlying(), _database.underlying(), &cursor); + + MDB_val dbKey; + dbKey.mv_size = sizeof(T); + dbKey.mv_data = (void*)&keyCopy; + + MDB_val dbVal; + + bool success = false; + + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + if (code == 0) { + // we found the key, now determine if it is the exact key + if (dbKey.mv_size == sizeof(T) && std::memcmp(dbKey.mv_data, &key, dbKey.mv_size) == 0) { + // we have the exact key + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + success = true; + } else { + // We have a key of the same size but larger value OR a larger size + // either way we now need to find the previous key + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); + if (code == 0) { + // We have found a previous key. It could be of the same size but smaller value, or smaller size which + // is equal to not found + if (dbKey.mv_size != sizeof(T)) { + // There is no previous key, do nothing + } else { + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + std::memcpy(&key, dbKey.mv_data, dbKey.mv_size); + success = true; + } + } else if (code == MDB_NOTFOUND) { + // There is no previous key, do nothing + } else { + std::stringstream ss; + ss << "ERROR: " << mdb_strerror(code) << std::endl; + std::cout << ss.str(); + } + } + } else if (code == MDB_NOTFOUND) { + // The key was not found, use the last key in the db + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); + if (code == 0) { + // We found the last key, but we need to ensure it is the same size + if (dbKey.mv_size != sizeof(T)) { + // The key is not the same size, same as not found, do nothing + } else { + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + std::memcpy(&key, dbKey.mv_data, dbKey.mv_size); + success = true; + } + } else if (code == MDB_NOTFOUND) { + // DB is empty? + } else { + std::stringstream ss; + ss << "ERROR: " << mdb_strerror(code) << std::endl; + std::cout << ss.str(); + } + } else { + std::stringstream ss; + ss << "ERROR: " << mdb_strerror(code) << std::endl; + std::cout << ss.str(); + } + call_lmdb_func(mdb_cursor_close, cursor); + return success; +} + +class LMDBWriteTransaction : public LMDBTransaction { + public: + using Ptr = std::unique_ptr; + + LMDBWriteTransaction(LMDBEnvironment& env, const LMDBDatabase& database) + : LMDBTransaction(env) + , _database(database) + {} + + ~LMDBWriteTransaction() override { commit(); } + + void put_node(uint32_t level, index_t index, std::vector& data); + + template void put_value_by_integer(T& key, std::vector& data); + + void put_value(std::vector& key, std::vector& data); + + bool commit() + { + if (_committed) { + return true; + } + _committed = call_lmdb_func(mdb_txn_commit, _transaction); + return _committed; + } + + protected: + const LMDBDatabase& _database; + bool _committed{}; +}; + +template void LMDBWriteTransaction::put_value_by_integer(T& key, std::vector& data) +{ + MDB_val dbKey; + dbKey.mv_size = sizeof(T); + dbKey.mv_data = &key; + + MDB_val dbVal; + dbVal.mv_size = data.size(); + dbVal.mv_data = (void*)data.data(); + call_lmdb_func(mdb_put, underlying(), _database.underlying(), &dbKey, &dbVal, 0U); +} + +class LMDBDatabaseCreationTransaction : public LMDBTransaction { + public: + using Ptr = std::unique_ptr; + + LMDBDatabaseCreationTransaction(LMDBEnvironment& env) + : LMDBTransaction(env) + {} + + ~LMDBDatabaseCreationTransaction() override { commit(); } + + private: + bool commit() { return call_lmdb_func(mdb_txn_commit, _transaction); } +}; + +class LMDBStore { + + public: + using ReadTransaction = LMDBReadTransaction; + using WriteTransaction = LMDBWriteTransaction; + LMDBStore(LMDBEnvironment& environment, + std::string name, + bool integerKeys = false, + bool reverseKeys = false, + MDB_cmp_func* cmp = nullptr) + : _environment(environment) + , _name(std::move(name)) + , _database(_environment, LMDBDatabaseCreationTransaction(_environment), _name, integerKeys, reverseKeys, cmp) + {} + ~LMDBStore() = default; + + LMDBWriteTransaction::Ptr createWriteTransaction() const + { + return std::make_unique(_environment, _database); + } + LMDBReadTransaction::Ptr createReadTransaction(); + + private: + LMDBEnvironment& _environment; + const std::string _name; + LMDBDatabase _database; +}; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp new file mode 100644 index 00000000000..7747d7721dc --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp @@ -0,0 +1,476 @@ +#include +#include + +#include +#include +#include +#include + +#include "barretenberg/common/serialize.hpp" +#include "barretenberg/common/streams.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/crypto/merkle_tree/fixtures.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include "barretenberg/numeric/uint128/uint128.hpp" +#include "barretenberg/numeric/uint256/uint256.hpp" +#include "barretenberg/stdlib/primitives/field/field.hpp" +#include "lmdb_store.hpp" + +using namespace bb::stdlib; +using namespace bb::crypto::merkle_tree; + +using Builder = bb::UltraCircuitBuilder; + +using field_ct = field_t; +using witness_ct = witness_t; + +const int SAMPLE_DATA_SIZE = 1024; + +class LMDBStoreTest : public testing::Test { + protected: + void SetUp() override + { + // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers + _directory = randomTempDirectory(); + std::filesystem::create_directories(_directory); + _environment = std::make_unique(_directory, 1, 2, 2); + } + + void TearDown() override { std::filesystem::remove_all(_directory); } + + static std::string _directory; + + std::unique_ptr _environment; +}; + +std::string LMDBStoreTest::_directory; + +TEST_F(LMDBStoreTest, can_write_to_and_read_from_store) +{ + { + LMDBStore store(*_environment, "DB1"); + { + std::vector buf; + write(buf, VALUES[0]); + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + transaction->put_node(0, 0, buf); + } + + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector buf2; + bool success = transaction->get_node(0, 0, buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[0]); + } + } + + { + LMDBStore store(*_environment, "DB2"); + { + std::vector key; + write(key, VALUES[0]); + std::vector value; + write(value, VALUES[1]); + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + transaction->put_value(key, value); + } + + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector key; + write(key, VALUES[0]); + std::vector value; + bool success = transaction->get_value(key, value); + EXPECT_EQ(success, true); + bb::fr v = from_buffer(value, 0); + EXPECT_EQ(v, VALUES[1]); + } + } +} + +TEST_F(LMDBStoreTest, reading_an_empty_key_reports_correctly) +{ + { + LMDBStore store(*_environment, "DB1"); + + { + std::vector buf; + write(buf, VALUES[0]); + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + transaction->put_node(0, 0, buf); + } + + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector buf2; + bool success = transaction->get_node(0, 1, buf2); + EXPECT_EQ(success, false); + } + } + + { + LMDBStore store(*_environment, "DB2"); + + { + std::vector key; + write(key, VALUES[0]); + std::vector value; + write(value, VALUES[1]); + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + transaction->put_value(key, value); + } + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector key2; + write(key2, VALUES[5]); + std::vector value; + bool success = transaction->get_value(key2, value); + EXPECT_EQ(success, false); + } + } +} + +TEST_F(LMDBStoreTest, can_write_and_read_multiple) +{ + + { + LMDBStore store(*_environment, "DB1"); + + { + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf; + write(buf, VALUES[i]); + transaction->put_node(10, i, buf); + } + } + + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf2; + bool success = transaction->get_node(10, i, buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[i]); + } + } + } + + { + LMDBStore store(*_environment, "DB2"); + uint32_t num_reads = 128; + + { + + for (size_t i = 0; i < num_reads; i++) { + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + std::vector key; + write(key, VALUES[i]); + std::vector buf; + write(buf, VALUES[i + 128]); + transaction->put_value(key, buf); + } + } + + { + for (size_t i = 0; i < num_reads; i++) { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector key; + write(key, VALUES[i]); + std::vector buf2; + bool success = transaction->get_value(key, buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[i + 128]); + } + } + } +} + +TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) +{ + + // This test is performed using integer keys of uint128_t + // We also add keys of different sizes but with larger and smaller values + // This ensures we don't erroneously return keys of different sizes + LMDBStore store(*_environment, "note hash tree", false, false, IntegerKeyCmp); + + std::vector values{ 1, 2, 3, 4, 5 }; + + auto& random_engine = bb::numeric::get_randomness(); + uint32_t num_keys = static_cast(values.size()); + // ensure first key is at least 100 + uint128_t key = random_engine.get_random_uint32() + 100; + std::vector keys; + for (uint32_t i = 0; i < num_keys; i++) { + keys.push_back(key); + // ensure space of at least 50 + key += random_engine.get_random_uint32(); + key += 50; + } + + { + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + for (size_t i = 0; i < num_keys; i++) { + std::vector value; + write(value, values[i]); + transaction->put_value_by_integer(keys[i], value); + } + + // Now put keys in the db that are smaller and larger in length and smaller and larger in value + // First key is at least 100 + uint64_t lower64 = static_cast(keys[0]) - 50; + uint64_t higher64 = static_cast(keys[4]) + 50; + uint256_t lower256 = uint256_t::from_uint128(keys[0]) - 50; + uint256_t higher256 = uint256_t::from_uint128(keys[4]) + 50; + uint32_t value_to_write = 6; + std::vector value; + write(value, value_to_write); + transaction->put_value_by_integer(lower64, value); + transaction->put_value_by_integer(higher64, value); + transaction->put_value_by_integer(lower256, value); + transaction->put_value_by_integer(higher256, value); + } + + { + // Values are at keys keys[0] -> keys[4] + + // First look for the value at each key, should return the exact keys + for (uint32_t i = 0; i < num_keys; i++) { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector data; + uint128_t key_copy = keys[i]; + bool success = transaction->get_value_or_previous(key_copy, data); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(data, 0); + EXPECT_EQ(value, values[i]); + // uint256_t new_key = from_buffer(key, 0); + EXPECT_EQ(key_copy, keys[i]); + } + + // Now look for the value at key <= key[1] + 5 (does not exist), should be at keys[1] + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector data; + uint128_t key = keys[1] + 5; + bool success = transaction->get_value_or_previous(key, data); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(data, 0); + EXPECT_EQ(value, values[1]); + // uint256_t new_key = from_buffer(key, 0); + EXPECT_EQ(key, keys[1]); + } + + // Now look for the value at key <= keys[4] + 5 (beyond the range of the current set of keys), should be at + // keys[4] + + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector data; + uint128_t key = keys[4] + 5; + bool success = transaction->get_value_or_previous(key, data); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(data, 0); + EXPECT_EQ(value, values[4]); + EXPECT_EQ(key, keys[4]); + } + + // Now look for the value at key <= keys[0] - 5 (does not exist, less than first key), should not exist + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector data; + uint128_t key = keys[0] - 5; + bool success = transaction->get_value_or_previous(key, data); + EXPECT_EQ(success, false); + } + } +} + +TEST_F(LMDBStoreTest, can_not_retrieve_previous_key_from_empty_db) +{ + LMDBStore store(*_environment, "note hash tree", false, false); + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + uint128_t key = 20; + std::vector data; + bool success = transaction->get_value_by_integer(key, data); + EXPECT_EQ(success, false); +} + +TEST_F(LMDBStoreTest, can_write_and_read_at_random_keys) +{ + LMDBStore store(*_environment, "note hash tree"); + + std::vector keys; + + { + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf; + write(buf, VALUES[i]); + size_t key = static_cast(rand() % 10000000); + keys.push_back(key); + transaction->put_node(0, key, buf); + } + } + + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf2; + bool success = transaction->get_node(0, keys[i], buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[i]); + } + } +} + +TEST_F(LMDBStoreTest, can_recreate_the_store_and_use_again) +{ + std::vector keys; + { + LMDBStore store(*_environment, "note hash tree"); + + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf; + write(buf, VALUES[i]); + size_t key = static_cast(rand() % 10000000); + keys.push_back(key); + transaction->put_node(0, key, buf); + } + } + + { + LMDBStore store(*_environment, "note hash tree"); + + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { + std::vector buf2; + bool success = transaction->get_node(0, keys[i], buf2); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf2, 0); + EXPECT_EQ(value, VALUES[i]); + } + } +} + +void read_loop(LMDBStore& store, size_t key, std::atomic& flag, bb::fr starting_value) +{ + bool seen = false; + while (true) { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + std::vector buf; + bool success = transaction->get_node(0, key, buf); + EXPECT_EQ(success, true); + bb::fr value = from_buffer(buf, 0); + if (value == starting_value && !seen) { + // acknowledge that we have seen the old value + flag--; + seen = true; + } + if (value == starting_value + bb::fr(1)) { + // exit now that we have seen the new value + break; + } + } +} + +TEST_F(LMDBStoreTest, can_read_from_multiple_threads) +{ + LMDBStore store(*_environment, "note hash tree"); + const int num_threads = 50; + + size_t key = static_cast(rand() % 1000000); + + { + // we write VALUES[0] to a slot + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + std::vector buf; + write(buf, VALUES[0]); + transaction->put_node(0, key, buf); + } + + { + // we setup multiple threads to read the slot and check they shutdown when the value changes + std::vector threads; + std::atomic flag = num_threads; + for (size_t i = 0; i < num_threads; i++) { + threads.emplace_back(read_loop, std::ref(store), key, std::ref(flag), VALUES[0]); + } + // wait until all threads have seen the old value + while (flag != 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + { + // we write VALUES[0] + 1 to the slot + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + std::vector buf; + write(buf, VALUES[0] + 1); + transaction->put_node(0, key, buf); + } + // now wait for all threads to exit having seen the new value + for (size_t i = 0; i < num_threads; i++) { + threads[i].join(); + } + } +} + +TEST_F(LMDBStoreTest, can_handle_different_key_spaces) +{ + LMDBStore store(*_environment, "DB1", false, false, IntegerKeyCmp); + + // create a set of keys + std::vector keys{ 10, 20, 30, 40, 50 }; + + // create values for each key space + std::vector> values{ + { 10, 20, 30, 40, 50 }, + { 100, 200, 300, 400, 500 }, + { 1000, 2000, 3000, 4000, 5000 }, + { 10000, 20000, 30000, 40000, 50000 }, + }; + + auto write_values = [&](LMDBWriteTransaction& transaction, uint32_t values_index) { + for (uint32_t j = 0; j < keys.size(); j++) { + std::vector data; + write(data, values[values_index][j]); + auto key = static_cast(keys[j]); + transaction.put_value_by_integer(key, data); + } + }; + + auto check_values = [&](LMDBReadTransaction& transaction, uint32_t values_index) { + for (uint32_t j = 0; j < keys.size(); j++) { + std::vector data; + auto key = static_cast(keys[j]); + transaction.get_value_by_integer(key, data); + auto retrieved_value = from_buffer(data); + ASSERT_EQ(retrieved_value, values[values_index][j]); + } + }; + + { + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + + write_values.template operator()(*transaction, 0); + write_values.template operator()(*transaction, 1); + write_values.template operator()(*transaction, 2); + write_values.template operator()(*transaction, 3); + } + + { + LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + + // we should be able to read values from different key spaces + check_values.template operator()(*transaction, 0); + check_values.template operator()(*transaction, 1); + check_values.template operator()(*transaction, 2); + check_values.template operator()(*transaction, 3); + } +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp new file mode 100644 index 00000000000..cc30f7a3e43 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp @@ -0,0 +1,94 @@ +#pragma once +#include "./tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/stdlib/primitives/field/field.hpp" +#include + +namespace bb::crypto::merkle_tree { + +class MockTransaction { + public: + using Ptr = std::unique_ptr; + bool get_node(uint32_t, index_t, std::vector&) const { return false; } + + template void get_value_by_integer(T&, std::vector&){}; + + void get_value(std::vector&, std::vector&){}; + + void put_node(uint32_t, index_t, const std::vector&) {} + + template void put_value_by_integer(T&, std::vector&){}; + + void put_value(std::vector&, std::vector&){}; +}; + +class MockPersistedStore { + public: + using ReadTransaction = MockTransaction; + using WriteTransaction = MockTransaction; + MockTransaction::Ptr createWriteTransaction() const { return std::make_unique(); } + MockTransaction::Ptr createReadTransaction() { return std::make_unique(); } +}; + +/** + * @brief A very basic 2-d array for use as a backing store for merkle trees. + * Can store up to 'indices' nodes per row and 'levels' rows. + */ +template class ArrayStore { + + public: + using ReadTransaction = typename PersistedStore::ReadTransaction; + using WriteTransaction = typename PersistedStore::WriteTransaction; + using ReadTransactionPtr = std::unique_ptr; + ArrayStore(const std::string& name, uint32_t depth, index_t indices = 1024) + : map(std::vector>>>( + depth + 1, + std::vector>>( + indices, std::pair>(false, std::vector())))) + { + meta.depth = depth; + meta.name = name; + meta.size = 0; + } + ~ArrayStore() = default; + + ArrayStore() = delete; + ArrayStore(ArrayStore const& other) = delete; + ArrayStore(ArrayStore const&& other) = delete; + ArrayStore& operator=(ArrayStore const& other) = delete; + ArrayStore& operator=(ArrayStore const&& other) = delete; + + void put_node(uint32_t level, index_t index, const std::vector& data) + { + map[level][index] = std::make_pair(true, data); + } + bool get_node(uint32_t level, index_t index, std::vector& data, ReadTransaction&, bool) const + { + const std::pair>& slot = map[level][index]; + if (slot.first) { + data = slot.second; + } + return slot.first; + } + void put_meta(const index_t& size, const bb::fr& root) + { + meta.root = root; + meta.size = size; + } + + void get_meta(index_t& size, bb::fr& root, ReadTransaction&, bool) const + { + size = meta.size; + root = meta.root; + } + + void commit(){}; + void rollback(){}; + + ReadTransactionPtr createReadTransaction() { return std::make_unique(); } + + private: + std::vector>>> map; + TreeMeta meta; +}; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_leaves_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_leaves_store.hpp new file mode 100644 index 00000000000..006f011aea4 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_leaves_store.hpp @@ -0,0 +1,126 @@ +#pragma once +#include "./tree_meta.hpp" +#include "barretenberg/common/serialize.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/numeric/uint256/uint256.hpp" +#include "barretenberg/stdlib/primitives/field/field.hpp" +#include "msgpack/assert.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace bb::crypto::merkle_tree { + +/** + * @brief Serves as a key-value node store for merkle trees, uses an unordered_map as a cache + */ +template class CachedLeavesStore { + + public: + using ReadTransaction = typename PersistedStore::ReadTransaction; + using WriteTransaction = typename PersistedStore::WriteTransaction; + using ReadTransactionPtr = std::unique_ptr; + using WriteTransactionPtr = std::unique_ptr; + + CachedLeavesStore(PersistedStore& leavesStore, index_t expected_size) + : leavesStore(leavesStore) + { + initialise(expected_size); + } + ~CachedLeavesStore() = default; + + CachedLeavesStore() = delete; + CachedLeavesStore(CachedLeavesStore const& other) = delete; + CachedLeavesStore(CachedLeavesStore const&& other) = delete; + CachedLeavesStore& operator=(CachedLeavesStore const& other) = delete; + CachedLeavesStore& operator=(CachedLeavesStore const&& other) = delete; + + index_t get_size(bool includeUncommitted) const { return meta.size + (includeUncommitted ? leaves_.size() : 0); } + + void commit() + { + { + if (leaves_.size() != indices_.size()) { + throw std::runtime_error("Inconsistent sizes in leaves data store"); + } + WriteTransactionPtr tx = createWriteTransaction(); + for (auto& idx : indices_) { + std::vector key; + std::vector value; + write(key, idx.first); + write(value, idx.second); + tx->put_value(key, value); + } + for (uint32_t i = 0; i < leaves_.size(); ++i) { + msgpack::sbuffer buffer; + msgpack::pack(buffer, leaves_[i]); + std::vector value(buffer.data(), buffer.data() + buffer.size()); + std::vector key; + write(key, index_t(meta.size + i)); + tx->put_value(key, value); + } + meta.size += leaves_.size(); + persistMeta(meta, *tx); + } + rollback(); + }; + void rollback() + { + indices_ = std::map(); + leaves_ = std::vector(); + }; + + ReadTransactionPtr createReadTransaction() const { return leavesStore.createReadNodeTransaction(); } + WriteTransactionPtr createWriteTransaction() const { return leavesStore.createWriteNodeTransaction(); } + + private: + std::map indices_; + std::vector leaves_; + PersistedStore& leavesStore; + LeavesMeta meta; + + bool readPersistedMeta(LeavesMeta& m, ReadTransaction& tx) const + { + std::vector data; + std::vector key{ 0 }; + bool success = tx.get_value(key, data); + if (success) { + msgpack::unpack((const char*)data.data(), data.size()).get().convert(m); + } + return success; + } + + void persistMeta(LeavesMeta& m, WriteTransaction& tx) + { + msgpack::sbuffer buffer; + msgpack::pack(buffer, m); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + std::vector key{ 0 }; + tx.put_value(key, encoded); + } + + void initialise(index_t expected_size) + { + std::vector data; + { + ReadTransactionPtr tx = createReadTransaction(); + bool success = readPersistedMeta(meta, *tx); + if (success) { + if (expected_size == meta.size) { + return; + } + throw std::runtime_error("Invalid tree meta data"); + } + } + + meta.size = 0; + WriteTransactionPtr tx = createWriteTransaction(); + persistMeta(meta, *tx); + } +}; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp new file mode 100644 index 00000000000..afce7ba957c --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp @@ -0,0 +1,319 @@ +#pragma once +#include "./tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/stdlib/primitives/field/field.hpp" +#include "msgpack/assert.hpp" +#include +#include +#include +#include +#include + +namespace bb::crypto::merkle_tree { + +/** + * @brief Serves as a key-value node store for merkle trees, uses an unordered_map as a cache + */ +template class CachedTreeStore { + + public: + using ReadTransaction = typename PersistedStore::ReadTransaction; + using WriteTransaction = typename PersistedStore::WriteTransaction; + using ReadTransactionPtr = std::unique_ptr; + using WriteTransactionPtr = std::unique_ptr; + + CachedTreeStore(std::string name, uint32_t levels, PersistedStore& dataStore) + : name(std::move(name)) + , depth(levels) + , nodes(std::vector>>( + depth + 1, std::unordered_map>())) + , dataStore(dataStore) + { + initialise(); + } + ~CachedTreeStore() = default; + + CachedTreeStore() = delete; + CachedTreeStore(CachedTreeStore const& other) = delete; + CachedTreeStore(CachedTreeStore const&& other) = delete; + CachedTreeStore& operator=(CachedTreeStore const& other) = delete; + CachedTreeStore& operator=(CachedTreeStore const&& other) = delete; + + std::pair find_low_value(const bb::fr& new_value, + bool includeUncommitted, + ReadTransaction& tx) const; + + LeafType get_leaf(const index_t& index, ReadTransaction& tx, bool includeUncommitted) const; + + void set_at_index(const index_t& index, const LeafType& leaf, bool add_to_index); + + void put_node(uint32_t level, index_t index, const std::vector& data); + + bool get_node(uint32_t level, + index_t index, + std::vector& data, + ReadTransaction& transaction, + bool includeUncommitted) const; + + void put_meta(const index_t& size, const bb::fr& root); + + void get_meta(index_t& size, bb::fr& root, ReadTransaction& tx, bool includeUncommitted) const; + + void get_full_meta(index_t& size, + bb::fr& root, + std::string& name, + uint32_t& depth, + ReadTransaction& tx, + bool includeUncommitted) const; + + void commit(); + + void rollback(); + + ReadTransactionPtr createReadTransaction() const { return dataStore.createReadTransaction(); } + WriteTransactionPtr createWriteTransaction() const { return dataStore.createWriteTransaction(); } + + private: + std::string name; + uint32_t depth; + std::vector>> nodes; + std::map indices_; + std::unordered_map leaves_; + PersistedStore& dataStore; + TreeMeta meta; + + void initialise(); + + bool readPersistedMeta(TreeMeta& m, ReadTransaction& tx) const; + + void persistMeta(TreeMeta& m, WriteTransaction& tx); +}; + +template +std::pair CachedTreeStore::find_low_value(const bb::fr& new_value, + bool includeUncommitted, + ReadTransaction& tx) const +{ + uint256_t new_value_as_number = uint256_t(new_value); + std::vector data; + FrKeyType key(new_value); + tx.get_value_or_previous(key, data); + index_t db_index = from_buffer(data, 0); + uint256_t retrieved_value = key; + if (!includeUncommitted || retrieved_value == new_value_as_number || indices_.empty()) { + return std::make_pair(new_value_as_number == retrieved_value, db_index); + } + + // At this stage, we have been asked to include uncommitted and the value was not exactly found in the db + std::map::const_iterator it = indices_.lower_bound(new_value); + if (it == indices_.end()) { + // there is no element >= the requested value. + // decrement the iterator to get the value preceeding the requested value + --it; + // we need to return the larger of the db value or the cached value + + return std::make_pair(false, it->first > retrieved_value ? it->second : db_index); + } + + if (it->first == uint256_t(new_value)) { + // the value is already present and the iterator points to it + return std::make_pair(true, it->second); + } + // the iterator points to the element immediately larger than the requested value + // We need to return the highest value from + // 1. The next lowest cached value, if there is one + // 2. The value retrieved from the db + if (it == indices_.begin()) { + // No cached lower value, return the db index + return std::make_pair(false, db_index); + } + --it; + // it now points to the value less than that requested + return std::make_pair(false, it->first > retrieved_value ? it->second : db_index); +} + +template +LeafType CachedTreeStore::get_leaf(const index_t& index, + ReadTransaction& tx, + bool includeUncommitted) const +{ + LeafType return_value; + if (includeUncommitted) { + typename std::unordered_map::const_iterator it = leaves_.find(index); + if (it != leaves_.end()) { + return it->second; + } + } + LeafIndexKeyType key = index; + std::vector data; + bool success = tx.get_value_by_integer(key, data); + if (success) { + msgpack::unpack((const char*)data.data(), data.size()).get().convert(return_value); + } + return return_value; +} + +template +void CachedTreeStore::set_at_index(const index_t& index, + const LeafType& leaf, + bool add_to_index) +{ + leaves_[index] = leaf; + if (add_to_index) { + indices_[uint256_t(leaf.value)] = index; + } +} + +template +void CachedTreeStore::put_node(uint32_t level, + index_t index, + const std::vector& data) +{ + nodes[level][index] = data; +} + +template +bool CachedTreeStore::get_node(uint32_t level, + index_t index, + std::vector& data, + ReadTransaction& transaction, + bool includeUncommitted) const +{ + if (includeUncommitted) { + const auto& level_map = nodes[level]; + auto it = level_map.find(index); + if (it != level_map.end()) { + data = it->second; + return true; + } + } + return transaction.get_node(level, index, data); +} + +template +void CachedTreeStore::put_meta(const index_t& size, const bb::fr& root) +{ + meta.root = root; + meta.size = size; +} + +template +void CachedTreeStore::get_meta(index_t& size, + bb::fr& root, + ReadTransaction& tx, + bool includeUncommitted) const +{ + if (includeUncommitted) { + size = meta.size; + root = meta.root; + return; + } + TreeMeta m; + readPersistedMeta(m, tx); + size = m.size; + root = m.root; +} + +template +void CachedTreeStore::get_full_meta( + index_t& size, bb::fr& root, std::string& name, uint32_t& depth, ReadTransaction& tx, bool includeUncommitted) const +{ + if (includeUncommitted) { + size = meta.size; + root = meta.root; + name = meta.name; + depth = meta.depth; + return; + } + TreeMeta m; + readPersistedMeta(m, tx); + size = m.size; + root = m.root; + depth = m.depth; + name = m.name; +} + +template void CachedTreeStore::commit() +{ + { + WriteTransactionPtr tx = createWriteTransaction(); + for (uint32_t i = 1; i < nodes.size(); i++) { + auto& level = nodes[i]; + for (auto& item : level) { + index_t index = item.first; + std::vector& data = item.second; + tx->put_node(i, index, data); + } + } + for (auto& idx : indices_) { + std::vector value; + write(value, idx.second); + FrKeyType key = idx.first; + tx->put_value_by_integer(key, value); + } + for (const auto& leaf : leaves_) { + msgpack::sbuffer buffer; + msgpack::pack(buffer, leaf.second); + std::vector value(buffer.data(), buffer.data() + buffer.size()); + LeafIndexKeyType key = leaf.first; + tx->put_value_by_integer(key, value); + } + persistMeta(meta, *tx); + } + rollback(); +} + +template void CachedTreeStore::rollback() +{ + nodes = std::vector>>( + depth + 1, std::unordered_map>()); + indices_ = std::map(); + leaves_ = std::unordered_map(); + ReadTransactionPtr tx = createReadTransaction(); + readPersistedMeta(meta, *tx); +} + +template +bool CachedTreeStore::readPersistedMeta(TreeMeta& m, ReadTransaction& tx) const +{ + std::vector data; + bool success = tx.get_node(0, 0, data); + if (success) { + msgpack::unpack((const char*)data.data(), data.size()).get().convert(m); + } + return success; +} + +template +void CachedTreeStore::persistMeta(TreeMeta& m, WriteTransaction& tx) +{ + msgpack::sbuffer buffer; + msgpack::pack(buffer, m); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + tx.put_node(0, 0, encoded); +} + +template void CachedTreeStore::initialise() +{ + std::vector data; + { + ReadTransactionPtr tx = createReadTransaction(); + bool success = readPersistedMeta(meta, *tx); + if (success) { + if (name == meta.name && depth == meta.depth) { + return; + } + throw std::runtime_error("Invalid tree meta data"); + } + } + + meta.name = name; + meta.size = 0; + meta.depth = depth; + WriteTransactionPtr tx = createWriteTransaction(); + persistMeta(meta, *tx); +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp new file mode 100644 index 00000000000..69aa147073a --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp @@ -0,0 +1,24 @@ +#pragma once +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/serialize/msgpack.hpp" +#include +#include + +namespace bb::crypto::merkle_tree { + +struct TreeMeta { + std::string name; + uint32_t depth; + index_t size; + bb::fr root; + + MSGPACK_FIELDS(name, depth, size, root) +}; + +struct LeavesMeta { + index_t size; + + MSGPACK_FIELDS(size) +}; + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt deleted file mode 100644 index 9050a6601f6..00000000000 --- a/barretenberg/cpp/src/barretenberg/lmdb_store/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -barretenberg_module(lmdb_store lmdb stdlib_primitives) - diff --git a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp deleted file mode 100644 index 72e34ec22ab..00000000000 --- a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.cpp +++ /dev/null @@ -1,85 +0,0 @@ -#include "lmdb_store.hpp" - -namespace bb::lmdb_store { - -LMDBEnvironment::LMDBEnvironment(const std::string& directory, - unsigned long mapSizeMB, - unsigned long maxNumDBs, - unsigned int maxNumReaders) - : _maxReaders(maxNumReaders) - , _numReaders(0) -{ - if (!call_lmdb_func(mdb_env_create, &_mdbEnv)) { - // throw here - } - size_t totalMapSize = 1024 * 1024 * mapSizeMB; - call_lmdb_func(mdb_env_set_mapsize, _mdbEnv, totalMapSize); - call_lmdb_func(mdb_env_set_maxdbs, _mdbEnv, static_cast(maxNumDBs)); - call_lmdb_func(mdb_env_set_maxreaders, _mdbEnv, maxNumReaders); - unsigned int flags = MDB_NOTLS; - if (!call_lmdb_func( - mdb_env_open, _mdbEnv, directory.c_str(), flags, static_cast(S_IRWXU | S_IRWXG | S_IRWXO))) { - call_lmdb_func(mdb_env_close, _mdbEnv); - // throw here - } -} - -void LMDBEnvironment::waitForReader() -{ - std::unique_lock lock(_readersLock); - if (_numReaders >= _maxReaders) { - _readersCondition.wait(lock, [&] { return _numReaders < _maxReaders; }); - } - ++_numReaders; -} - -void LMDBEnvironment::releaseReader() -{ - std::unique_lock lock(_readersLock); - --_numReaders; - _readersCondition.notify_one(); -} - -void LMDBWriteTransaction::put(size_t level, size_t start_index, std::vector& data, size_t element_size) -{ - MDB_cursor* cursor; - call_lmdb_func(mdb_cursor_open, underlying(), _database.underlying(), &cursor); - size_t key = (1 << level) + start_index - 1; - size_t num_elements = data.size() / element_size; - - for (size_t i = 0; i < num_elements; ++i, ++key) { - MDB_val dbKey; - dbKey.mv_size = sizeof(key); - dbKey.mv_data = &key; - - MDB_val dbVal; - dbVal.mv_size = element_size; - dbVal.mv_data = static_cast(&data[i * element_size]); - call_lmdb_func(mdb_cursor_put, cursor, &dbKey, &dbVal, 0U); - } -} - -bool LMDBReadTransaction::get(size_t level, size_t index, std::vector& data) const -{ - size_t key = (1 << level) + index - 1; - - MDB_val dbKey; - dbKey.mv_size = sizeof(key); - dbKey.mv_data = &key; - - MDB_val dbVal; - if (!call_lmdb_func(mdb_get, underlying(), _database.underlying(), &dbKey, &dbVal)) { - return false; - } - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); - return true; -} - -LMDBReadTransaction::Ptr LMDBStore::createReadTransaction() -{ - _environment.waitForReader(); - return std::make_unique(_environment, _database); -} - -} // namespace bb::lmdb_store \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp deleted file mode 100644 index 856f3d303f9..00000000000 --- a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.hpp +++ /dev/null @@ -1,182 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace bb::lmdb_store { - -template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) -{ - int error = f(args...); - if (error != 0 && error != MDB_NOTFOUND) { - std::stringstream ss; - ss << "ERROR: " << mdb_strerror(error) << std::endl; - std::cout << ss.str(); - } - return error == 0; -} - -template void call_lmdb_func(void (*f)(TArgs...), TArgs... args) -{ - f(args...); -} - -class LMDBEnvironment { - public: - LMDBEnvironment(const std::string& directory, - unsigned long mapSizeMB, - unsigned long maxNumDBs, - unsigned int maxNumReaders); - - ~LMDBEnvironment() { call_lmdb_func(mdb_env_close, _mdbEnv); } - - MDB_env* underlying() const { return _mdbEnv; } - - void waitForReader(); - - void releaseReader(); - - private: - MDB_env* _mdbEnv; - unsigned int _maxReaders; - unsigned int _numReaders; - std::mutex _readersLock; - std::condition_variable _readersCondition; -}; - -class LMDBDatabase; - -class LMDBTransaction { - public: - LMDBTransaction(LMDBEnvironment& env, bool readOnly = false) - : _environment(env) - { - MDB_txn* p = nullptr; - if (!call_lmdb_func(mdb_txn_begin, _environment.underlying(), p, readOnly ? MDB_RDONLY : 0U, &_transaction)) { - // throw here - } - } - - virtual ~LMDBTransaction() = default; - - MDB_txn* underlying() const { return _transaction; } - - protected: - LMDBEnvironment& _environment; - MDB_txn* _transaction; -}; - -class LMDBReadTransaction : public LMDBTransaction { - public: - using Ptr = std::unique_ptr; - - LMDBReadTransaction(LMDBEnvironment& env, const LMDBDatabase& database) - : LMDBTransaction(env, true) - , _database(database) - {} - - ~LMDBReadTransaction() override { abort(); } - - bool get(size_t level, size_t index, std::vector& data) const; - - private: - const LMDBDatabase& _database; - - void abort() - { - call_lmdb_func(mdb_txn_abort, _transaction); - _environment.releaseReader(); - } -}; - -class LMDBWriteTransaction : public LMDBTransaction { - public: - using Ptr = std::unique_ptr; - - LMDBWriteTransaction(LMDBEnvironment& env, const LMDBDatabase& database) - : LMDBTransaction(env) - , _database(database) - {} - - virtual ~LMDBWriteTransaction() { commit(); } - - void put(size_t level, size_t index, std::vector& data) { put(level, index, data, data.size()); } - void put(size_t level, size_t start_index, std::vector& data, size_t element_size); - - bool commit() - { - if (_committed) { - return true; - } - _committed = call_lmdb_func(mdb_txn_commit, _transaction); - return _committed; - } - - private: - bool _committed{}; - const LMDBDatabase& _database; -}; - -class LMDBDatabaseCreationTransaction : public LMDBTransaction { - public: - using Ptr = std::unique_ptr; - - LMDBDatabaseCreationTransaction(LMDBEnvironment& env) - : LMDBTransaction(env) - {} - - virtual ~LMDBDatabaseCreationTransaction() { commit(); } - - private: - bool commit() { return call_lmdb_func(mdb_txn_commit, _transaction); } -}; - -class LMDBDatabase { - public: - LMDBDatabase(const LMDBEnvironment& env, const LMDBTransaction& transaction, const std::string& name) - : _environment(env) - { - unsigned int flags = MDB_CREATE | MDB_INTEGERKEY; - if (!call_lmdb_func(mdb_dbi_open, transaction.underlying(), name.c_str(), flags, &_dbi)) { - // throw here - } - } - - ~LMDBDatabase() { call_lmdb_func(mdb_dbi_close, _environment.underlying(), _dbi); } - - const MDB_dbi& underlying() const { return _dbi; } - - private: - MDB_dbi _dbi; - const LMDBEnvironment& _environment; -}; - -class LMDBStore { - - public: - LMDBStore(LMDBEnvironment& environment, std::string name) - : _environment(environment) - , _name(std::move(name)) - , _database(_environment, LMDBDatabaseCreationTransaction(_environment), _name) - {} - ~LMDBStore() {} - - LMDBWriteTransaction::Ptr createWriteTransaction() const - { - return std::make_unique(_environment, _database); - } - LMDBReadTransaction::Ptr createReadTransaction(); - - private: - LMDBEnvironment& _environment; - const std::string _name; - LMDBDatabase _database; -}; -} // namespace bb::lmdb_store \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp deleted file mode 100644 index eca52cc7e81..00000000000 --- a/barretenberg/cpp/src/barretenberg/lmdb_store/lmdb_store.test.cpp +++ /dev/null @@ -1,273 +0,0 @@ -#include - -#include -#include -#include - -#include "barretenberg/common/streams.hpp" -#include "barretenberg/common/test.hpp" -#include "barretenberg/numeric/random/engine.hpp" -#include "barretenberg/stdlib/primitives/field/field.hpp" -#include "lmdb_store.hpp" - -using namespace bb::stdlib; -using namespace bb::lmdb_store; - -using Builder = bb::UltraCircuitBuilder; - -using field_ct = field_t; -using witness_ct = witness_t; -namespace { -auto& engine = bb::numeric::get_debug_randomness(); -auto& random_engine = bb::numeric::get_randomness(); -} // namespace - -const int SAMPLE_DATA_SIZE = 1024; - -static std::vector VALUES = []() { - std::vector values(SAMPLE_DATA_SIZE); - for (size_t i = 0; i < SAMPLE_DATA_SIZE; ++i) { - values[i] = bb::fr::random_element(&engine); - } - return values; -}(); - -class LMDBStoreTest : public testing::Test { - protected: - static void SetUpTestSuite() - { - const int random_directory_value = rand(); - std::stringstream ss; - ss << "/tmp/lmdb" << random_directory_value; - _directory = ss.str(); - std::filesystem::create_directories(_directory); - } - - static void TearDownTestSuite() { std::filesystem::remove_all(_directory); } - - void SetUp() - { - // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers - _environment.reset(new LMDBEnvironment(_directory, 1, 1, 2)); - } - - static std::string _directory; - - std::unique_ptr _environment; -}; - -std::string LMDBStoreTest::_directory; - -TEST_F(LMDBStoreTest, can_write_to_and_read_from_store) -{ - LMDBStore store(*_environment, "note hash tree"); - - { - std::vector buf; - write(buf, VALUES[0]); - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); - transaction->put(0, 0, buf); - } - - { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); - std::vector buf2; - bool success = transaction->get(0, 0, buf2); - EXPECT_EQ(success, true); - bb::fr value = from_buffer(buf2, 0); - EXPECT_EQ(value, VALUES[0]); - } -} - -TEST_F(LMDBStoreTest, reading_an_empty_key_reports_correctly) -{ - LMDBStore store(*_environment, "note hash tree"); - - { - std::vector buf; - write(buf, VALUES[0]); - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); - transaction->put(0, 0, buf); - } - - { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); - std::vector buf2; - bool success = transaction->get(0, 1, buf2); - EXPECT_EQ(success, false); - } -} - -TEST_F(LMDBStoreTest, can_write_and_read_multiple) -{ - LMDBStore store(*_environment, "note hash tree"); - - { - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { - std::vector buf; - write(buf, VALUES[i]); - transaction->put(10, i, buf); - } - } - - { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { - std::vector buf2; - bool success = transaction->get(10, i, buf2); - EXPECT_EQ(success, true); - bb::fr value = from_buffer(buf2, 0); - EXPECT_EQ(value, VALUES[i]); - } - } -} - -TEST_F(LMDBStoreTest, can_write_batch_and_read_back) -{ - LMDBStore store(*_environment, "note hash tree"); - - std::vector buf; - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { - std::vector temp; - write(temp, VALUES[i]); - int old_size = static_cast(buf.size()); - buf.resize(buf.size() + temp.size()); - copy(temp.begin(), temp.end(), buf.begin() + old_size); - } - - { - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); - transaction->put(10, 0, buf, 32); - } - - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { - std::vector buf2; - bool success = store.createReadTransaction()->get(10, i, buf2); - EXPECT_EQ(success, true); - bb::fr value = from_buffer(buf2, 0); - EXPECT_EQ(value, VALUES[i]); - } -} - -TEST_F(LMDBStoreTest, can_write_and_read_at_random_keys) -{ - LMDBStore store(*_environment, "note hash tree"); - - std::vector keys; - - { - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); - - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { - std::vector buf; - write(buf, VALUES[i]); - size_t key = static_cast(rand() % 10000000); - keys.push_back(key); - transaction->put(0, key, buf); - } - } - - { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { - std::vector buf2; - bool success = transaction->get(0, keys[i], buf2); - EXPECT_EQ(success, true); - bb::fr value = from_buffer(buf2, 0); - EXPECT_EQ(value, VALUES[i]); - } - } -} - -TEST_F(LMDBStoreTest, can_recreate_the_store_and_use_again) -{ - std::vector keys; - { - LMDBStore store(*_environment, "note hash tree"); - - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); - - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { - std::vector buf; - write(buf, VALUES[i]); - size_t key = static_cast(rand() % 10000000); - keys.push_back(key); - transaction->put(0, key, buf); - } - } - - { - LMDBStore store(*_environment, "note hash tree"); - - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); - for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { - std::vector buf2; - bool success = transaction->get(0, keys[i], buf2); - EXPECT_EQ(success, true); - bb::fr value = from_buffer(buf2, 0); - EXPECT_EQ(value, VALUES[i]); - } - } -} - -void read_loop(LMDBStore& store, size_t key, std::atomic& flag, bb::fr starting_value) -{ - bool seen = false; - while (true) { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); - std::vector buf; - bool success = transaction->get(0, key, buf); - EXPECT_EQ(success, true); - bb::fr value = from_buffer(buf, 0); - if (value == starting_value && !seen) { - // acknowledge that we have seen the old value - flag--; - seen = true; - } - if (value == starting_value + bb::fr(1)) { - // exit now that we have seen the new value - break; - } - } -} - -TEST_F(LMDBStoreTest, can_read_from_multiple_threads) -{ - LMDBStore store(*_environment, "note hash tree"); - const int num_threads = 5; - - size_t key = static_cast(rand() % 1000000); - - { - // we write VALUES[0] to a slot - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); - std::vector buf; - write(buf, VALUES[0]); - transaction->put(0, key, buf); - } - - { - // we setup multiple threads to read the slot and check they shutdown when the value changes - std::vector threads; - std::atomic flag = num_threads; - for (size_t i = 0; i < num_threads; i++) { - threads.emplace_back(read_loop, std::ref(store), key, std::ref(flag), VALUES[0]); - } - // wait until all threads have seen the old value - while (flag != 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - { - // we write VALUES[0] + 1 to the slot - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); - std::vector buf; - write(buf, VALUES[0] + 1); - transaction->put(0, key, buf); - } - // now wait for all threads to exit having seen the new value - for (size_t i = 0; i < 5; i++) { - threads[i].join(); - } - } -} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp b/barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp deleted file mode 100644 index 2c171ab699e..00000000000 --- a/barretenberg/cpp/src/barretenberg/messaging/stream_parser.test.cpp +++ /dev/null @@ -1,184 +0,0 @@ -#include "stream_parser.hpp" -#include "barretenberg/common/test.hpp" -#include "barretenberg/messaging/header.hpp" -#include "gtest/gtest.h" -#include -#include -#include -#include -#include - -using namespace bb::messaging; - -std::vector createMessage(uint16_t type, uint32_t messageId, uint32_t requestId, uint32_t dataSize) -{ - MsgHeader header(type, requestId, messageId, HEADER_SIZE + dataSize); - std::vector message(header.msgLength); - std::memcpy(&message[0], &header, HEADER_SIZE); - return message; -} - -void assertHandledMessage(std::vector& inputMessage, const MsgHeader* parsedHeader, const char* parsedData) -{ - EXPECT_EQ(0, std::memcmp(&inputMessage[0], parsedHeader, HEADER_SIZE)); - EXPECT_EQ(0, std::memcmp(&inputMessage[HEADER_SIZE], parsedData, inputMessage.size() - HEADER_SIZE)); -} - -TEST(messaging_stream_parser, correctly_parses_basic_message) -{ - const MsgHeader* parsedHeader = nullptr; - const char* receivedData = nullptr; - int numCalls = 0; - - std::function handler = [&](const MsgHeader* header, const char* data) { - parsedHeader = header; - receivedData = data; - numCalls++; - return true; - }; - StreamParser parser(handler); - std::vector message = createMessage(5, 57, 56, 14); - parser.onNewData(&message[0], static_cast(message.size())); - EXPECT_EQ(numCalls, 1); - assertHandledMessage(message, parsedHeader, receivedData); -} - -TEST(messaging_stream_parser, correctly_parses_split_message) -{ - const uint32_t dataSize = 14; - const MsgHeader* parsedHeader = nullptr; - const char* receivedData = nullptr; - int numCalls = 0; - - std::function handler = [&](const MsgHeader* header, const char* data) { - parsedHeader = header; - receivedData = data; - numCalls++; - return true; - }; - StreamParser parser(handler); - std::vector message = createMessage(5, 57, 56, dataSize); - - size_t readPointer = 0; - // add less than the magic string - parser.onNewData(&message[readPointer], static_cast(MAGIC_STRING_LENGTH - 1)); - readPointer += MAGIC_STRING_LENGTH - 1; - - // handler not yet called - EXPECT_EQ(numCalls, 0); - - // now add more than magic string, less than header - parser.onNewData(&message[readPointer], 2); - readPointer += 2; - - // should still not have been called - EXPECT_EQ(numCalls, 0); - - // now add up to the header - parser.onNewData(&message[readPointer], static_cast(HEADER_SIZE - readPointer)); - readPointer = HEADER_SIZE; - - // should still not have been called - EXPECT_EQ(numCalls, 0); - - // now add part of the data - parser.onNewData(&message[readPointer], static_cast(dataSize / 2)); - readPointer += dataSize / 2; - - // should still not have been called - EXPECT_EQ(numCalls, 0); - - // now add the rest of the data - parser.onNewData(&message[readPointer], static_cast(dataSize / 2)); - readPointer += dataSize / 2; - - // should still not have been called - EXPECT_EQ(numCalls, 1); - - assertHandledMessage(message, parsedHeader, receivedData); -} - -TEST(messaging_stream_parser, correctly_parses_multiple_messages) -{ - std::array, 3> messages = { createMessage(5, 57, 56, 14), - createMessage(6, 58, 57, 18), - createMessage(7, 59, 58, 20) }; - std::vector fullStream; - for (const auto& message : messages) { - fullStream.insert(fullStream.end(), message.begin(), message.end()); - } - uint32_t numCalls = 0; - std::function handler = [&](const MsgHeader* header, const char* data) { - assertHandledMessage(messages[numCalls], header, data); - numCalls++; - return true; - }; - StreamParser parser(handler); - parser.onNewData(&fullStream[0], static_cast(fullStream.size())); - EXPECT_EQ(numCalls, 3); -} - -TEST(messaging_stream_parser, correctly_parses_split_multiple_messages) -{ - std::array, 3> messages = { createMessage(5, 57, 56, 14), - createMessage(6, 58, 57, 14), - createMessage(7, 59, 58, 14) }; - std::vector fullStream; - for (const auto& message : messages) { - fullStream.insert(fullStream.end(), message.begin(), message.end()); - } - uint32_t numCalls = 0; - std::function handler = [&](const MsgHeader* header, const char* data) { - assertHandledMessage(messages[numCalls], header, data); - numCalls++; - return true; - }; - StreamParser parser(handler); - - // add the data in chunks - std::array chunks = { static_cast(fullStream.size() / 6), - static_cast((fullStream.size() / 6) * 2), - static_cast((fullStream.size() / 6) * 2) }; - uint32_t readPointer = 0; - for (size_t i = 0; i < chunks.size(); i++) { - parser.onNewData(&fullStream[readPointer], chunks[i]); - readPointer += chunks[i]; - } - uint32_t remaining = static_cast(fullStream.size()) - readPointer; - parser.onNewData(&fullStream[readPointer], remaining); - EXPECT_EQ(numCalls, 3); -} - -TEST(messaging_stream_parser, correctly_parses_split_multiple_messages_with_garbage) -{ - std::array, 3> messages = { createMessage(5, 57, 56, 14), - createMessage(6, 58, 57, 14), - createMessage(7, 59, 58, 14) }; - std::vector fullStream; - // build the full stream with garbage bytes inserted before each real message - for (const auto& message : messages) { - std::string garbage{ "epfnwpcndcqwedoceodinc" }; - fullStream.insert(fullStream.end(), garbage.begin(), garbage.end()); - fullStream.insert(fullStream.end(), message.begin(), message.end()); - } - uint32_t numCalls = 0; - std::function handler = [&](const MsgHeader* header, const char* data) { - assertHandledMessage(messages[numCalls], header, data); - numCalls++; - return true; - }; - StreamParser parser(handler); - - // add the data in chunks - std::array chunks = { static_cast(fullStream.size() / 6), - static_cast((fullStream.size() / 6) * 2), - static_cast((fullStream.size() / 6) * 2) }; - uint32_t readPointer = 0; - for (size_t i = 0; i < chunks.size(); i++) { - parser.onNewData(&fullStream[readPointer], chunks[i]); - readPointer += chunks[i]; - } - uint32_t remaining = static_cast(fullStream.size()) - readPointer; - parser.onNewData(&fullStream[readPointer], remaining); - EXPECT_EQ(numCalls, 3); -} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp index 3a5e3901a69..92f3325e9c1 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp @@ -1,11 +1,12 @@ #pragma once #include "barretenberg/common/thread_pool.hpp" +#include "barretenberg/crypto/merkle_tree//lmdb_store/lmdb_store.hpp" #include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/array_store.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/cached_store.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/cbind.hpp" #include "barretenberg/world_state/service/message.hpp" @@ -13,6 +14,7 @@ #include #include #include +#include #include #include @@ -21,11 +23,31 @@ using namespace bb::messaging; using namespace bb::crypto::merkle_tree; using HashPolicy = PedersenHashPolicy; +struct TreeWithStore { + typedef CachedNodeStore Store; + std::unique_ptr> tree; + std::unique_ptr store; + + TreeWithStore(std::unique_ptr> t, std::unique_ptr s) + : tree(std::move(t)) + , store(std::move(s)) + {} + + TreeWithStore(TreeWithStore&& other) noexcept + : tree(std::move(other.tree)) + , store(std::move(other.store)) + {} + + TreeWithStore& operator=(TreeWithStore&& other) = delete; + ~TreeWithStore() = default; + TreeWithStore(const TreeWithStore& other) = delete; + TreeWithStore& operator=(const TreeWithStore& other) = delete; +}; + template class WorldStateService { private: OutputStream& outputStream; - std::unordered_map>> trees; - std::vector> stores; + std::unordered_map> trees; ThreadPool workers; public: @@ -47,19 +69,20 @@ template bool WorldStateService::startTree auto it = trees.find(startTreeRequest.value.name); if (it == trees.end()) { - auto store = std::make_shared(startTreeRequest.value.depth, 1024 * 16); - stores.push_back(store); + auto store = std::make_unique(startTreeRequest.value.depth); + auto tree = - std::make_shared>(*store, - startTreeRequest.value.depth, - workers, - startTreeRequest.value.preFilledSize, - startTreeRequest.value.name); - std::unordered_map>>::value_type - v = { startTreeRequest.value.name, tree }; - trees.insert(v); + std::make_unique>(*store, + startTreeRequest.value.depth, + workers, + startTreeRequest.value.preFilledSize, + startTreeRequest.value.name); + + auto treeWithStore = std::make_unique(std::move(tree), std::move(store)); + trees[startTreeRequest.value.name] = std::move(treeWithStore); + } else { - actualDepth = static_cast(it->second->depth()); + actualDepth = static_cast(it->second->tree->depth()); } StartTreeResponse response; @@ -89,9 +112,9 @@ template bool WorldStateService::getTreeIn } else { treeInfoResponse.message = ""; treeInfoResponse.success = true; - treeInfoResponse.depth = static_cast(it->second->depth()); - treeInfoResponse.root = it->second->root(); - treeInfoResponse.size = static_cast(it->second->size()); + treeInfoResponse.depth = static_cast(it->second->tree->depth()); + treeInfoResponse.root = it->second->tree->root(); + treeInfoResponse.size = static_cast(it->second->tree->size()); } MsgHeader header(treeInfoRequest.header.messageId); @@ -134,7 +157,7 @@ template bool WorldStateService::insertLea WorldStateMsgTypes::INSERT_LEAVES_RESPONSE, header, response); outputStream.sendPackedObject(insertLeavesResponse); }; - it->second->add_or_update_values(insertLeavesRequest.value.leaves, completion); + it->second->tree->add_or_update_values(insertLeavesRequest.value.leaves, completion); } return true; From 75b077ff11300fc7c76fa9fee215c35b6449eb21 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 25 Jun 2024 17:55:20 +0100 Subject: [PATCH 22/63] WIP --- barretenberg/cpp/build_and_test.sh | 5 +++++ .../indexed_tree/indexed_tree.test.cpp | 22 ------------------- 2 files changed, 5 insertions(+), 22 deletions(-) create mode 100755 barretenberg/cpp/build_and_test.sh diff --git a/barretenberg/cpp/build_and_test.sh b/barretenberg/cpp/build_and_test.sh new file mode 100755 index 00000000000..013bfe4778b --- /dev/null +++ b/barretenberg/cpp/build_and_test.sh @@ -0,0 +1,5 @@ +cmake --build --preset default --target crypto_merkle_tree_tests +#cmake --build --preset default --target world_state +./build/bin/crypto_merkle_tree_tests --gtest_filter=PersistedIndexedTreeTest.* +./build/bin/crypto_merkle_tree_tests --gtest_filter=PersistedAppendOnlyTreeTest* +./build/bin/crypto_merkle_tree_tests --gtest_filter=LMDBStoreTest* diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index f1557e7b67f..2bd6c422c21 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -204,12 +204,9 @@ TEST_F(PersistedIndexedTreeTest, test_get_hash_path) uint32_t num_to_append = 512; for (uint32_t i = 0; i < num_to_append; ++i) { - std::cout << i + 1 << std::endl; memdb.update_element(VALUES[i]); add_value(tree, VALUES[i]); } - - std::cout << "Append " << num_to_append + 2 << std::endl; check_size(tree, num_to_append + 2); check_hash_path(tree, 0, memdb.get_hash_path(0)); check_hash_path(tree, 512, memdb.get_hash_path(512)); @@ -392,8 +389,6 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) check_size(tree, 1); EXPECT_EQ(get_leaf(tree, 0), zero_leaf); - std::cout << "1" << std::endl; - /** * Add new value 30: * @@ -404,17 +399,10 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) * nextVal 30 0 0 0 0 0 0 0 */ add_value(tree, 30); - std::cout << "After add" << std::endl; check_size(tree, 2); - indexed_leaf test = { 0, 1, 30 }; - EXPECT_EQ(get_leaf(tree, 0), test); - test = { 30, 0, 0 }; - EXPECT_EQ(get_leaf(tree, 1), test); EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 1, 30 })); EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 0, 0 })); - std::cout << "2" << std::endl; - /** * Add new value 10: * @@ -426,14 +414,10 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) */ add_value(tree, 10); check_size(tree, 3); - test = { 0, 2, 10 }; - EXPECT_EQ(get_leaf(tree, 0), test); EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 2, 10 })); EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 0, 0 })); EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf({ 10, 1, 30 })); - std::cout << "3" << std::endl; - /** * Add new value 20: * @@ -445,15 +429,11 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) */ add_value(tree, 20); check_size(tree, 4); - test = { 0, 2, 10 }; - EXPECT_EQ(get_leaf(tree, 0), test); EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 2, 10 })); EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 0, 0 })); EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf({ 10, 3, 20 })); EXPECT_EQ(hash_leaf(get_leaf(tree, 3)), hash_leaf({ 20, 1, 30 })); - std::cout << "4" << std::endl; - // Adding the same value must not affect anything // tree.update_element(20); // EXPECT_EQ(tree.get_leaves().size(), 4); @@ -479,8 +459,6 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) EXPECT_EQ(hash_leaf(get_leaf(tree, 3)), hash_leaf({ 20, 1, 30 })); EXPECT_EQ(hash_leaf(get_leaf(tree, 4)), hash_leaf({ 50, 0, 0 })); - std::cout << "5" << std::endl; - // Manually compute the node values auto e000 = hash_leaf(get_leaf(tree, 0)); auto e001 = hash_leaf(get_leaf(tree, 1)); From 0f6a769daf45c25e7cb21f772f964bb70f816a68 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 26 Jun 2024 16:01:58 +0100 Subject: [PATCH 23/63] WIP --- .../indexed_tree_bench/indexed_tree.bench.cpp | 2 +- .../append_only_tree/append_only_tree.hpp | 1 + .../merkle_tree/indexed_tree/indexed_leaf.hpp | 88 ++++++++++-- .../merkle_tree/indexed_tree/indexed_tree.hpp | 90 +++++++------ .../indexed_tree/indexed_tree.test.cpp | 62 +++++---- .../merkle_tree/indexed_tree/leaves_cache.cpp | 49 ------- .../merkle_tree/indexed_tree/leaves_cache.hpp | 25 ---- .../merkle_tree/lmdb_store/lmdb_store.cpp | 1 - .../node_store/cached_leaves_store.hpp | 126 ------------------ .../node_store/cached_tree_store.hpp | 107 +++++++-------- .../nullifier_tree/nullifier_leaf.hpp | 17 ++- .../nullifier_tree/nullifier_memory_tree.hpp | 17 +-- .../nullifier_memory_tree.test.cpp | 9 +- .../nullifier_tree/nullifier_tree.cpp | 11 +- .../service/world_state_service.hpp | 2 - 15 files changed, 247 insertions(+), 360 deletions(-) delete mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp delete mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_leaves_store.hpp diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp index 33afba6bd1f..ff7eff0a1ba 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp @@ -11,7 +11,7 @@ using namespace benchmark; using namespace bb::crypto::merkle_tree; -using StoreType = CachedTreeStore; +using StoreType = CachedTreeStore; using Poseidon2 = IndexedTree; using Pedersen = IndexedTree; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 773859f6661..a2d4fef481f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -3,6 +3,7 @@ #include "../node_store//tree_meta.hpp" #include "../types.hpp" #include "barretenberg/common/thread_pool.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include #include #include diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp index cb67ddd4592..5f492dfcbe8 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp @@ -6,30 +6,100 @@ namespace bb::crypto::merkle_tree { -struct indexed_leaf { +struct nullifier_leaf_value { fr value; + + MSGPACK_FIELDS(value) + + nullifier_leaf_value() = default; + nullifier_leaf_value(const fr& v) + : value(v) + {} + nullifier_leaf_value(const nullifier_leaf_value& other) = default; + nullifier_leaf_value(nullifier_leaf_value&& other) = default; + nullifier_leaf_value& operator=(const nullifier_leaf_value& other) + { + if (this != &other) { + value = other.value; + } + return *this; + } + + nullifier_leaf_value& operator=(nullifier_leaf_value&& other) + { + if (this != &other) { + value = other.value; + } + return *this; + } + ~nullifier_leaf_value() = default; + + bool operator==(nullifier_leaf_value const& other) const { return value == other.value; } + + friend std::ostream& operator<<(std::ostream& os, const nullifier_leaf_value& v) + { + os << "value = " << v.value; + return os; + } + + fr get_fr_value() const { return value; } + + static nullifier_leaf_value empty() { return { 0 }; } +}; + +template struct indexed_leaf { + LeafType value; index_t nextIndex; fr nextValue; MSGPACK_FIELDS(value, nextIndex, nextValue) - bool operator==(indexed_leaf const& other) const + indexed_leaf() = default; + + indexed_leaf(const LeafType& val, index_t nextIdx, fr nextVal) + : value(val) + , nextIndex(nextIdx) + , nextValue(nextVal) + {} + + indexed_leaf(const indexed_leaf& other) = default; + indexed_leaf(indexed_leaf&& other) noexcept = default; + ~indexed_leaf() = default; + + bool operator==(indexed_leaf const& other) const { return value == other.value && nextValue == other.nextValue && nextIndex == other.nextIndex; } - // indexed_leaf operator=(indexed_leaf const& other) const - // { - // return indexed_leaf{ .value = other.value, .nextIndex = other.nextIndex, .nextValue = other.nextValue }; - // } + indexed_leaf& operator=(indexed_leaf const& other) + { + if (this != &other) { + value = other.value; + nextValue = other.nextValue; + nextIndex = other.nextIndex; + } + return *this; + } + + indexed_leaf& operator=(indexed_leaf&& other) noexcept + { + if (this != &other) { + value = other.value; + nextValue = other.nextValue; + nextIndex = other.nextIndex; + } + return *this; + } - std::ostream& operator<<(std::ostream& os) + friend std::ostream& operator<<(std::ostream& os, const indexed_leaf& leaf) { - os << "value = " << value << "\nnextIdx = " << nextIndex << "\nnextVal = " << nextValue; + os << leaf.value << "\nnextIdx = " << leaf.nextIndex << "\nnextVal = " << leaf.nextValue; return os; } - std::vector get_hash_inputs() const { return std::vector({ value, nextIndex, nextValue }); } + std::vector get_hash_inputs() const { return std::vector({ value.value, nextIndex, nextValue }); } + + static indexed_leaf empty() { return { LeafType::empty(), 0, 0 }; } }; } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 40b2ace0f0c..76ec32ce3eb 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -69,8 +69,10 @@ class Signal { */ template class IndexedTree : public AppendOnlyTree { public: + using LeafValueType = typename Store::LeafType; + using IndexedLeafValueType = typename Store::IndexedLeafValueType; using add_completion_callback = std::function&, fr&, index_t)>; - using leaf_callback = std::function; + using leaf_callback = std::function; IndexedTree(Store& store, ThreadPool& workers, index_t initial_size); IndexedTree(IndexedTree const& other) = delete; @@ -82,7 +84,7 @@ template class IndexedTree : public App * @param value The value to be added or updated * @returns The 'previous' hash paths of all updated values */ - void add_or_update_value(const fr& value, const add_completion_callback& completion); + void add_or_update_value(const LeafValueType& value, const add_completion_callback& completion); /** * @brief Adds or updates the given set of values in the tree (updates not currently supported) @@ -90,7 +92,7 @@ template class IndexedTree : public App * @param no_multithreading Performs single threaded insertion, just used whilst prototyping and benchmarking * @returns The 'previous' hash paths of all updated values */ - void add_or_update_values(const std::vector& values, const add_completion_callback& completion); + void add_or_update_values(const std::vector& values, const add_completion_callback& completion); void get_leaf(const index_t& index, bool includeUncommitted, const leaf_callback& completion); @@ -103,20 +105,19 @@ template class IndexedTree : public App struct leaf_insertion { index_t low_leaf_index; - indexed_leaf low_leaf; + IndexedLeafValueType low_leaf; }; - // fr update_leaf_and_hash_to_root(const index_t& index, const indexed_leaf& leaf); void update_leaf_and_hash_to_root(const index_t& index, - const indexed_leaf& leaf, + const IndexedLeafValueType& leaf, Signal& leader, Signal& follower, fr_hash_path& previous_hash_path, ReadTransaction& tx); - using insertion_generation_callback = - std::function>, std::shared_ptr>)>; - void generate_insertions(std::shared_ptr>> values_to_be_sorted, + using insertion_generation_callback = std::function>, + std::shared_ptr>)>; + void generate_insertions(const std::shared_ptr>>& values_to_be_sorted, const insertion_generation_callback& completion); using insertion_completion_callback = std::function> paths)>; @@ -124,7 +125,7 @@ template class IndexedTree : public App const insertion_completion_callback& completion); using hash_generation_callback = std::function> hashes)>; - void generate_hashes_for_appending(std::shared_ptr> leaves_to_hash, + void generate_hashes_for_appending(std::shared_ptr> leaves_to_hash, const hash_generation_callback& completion); using AppendOnlyTree::get_element_or_zero; @@ -134,7 +135,6 @@ template class IndexedTree : public App using AppendOnlyTree::add_value; using AppendOnlyTree::add_values; - private: using AppendOnlyTree::store_; using AppendOnlyTree::zero_hashes_; using AppendOnlyTree::depth_; @@ -150,7 +150,7 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers zero_hashes_.resize(depth_ + 1); // Create the zero hashes for the tree - // indexed_leaf zero_leaf{ 0, 0, 0 }; + // Indexed_LeafType zero_leaf{ 0, 0, 0 }; auto current = fr::zero(); for (uint32_t i = depth_; i > 0; --i) { zero_hashes_[i] = current; @@ -171,14 +171,15 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers return; } - std::vector appended_leaves; + std::vector appended_leaves; std::vector appended_hashes; // Inserts the initial set of leaves as a chain in incrementing value order for (uint32_t i = 0; i < initial_size; ++i) { // Insert the zero leaf to the `leaves` and also to the tree at index 0. bool last = i == (initial_size - 1); - indexed_leaf initial_leaf = - indexed_leaf{ .value = i, .nextIndex = last ? 0 : i + 1, .nextValue = last ? 0 : i + 1 }; + IndexedLeafValueType initial_leaf = IndexedLeafValueType::empty(); + initial_leaf.nextIndex = last ? 0 : i + 1; + initial_leaf.nextValue = last ? 0 : i + 1; appended_leaves.push_back(initial_leaf); appended_hashes.push_back(HashingPolicy::hash(initial_leaf.get_hash_inputs())); store_.set_at_index(i, initial_leaf, true); @@ -197,25 +198,29 @@ void IndexedTree::get_leaf(const index_t& index, const leaf_callback& completion) { auto job = [=]() { - ReadTransactionPtr tx = store_.createReadTransaction(); - indexed_leaf leaf = store_.get_leaf(index, *tx, includeUncommitted); + IndexedLeafValueType leaf; + { + ReadTransactionPtr tx = store_.createReadTransaction(); + leaf = store_.get_leaf(index, *tx, includeUncommitted); + } completion(leaf); }; workers_.enqueue(job); } template -void IndexedTree::add_or_update_value(const fr& value, const add_completion_callback& completion) +void IndexedTree::add_or_update_value(const LeafValueType& value, + const add_completion_callback& completion) { - add_or_update_values(std::vector{ value }, completion); + add_or_update_values(std::vector{ value }, completion); } template -void IndexedTree::add_or_update_values(const std::vector& values, +void IndexedTree::add_or_update_values(const std::vector& values, const add_completion_callback& completion) { - std::shared_ptr>> values_to_be_sorted = - std::make_shared>>(values.size()); + std::shared_ptr>> values_to_be_sorted = + std::make_shared>>(values.size()); for (size_t i = 0; i < values.size(); ++i) { (*values_to_be_sorted)[i] = std::make_pair(values[i], i); } @@ -234,14 +239,14 @@ void IndexedTree::add_or_update_values(const std::vectorpaths, root, size); }; - hash_generation_callback hash_completion = [=](std::shared_ptr> hashes_to_append) { + hash_generation_callback hash_completion = [=](const std::shared_ptr>& hashes_to_append) { results->hashes_to_append = hashes_to_append; if (results->count.fetch_sub(1) == 1) { AppendOnlyTree::add_values(*hashes_to_append, final_completion); } }; - insertion_completion_callback insertion_completion = [=](std::shared_ptr> paths) { + insertion_completion_callback insertion_completion = [=](const std::shared_ptr>& paths) { results->paths = paths; if (results->count.fetch_sub(1) == 1) { AppendOnlyTree::add_values(*results->hashes_to_append, final_completion); @@ -250,7 +255,7 @@ void IndexedTree::add_or_update_values(const std::vector> insertions, - std::shared_ptr> leaves_to_append) { + std::shared_ptr> leaves_to_append) { workers_.enqueue([=]() { generate_hashes_for_appending(leaves_to_append, hash_completion); }); perform_insertions(insertions, insertion_completion); }; @@ -289,10 +294,10 @@ void IndexedTree::perform_insertions(std::shared_ptr void IndexedTree::generate_hashes_for_appending( - std::shared_ptr> leaves_to_hash, const hash_generation_callback& completion) + std::shared_ptr> leaves_to_hash, const hash_generation_callback& completion) { std::shared_ptr> hashed = std::make_shared>(leaves_to_hash->size()); - std::vector& leaves = *leaves_to_hash; + std::vector& leaves = *leaves_to_hash; for (uint32_t i = 0; i < leaves.size(); ++i) { (*hashed)[i] = HashingPolicy::hash(leaves[i].get_hash_inputs()); } @@ -301,45 +306,44 @@ void IndexedTree::generate_hashes_for_appending( template void IndexedTree::generate_insertions( - std::shared_ptr>> values_to_be_sorted, + const std::shared_ptr>>& values_to_be_sorted, const insertion_generation_callback& on_completion) { // The first thing we do is sort the values into descending order but maintain knowledge of their orignal order struct { - bool operator()(const std::pair& a, const std::pair& b) const + bool operator()(std::pair& a, std::pair& b) const { - return uint256_t(a.first) > uint256_t(b.first); + return uint256_t(a.first.get_fr_value()) > uint256_t(b.first.get_fr_value()); } } comp; std::sort(values_to_be_sorted->begin(), values_to_be_sorted->end(), comp); - std::vector>& values = *values_to_be_sorted; + std::vector>& values = *values_to_be_sorted; // Now that we have the sorted values we need to identify the leaves that need updating. // This is performed sequentially and is stored in this 'leaf_insertion' struct std::shared_ptr> insertions = std::make_shared>(values.size()); - std::shared_ptr> leaves_to_append = - std::make_shared>(values.size()); + std::shared_ptr> leaves_to_append = + std::make_shared>(values.size()); index_t old_size = 0; { ReadTransactionPtr tx = store_.createReadTransaction(); bb::fr old_root = fr::zero(); store_.get_meta(old_size, old_root, *tx, true); for (size_t i = 0; i < values.size(); ++i) { - fr value = values[i].first; + fr value = values[i].first.get_fr_value(); size_t index_into_appended_leaves = values[i].second; - index_t index_of_new_leaf = index_t(index_into_appended_leaves) + old_size; + index_t index_of_new_leaf = static_cast(index_into_appended_leaves) + old_size; // This gives us the leaf that need updating index_t current = 0; bool is_already_present = false; std::tie(is_already_present, current) = store_.find_low_value(values[i].first, true, *tx); - indexed_leaf current_leaf = store_.get_leaf(current, *tx, true); + IndexedLeafValueType current_leaf = store_.get_leaf(current, *tx, true); - indexed_leaf new_leaf = indexed_leaf{ .value = value, - .nextIndex = current_leaf.nextIndex, - .nextValue = current_leaf.nextValue }; + IndexedLeafValueType new_leaf = + IndexedLeafValueType(values[i].first, current_leaf.nextIndex, current_leaf.nextValue); // We only handle new values being added. We don't yet handle values being updated if (!is_already_present) { @@ -348,6 +352,7 @@ void IndexedTree::generate_insertions( current_leaf.nextValue = value; store_.set_at_index(current, current_leaf, false); store_.set_at_index(index_of_new_leaf, new_leaf, true); + } else { } (*leaves_to_append)[index_into_appended_leaves] = new_leaf; @@ -355,9 +360,8 @@ void IndexedTree::generate_insertions( // Capture the index and value of the updated 'low' leaf as well as the new leaf to be appended leaf_insertion& insertion = (*insertions)[i]; insertion.low_leaf_index = current; - insertion.low_leaf = indexed_leaf{ .value = current_leaf.value, - .nextIndex = current_leaf.nextIndex, - .nextValue = current_leaf.nextValue }; + insertion.low_leaf = + IndexedLeafValueType(current_leaf.value, current_leaf.nextIndex, current_leaf.nextValue); } } on_completion(insertions, leaves_to_append); @@ -365,7 +369,7 @@ void IndexedTree::generate_insertions( template void IndexedTree::update_leaf_and_hash_to_root(const index_t& leaf_index, - const indexed_leaf& leaf, + const IndexedLeafValueType& leaf, Signal& leader, Signal& follower, fr_hash_path& previous_hash_path, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index 2bd6c422c21..55c65a25521 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -9,10 +9,8 @@ #include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_leaves_store.hpp" #include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" #include "barretenberg/numeric/random/engine.hpp" -#include "leaves_cache.hpp" #include #include #include @@ -23,8 +21,9 @@ using namespace bb::crypto::merkle_tree; using HashPolicy = Poseidon2HashPolicy; -using Store = CachedTreeStore; +using Store = CachedTreeStore; using TreeType = IndexedTree; +using IndexedLeafType = indexed_leaf; using CompletionCallback = TreeType::add_completion_callback; @@ -90,11 +89,11 @@ fr_hash_path get_hash_path(TreeType& tree, index_t index, bool includeUncommitte return h; } -indexed_leaf get_leaf(TreeType& tree, index_t index, bool includeUncommitted = true) +IndexedLeafType get_leaf(TreeType& tree, index_t index, bool includeUncommitted = true) { - indexed_leaf l; + IndexedLeafType l; Signal signal(1); - auto completion = [&](const indexed_leaf& leaf) -> void { + auto completion = [&](const IndexedLeafType& leaf) -> void { l = leaf; signal.signal_level(0); }; @@ -126,7 +125,7 @@ void add_value(TreeType& tree, const fr& value) signal.wait_for_level(0); } -void add_values(TreeType& tree, const std::vector& values) +void add_values(TreeType& tree, const std::vector& values) { Signal signal(1); auto completion = [&](const std::vector&, fr&, index_t) -> void { signal.signal_level(0); }; @@ -303,11 +302,11 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) check_hash_path(tree2, 512, memdb.get_hash_path(512)); for (uint32_t i = 0; i < num_batches; i++) { - std::vector batch; + std::vector batch; std::vector memory_tree_hash_paths; for (uint32_t j = 0; j < batch_size; j++) { batch.emplace_back(random_engine.get_random_uint256()); - fr_hash_path path = memdb.update_element(batch[j]); + fr_hash_path path = memdb.update_element(batch[j].value); memory_tree_hash_paths.push_back(path); } std::vector tree1_hash_paths; @@ -345,12 +344,12 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) } } -fr hash_leaf(const indexed_leaf& leaf) +fr hash_leaf(const IndexedLeafType& leaf) { return HashPolicy::hash(leaf.get_hash_inputs()); } -bool verify_hash_path(TreeType& tree, const indexed_leaf& leaf_value, const uint32_t idx) +bool verify_hash_path(TreeType& tree, const IndexedLeafType& leaf_value, const uint32_t idx) { fr root = get_root(tree, true); fr_hash_path path = get_hash_path(tree, idx, true); @@ -366,6 +365,11 @@ bool verify_hash_path(TreeType& tree, const indexed_leaf& leaf_value, const uint return current == root; } +IndexedLeafType create_indexed_nullifier_leaf(const fr& value, index_t nextIndex, const fr& nextValue) +{ + return IndexedLeafType(nullifier_leaf_value(value), nextIndex, nextValue); +} + TEST_F(PersistedIndexedTreeTest, test_indexed_memory) { ThreadPool workers(8); @@ -385,7 +389,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) * nextIdx 0 0 0 0 0 0 0 0 * nextVal 0 0 0 0 0 0 0 0 */ - indexed_leaf zero_leaf = { 0, 0, 0 }; + IndexedLeafType zero_leaf(nullifier_leaf_value(0), 0, 0); check_size(tree, 1); EXPECT_EQ(get_leaf(tree, 0), zero_leaf); @@ -400,8 +404,8 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) */ add_value(tree, 30); check_size(tree, 2); - EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 1, 30 })); - EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(create_indexed_nullifier_leaf(0, 1, 30))); + EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf(create_indexed_nullifier_leaf(30, 0, 0))); /** * Add new value 10: @@ -414,9 +418,9 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) */ add_value(tree, 10); check_size(tree, 3); - EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 2, 10 })); - EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 0, 0 })); - EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf({ 10, 1, 30 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(create_indexed_nullifier_leaf(0, 2, 10))); + EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf(create_indexed_nullifier_leaf(30, 0, 0))); + EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf(create_indexed_nullifier_leaf(10, 1, 30))); /** * Add new value 20: @@ -429,10 +433,10 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) */ add_value(tree, 20); check_size(tree, 4); - EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 2, 10 })); - EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 0, 0 })); - EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf({ 10, 3, 20 })); - EXPECT_EQ(hash_leaf(get_leaf(tree, 3)), hash_leaf({ 20, 1, 30 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(create_indexed_nullifier_leaf(0, 2, 10))); + EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf(create_indexed_nullifier_leaf(30, 0, 0))); + EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf(create_indexed_nullifier_leaf(10, 3, 20))); + EXPECT_EQ(hash_leaf(get_leaf(tree, 3)), hash_leaf(create_indexed_nullifier_leaf(20, 1, 30))); // Adding the same value must not affect anything // tree.update_element(20); @@ -453,11 +457,11 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) */ add_value(tree, 50); check_size(tree, 5); - EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf({ 0, 2, 10 })); - EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf({ 30, 4, 50 })); - EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf({ 10, 3, 20 })); - EXPECT_EQ(hash_leaf(get_leaf(tree, 3)), hash_leaf({ 20, 1, 30 })); - EXPECT_EQ(hash_leaf(get_leaf(tree, 4)), hash_leaf({ 50, 0, 0 })); + EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(create_indexed_nullifier_leaf(0, 2, 10))); + EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf(create_indexed_nullifier_leaf(30, 4, 50))); + EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf(create_indexed_nullifier_leaf(10, 3, 20))); + EXPECT_EQ(hash_leaf(get_leaf(tree, 3)), hash_leaf(create_indexed_nullifier_leaf(20, 1, 30))); + EXPECT_EQ(hash_leaf(get_leaf(tree, 4)), hash_leaf(create_indexed_nullifier_leaf(50, 0, 0))); // Manually compute the node values auto e000 = hash_leaf(get_leaf(tree, 0)); @@ -516,7 +520,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_tree) Store store(name, depth, db); auto tree = TreeType(store, workers, 1); - indexed_leaf zero_leaf = { 0, 0, 0 }; + IndexedLeafType zero_leaf = create_indexed_nullifier_leaf(0, 0, 0); check_size(tree, 1); EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(zero_leaf)); @@ -540,8 +544,8 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_tree) fr new_member = fr::random_element(); std::vector differences; for (uint32_t i = 0; i < uint32_t(21); i++) { - uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(get_leaf(tree, i).value)); - uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(get_leaf(tree, i).value)); + uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(get_leaf(tree, i).value.get_fr_value())); + uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(get_leaf(tree, i).value.get_fr_value())); differences.push_back(diff_hi + diff_lo); } auto it = std::min_element(differences.begin(), differences.end()); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp deleted file mode 100644 index cd29811455d..00000000000 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "leaves_cache.hpp" - -namespace bb::crypto::merkle_tree { - -index_t LeavesCache::get_size() const -{ - return index_t(leaves_.size()); -} - -std::pair LeavesCache::find_low_value(const fr& new_value) const -{ - std::map::const_iterator it = indices_.lower_bound(new_value); - if (it == indices_.end()) { - // there is no element >= the requested value. - // decrement the iterator to get the value preceeding the requested value - --it; - return std::make_pair(false, it->second); - } - if (it->first == uint256_t(new_value)) { - // the value is already present and the iterator points to it - return std::make_pair(true, it->second); - } - // the iterator points to the element immediately larger than the requested value - --it; - // it now points to the value less than that requested - return std::make_pair(false, it->second); -} -indexed_leaf LeavesCache::get_leaf(const index_t& index) const -{ - ASSERT(index >= 0 && index < leaves_.size()); - return leaves_[size_t(index)]; -} -void LeavesCache::set_at_index(const index_t& index, const indexed_leaf& leaf, bool add_to_index) -{ - if (index >= leaves_.size()) { - leaves_.resize(size_t(index + 1)); - } - leaves_[size_t(index)] = leaf; - if (add_to_index) { - indices_[uint256_t(leaf.value)] = index; - } -} -void LeavesCache::append_leaf(const indexed_leaf& leaf) -{ - index_t next_index = leaves_.size(); - set_at_index(next_index, leaf, true); -} - -} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp deleted file mode 100644 index 9bf01c4015b..00000000000 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once -#include "barretenberg/stdlib/primitives/field/field.hpp" -#include "indexed_leaf.hpp" - -namespace bb::crypto::merkle_tree { - -/** - * @brief Used to facilitate testing of the IndexedTree. Stores leaves in memory with an index for O(logN) retrieval of - * 'low leaves' - * - */ -class LeavesCache { - public: - index_t get_size() const; - std::pair find_low_value(const bb::fr& new_value) const; - indexed_leaf get_leaf(const index_t& index) const; - void set_at_index(const index_t& index, const indexed_leaf& leaf, bool add_to_index); - void append_leaf(const indexed_leaf& leaf); - - private: - std::map indices_; - std::vector leaves_; -}; - -} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp index a9a6ea3615d..fb9f5527b19 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp @@ -28,7 +28,6 @@ int Invalid(const MDB_val*, const MDB_val*) int IntegerKeyCmp(const MDB_val* a, const MDB_val* b) { - // std::cout << "A size " << a->mv_size << " B size " << b->mv_size << std::endl; if (a->mv_size != b->mv_size) { return SizeCmp(a, b); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_leaves_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_leaves_store.hpp deleted file mode 100644 index 006f011aea4..00000000000 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_leaves_store.hpp +++ /dev/null @@ -1,126 +0,0 @@ -#pragma once -#include "./tree_meta.hpp" -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/numeric/uint256/uint256.hpp" -#include "barretenberg/stdlib/primitives/field/field.hpp" -#include "msgpack/assert.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace bb::crypto::merkle_tree { - -/** - * @brief Serves as a key-value node store for merkle trees, uses an unordered_map as a cache - */ -template class CachedLeavesStore { - - public: - using ReadTransaction = typename PersistedStore::ReadTransaction; - using WriteTransaction = typename PersistedStore::WriteTransaction; - using ReadTransactionPtr = std::unique_ptr; - using WriteTransactionPtr = std::unique_ptr; - - CachedLeavesStore(PersistedStore& leavesStore, index_t expected_size) - : leavesStore(leavesStore) - { - initialise(expected_size); - } - ~CachedLeavesStore() = default; - - CachedLeavesStore() = delete; - CachedLeavesStore(CachedLeavesStore const& other) = delete; - CachedLeavesStore(CachedLeavesStore const&& other) = delete; - CachedLeavesStore& operator=(CachedLeavesStore const& other) = delete; - CachedLeavesStore& operator=(CachedLeavesStore const&& other) = delete; - - index_t get_size(bool includeUncommitted) const { return meta.size + (includeUncommitted ? leaves_.size() : 0); } - - void commit() - { - { - if (leaves_.size() != indices_.size()) { - throw std::runtime_error("Inconsistent sizes in leaves data store"); - } - WriteTransactionPtr tx = createWriteTransaction(); - for (auto& idx : indices_) { - std::vector key; - std::vector value; - write(key, idx.first); - write(value, idx.second); - tx->put_value(key, value); - } - for (uint32_t i = 0; i < leaves_.size(); ++i) { - msgpack::sbuffer buffer; - msgpack::pack(buffer, leaves_[i]); - std::vector value(buffer.data(), buffer.data() + buffer.size()); - std::vector key; - write(key, index_t(meta.size + i)); - tx->put_value(key, value); - } - meta.size += leaves_.size(); - persistMeta(meta, *tx); - } - rollback(); - }; - void rollback() - { - indices_ = std::map(); - leaves_ = std::vector(); - }; - - ReadTransactionPtr createReadTransaction() const { return leavesStore.createReadNodeTransaction(); } - WriteTransactionPtr createWriteTransaction() const { return leavesStore.createWriteNodeTransaction(); } - - private: - std::map indices_; - std::vector leaves_; - PersistedStore& leavesStore; - LeavesMeta meta; - - bool readPersistedMeta(LeavesMeta& m, ReadTransaction& tx) const - { - std::vector data; - std::vector key{ 0 }; - bool success = tx.get_value(key, data); - if (success) { - msgpack::unpack((const char*)data.data(), data.size()).get().convert(m); - } - return success; - } - - void persistMeta(LeavesMeta& m, WriteTransaction& tx) - { - msgpack::sbuffer buffer; - msgpack::pack(buffer, m); - std::vector encoded(buffer.data(), buffer.data() + buffer.size()); - std::vector key{ 0 }; - tx.put_value(key, encoded); - } - - void initialise(index_t expected_size) - { - std::vector data; - { - ReadTransactionPtr tx = createReadTransaction(); - bool success = readPersistedMeta(meta, *tx); - if (success) { - if (expected_size == meta.size) { - return; - } - throw std::runtime_error("Invalid tree meta data"); - } - } - - meta.size = 0; - WriteTransactionPtr tx = createWriteTransaction(); - persistMeta(meta, *tx); - } -}; -} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp index afce7ba957c..5bab4133843 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp @@ -16,9 +16,10 @@ namespace bb::crypto::merkle_tree { /** * @brief Serves as a key-value node store for merkle trees, uses an unordered_map as a cache */ -template class CachedTreeStore { - +template class CachedTreeStore { public: + using LeafType = LeafValueType; + using IndexedLeafValueType = indexed_leaf; using ReadTransaction = typename PersistedStore::ReadTransaction; using WriteTransaction = typename PersistedStore::WriteTransaction; using ReadTransactionPtr = std::unique_ptr; @@ -41,13 +42,13 @@ template class CachedTreeStore { CachedTreeStore& operator=(CachedTreeStore const& other) = delete; CachedTreeStore& operator=(CachedTreeStore const&& other) = delete; - std::pair find_low_value(const bb::fr& new_value, + std::pair find_low_value(const LeafValueType& new_value, bool includeUncommitted, ReadTransaction& tx) const; - LeafType get_leaf(const index_t& index, ReadTransaction& tx, bool includeUncommitted) const; + IndexedLeafValueType get_leaf(const index_t& index, ReadTransaction& tx, bool includeUncommitted) const; - void set_at_index(const index_t& index, const LeafType& leaf, bool add_to_index); + void set_at_index(const index_t& index, const IndexedLeafValueType& leaf, bool add_to_index); void put_node(uint32_t level, index_t index, const std::vector& data); @@ -80,7 +81,7 @@ template class CachedTreeStore { uint32_t depth; std::vector>> nodes; std::map indices_; - std::unordered_map leaves_; + std::unordered_map leaves_; PersistedStore& dataStore; TreeMeta meta; @@ -91,14 +92,14 @@ template class CachedTreeStore { void persistMeta(TreeMeta& m, WriteTransaction& tx); }; -template -std::pair CachedTreeStore::find_low_value(const bb::fr& new_value, - bool includeUncommitted, - ReadTransaction& tx) const +template +std::pair CachedTreeStore::find_low_value(const LeafValueType& new_value, + bool includeUncommitted, + ReadTransaction& tx) const { - uint256_t new_value_as_number = uint256_t(new_value); + uint256_t new_value_as_number = uint256_t(new_value.get_fr_value()); std::vector data; - FrKeyType key(new_value); + FrKeyType key(new_value.get_fr_value()); tx.get_value_or_previous(key, data); index_t db_index = from_buffer(data, 0); uint256_t retrieved_value = key; @@ -107,7 +108,7 @@ std::pair CachedTreeStore::find_low_val } // At this stage, we have been asked to include uncommitted and the value was not exactly found in the db - std::map::const_iterator it = indices_.lower_bound(new_value); + std::map::const_iterator it = indices_.lower_bound(new_value_as_number); if (it == indices_.end()) { // there is no element >= the requested value. // decrement the iterator to get the value preceeding the requested value @@ -117,7 +118,7 @@ std::pair CachedTreeStore::find_low_val return std::make_pair(false, it->first > retrieved_value ? it->second : db_index); } - if (it->first == uint256_t(new_value)) { + if (it->first == uint256_t(new_value_as_number)) { // the value is already present and the iterator points to it return std::make_pair(true, it->second); } @@ -134,18 +135,18 @@ std::pair CachedTreeStore::find_low_val return std::make_pair(false, it->first > retrieved_value ? it->second : db_index); } -template -LeafType CachedTreeStore::get_leaf(const index_t& index, - ReadTransaction& tx, - bool includeUncommitted) const +template +typename CachedTreeStore::IndexedLeafValueType CachedTreeStore< + PersistedStore, + LeafValueType>::get_leaf(const index_t& index, ReadTransaction& tx, bool includeUncommitted) const { - LeafType return_value; if (includeUncommitted) { - typename std::unordered_map::const_iterator it = leaves_.find(index); + typename std::unordered_map::const_iterator it = leaves_.find(index); if (it != leaves_.end()) { return it->second; } } + IndexedLeafValueType return_value; LeafIndexKeyType key = index; std::vector data; bool success = tx.get_value_by_integer(key, data); @@ -155,31 +156,31 @@ LeafType CachedTreeStore::get_leaf(const index_t& inde return return_value; } -template -void CachedTreeStore::set_at_index(const index_t& index, - const LeafType& leaf, - bool add_to_index) +template +void CachedTreeStore::set_at_index(const index_t& index, + const IndexedLeafValueType& leaf, + bool add_to_index) { leaves_[index] = leaf; if (add_to_index) { - indices_[uint256_t(leaf.value)] = index; + indices_[uint256_t(leaf.value.get_fr_value())] = index; } } -template -void CachedTreeStore::put_node(uint32_t level, - index_t index, - const std::vector& data) +template +void CachedTreeStore::put_node(uint32_t level, + index_t index, + const std::vector& data) { nodes[level][index] = data; } -template -bool CachedTreeStore::get_node(uint32_t level, - index_t index, - std::vector& data, - ReadTransaction& transaction, - bool includeUncommitted) const +template +bool CachedTreeStore::get_node(uint32_t level, + index_t index, + std::vector& data, + ReadTransaction& transaction, + bool includeUncommitted) const { if (includeUncommitted) { const auto& level_map = nodes[level]; @@ -192,18 +193,18 @@ bool CachedTreeStore::get_node(uint32_t level, return transaction.get_node(level, index, data); } -template -void CachedTreeStore::put_meta(const index_t& size, const bb::fr& root) +template +void CachedTreeStore::put_meta(const index_t& size, const bb::fr& root) { meta.root = root; meta.size = size; } -template -void CachedTreeStore::get_meta(index_t& size, - bb::fr& root, - ReadTransaction& tx, - bool includeUncommitted) const +template +void CachedTreeStore::get_meta(index_t& size, + bb::fr& root, + ReadTransaction& tx, + bool includeUncommitted) const { if (includeUncommitted) { size = meta.size; @@ -216,8 +217,8 @@ void CachedTreeStore::get_meta(index_t& size, root = m.root; } -template -void CachedTreeStore::get_full_meta( +template +void CachedTreeStore::get_full_meta( index_t& size, bb::fr& root, std::string& name, uint32_t& depth, ReadTransaction& tx, bool includeUncommitted) const { if (includeUncommitted) { @@ -235,7 +236,7 @@ void CachedTreeStore::get_full_meta( name = m.name; } -template void CachedTreeStore::commit() +template void CachedTreeStore::commit() { { WriteTransactionPtr tx = createWriteTransaction(); @@ -265,18 +266,19 @@ template void CachedTreeStore void CachedTreeStore::rollback() +template +void CachedTreeStore::rollback() { nodes = std::vector>>( depth + 1, std::unordered_map>()); indices_ = std::map(); - leaves_ = std::unordered_map(); + leaves_ = std::unordered_map(); ReadTransactionPtr tx = createReadTransaction(); readPersistedMeta(meta, *tx); } -template -bool CachedTreeStore::readPersistedMeta(TreeMeta& m, ReadTransaction& tx) const +template +bool CachedTreeStore::readPersistedMeta(TreeMeta& m, ReadTransaction& tx) const { std::vector data; bool success = tx.get_node(0, 0, data); @@ -286,8 +288,8 @@ bool CachedTreeStore::readPersistedMeta(TreeMeta& m, R return success; } -template -void CachedTreeStore::persistMeta(TreeMeta& m, WriteTransaction& tx) +template +void CachedTreeStore::persistMeta(TreeMeta& m, WriteTransaction& tx) { msgpack::sbuffer buffer; msgpack::pack(buffer, m); @@ -295,7 +297,8 @@ void CachedTreeStore::persistMeta(TreeMeta& m, WriteTr tx.put_node(0, 0, encoded); } -template void CachedTreeStore::initialise() +template +void CachedTreeStore::initialise() { std::vector data; { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp index e85e3d2af4b..c3d8b3369ce 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp @@ -6,14 +6,14 @@ namespace bb::crypto::merkle_tree { -struct nullifier_leaf { +struct indexed_nullifier_leaf { fr value; index_t nextIndex; fr nextValue; // For serialization, update with any new fields MSGPACK_FIELDS(value, nextIndex, nextValue); - bool operator==(nullifier_leaf const&) const = default; + bool operator==(indexed_nullifier_leaf const&) const = default; std::ostream& operator<<(std::ostream& os) { @@ -23,7 +23,10 @@ struct nullifier_leaf { std::vector get_hash_inputs() const { return std::vector{ value, nextIndex, nextValue }; } - static nullifier_leaf zero() { return nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }; } + static indexed_nullifier_leaf zero() + { + return indexed_nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }; + } }; /** @@ -34,7 +37,7 @@ template class WrappedNullifierLeaf { public: // Initialize with a nullifier leaf - WrappedNullifierLeaf(nullifier_leaf value) + WrappedNullifierLeaf(indexed_nullifier_leaf value) : data(value) {} // Initialize an empty leaf @@ -57,14 +60,14 @@ template class WrappedNullifierLeaf { * * @return nullifier_leaf */ - nullifier_leaf unwrap() const { return data.value(); } + indexed_nullifier_leaf unwrap() const { return data.value(); } /** * @brief Set the wrapped nullifier_leaf object value * * @param value */ - void set(nullifier_leaf value) { data.emplace(value); } + void set(indexed_nullifier_leaf value) { data.emplace(value); } /** * @brief Return the hash of the wrapped object, other return the zero hash of 0 @@ -85,7 +88,7 @@ template class WrappedNullifierLeaf { private: // Underlying data - std::optional data; + std::optional data; }; template diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp index 550f4132663..b8f44e91463 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -77,7 +77,8 @@ template class NullifierMemoryTree : public MemoryTree< const std::vector& get_hashes() { return hashes_; } const WrappedNullifierLeaf get_leaf(size_t index) { - return (index < leaves_.size()) ? leaves_[index] : WrappedNullifierLeaf(nullifier_leaf::zero()); + return (index < leaves_.size()) ? leaves_[index] + : WrappedNullifierLeaf(indexed_nullifier_leaf::zero()); } const std::vector>& get_leaves() { return leaves_; } @@ -111,13 +112,13 @@ NullifierMemoryTree::NullifierMemoryTree(size_t depth, size_t ini // Insert the initial leaves for (size_t i = 0; i < initial_size; i++) { - auto initial_leaf = - WrappedNullifierLeaf(nullifier_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }); + auto initial_leaf = WrappedNullifierLeaf( + indexed_nullifier_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }); leaves_.push_back(initial_leaf); } leaves_[initial_size - 1] = WrappedNullifierLeaf( - nullifier_leaf{ .value = leaves_[initial_size - 1].unwrap().value, .nextIndex = 0, .nextValue = 0 }); + indexed_nullifier_leaf{ .value = leaves_[initial_size - 1].unwrap().value, .nextIndex = 0, .nextValue = 0 }); for (size_t i = 0; i < initial_size; ++i) { update_element(i, leaves_[i].hash()); @@ -142,10 +143,10 @@ template fr_hash_path NullifierMemoryTree hashes, std::string const& m } } -bool check_hash_path(const fr& root, const fr_hash_path& path, const nullifier_leaf& leaf_value, const size_t idx) +bool check_hash_path(const fr& root, + const fr_hash_path& path, + const indexed_nullifier_leaf& leaf_value, + const size_t idx) { auto current = WrappedLeaf(leaf_value).hash(); size_t depth_ = path.size(); @@ -50,7 +53,7 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) * nextIdx 0 0 0 0 0 0 0 0 * nextVal 0 0 0 0 0 0 0 0 */ - nullifier_leaf zero_leaf = { 0, 0, 0 }; + indexed_nullifier_leaf zero_leaf = { 0, 0, 0 }; EXPECT_EQ(tree.get_leaves().size(), 1); EXPECT_EQ(tree.get_leaves()[0].unwrap(), zero_leaf); @@ -344,7 +347,7 @@ TEST(crypto_nullifier_tree, test_nullifier_tree) constexpr size_t depth = 8; NullifierMemoryTree tree(depth); - nullifier_leaf zero_leaf = { 0, 0, 0 }; + indexed_nullifier_leaf zero_leaf = { 0, 0, 0 }; EXPECT_EQ(tree.get_leaves().size(), 1); EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf(zero_leaf).hash()); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp index 95ee0ea72ab..0b7f3adffe9 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp @@ -21,7 +21,8 @@ NullifierTree::NullifierTree(Store& store, size_t depth, s // Create the zero hashes for the tree auto current = - WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }).hash(); + WrappedNullifierLeaf(indexed_nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }) + .hash(); for (size_t i = 0; i < depth; ++i) { zero_hashes_[i] = current; current = HashingPolicy::hash_pair(current, current); @@ -29,13 +30,13 @@ NullifierTree::NullifierTree(Store& store, size_t depth, s // Insert the initial leaves for (size_t i = 0; i < initial_size; i++) { - auto initial_leaf = - WrappedNullifierLeaf(nullifier_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }); + auto initial_leaf = WrappedNullifierLeaf( + indexed_nullifier_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }); leaves.push_back(initial_leaf); } leaves[initial_size - 1] = WrappedNullifierLeaf( - nullifier_leaf{ .value = leaves[initial_size - 1].unwrap().value, .nextIndex = 0, .nextValue = 0 }); + indexed_nullifier_leaf{ .value = leaves[initial_size - 1].unwrap().value, .nextIndex = 0, .nextValue = 0 }); for (size_t i = 0; i < initial_size; ++i) { update_element(i, leaves[i].hash()); @@ -57,7 +58,7 @@ fr NullifierTree::update_element(fr const& value) bool is_already_present; std::tie(current, is_already_present) = find_closest_leaf(leaves, value); - nullifier_leaf current_leaf = leaves[current].unwrap(); + indexed_nullifier_leaf current_leaf = leaves[current].unwrap(); WrappedNullifierLeaf new_leaf = WrappedNullifierLeaf( { .value = value, .nextIndex = current_leaf.nextIndex, .nextValue = current_leaf.nextValue }); if (!is_already_present) { diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp index 92f3325e9c1..957177902b2 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp @@ -5,8 +5,6 @@ #include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_store.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/cbind.hpp" #include "barretenberg/world_state/service/message.hpp" From 6c7a0c33ca5aeb20e82cc0e5fca8b323458a283a Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 26 Jun 2024 16:49:08 +0100 Subject: [PATCH 24/63] WIP --- .../src/barretenberg/common/thread_pool.hpp | 1 + .../append_only_tree/append_only_tree.hpp | 2 + .../merkle_tree/indexed_tree/indexed_tree.hpp | 4 +- .../service/world_state_service.hpp | 50 ++++++++++++------- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp b/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp index fb6e7ca961c..bc8660238b4 100644 --- a/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp +++ b/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace bb { class ThreadPool { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index a2d4fef481f..0483a68c572 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -53,6 +53,8 @@ template class AppendOnlyTree { void commit(const commit_callback& on_completion); void rollback(const rollback_callback& on_completion); + uint32_t depth() const { return depth_; } + protected: using ReadTransaction = typename Store::ReadTransaction; using ReadTransactionPtr = typename Store::ReadTransactionPtr; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 76ec32ce3eb..9b16cb9e645 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -177,9 +177,7 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers for (uint32_t i = 0; i < initial_size; ++i) { // Insert the zero leaf to the `leaves` and also to the tree at index 0. bool last = i == (initial_size - 1); - IndexedLeafValueType initial_leaf = IndexedLeafValueType::empty(); - initial_leaf.nextIndex = last ? 0 : i + 1; - initial_leaf.nextValue = last ? 0 : i + 1; + IndexedLeafValueType initial_leaf = IndexedLeafValueType(LeafValueType(i), last ? 0 : i + 1, last ? 0 : i + 1); appended_leaves.push_back(initial_leaf); appended_hashes.push_back(HashingPolicy::hash(initial_leaf.get_hash_inputs())); store_.set_at_index(i, initial_leaf, true); diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp index 957177902b2..6c55cd004dd 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp @@ -2,9 +2,12 @@ #include "barretenberg/common/thread_pool.hpp" #include "barretenberg/crypto/merkle_tree//lmdb_store/lmdb_store.hpp" #include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp" +#include "barretenberg/crypto/merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" +#include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/cbind.hpp" #include "barretenberg/world_state/service/message.hpp" @@ -15,6 +18,7 @@ #include #include #include +#include namespace bb::world_state { using namespace bb::messaging; @@ -22,17 +26,22 @@ using namespace bb::crypto::merkle_tree; using HashPolicy = PedersenHashPolicy; struct TreeWithStore { - typedef CachedNodeStore Store; - std::unique_ptr> tree; + typedef CachedTreeStore Store; + std::unique_ptr lmdbStore; + std::unique_ptr> tree; std::unique_ptr store; - TreeWithStore(std::unique_ptr> t, std::unique_ptr s) - : tree(std::move(t)) + TreeWithStore(std::unique_ptr l, + std::unique_ptr> t, + std::unique_ptr s) + : lmdbStore(std::move(l)) + , tree(std::move(t)) , store(std::move(s)) {} TreeWithStore(TreeWithStore&& other) noexcept - : tree(std::move(other.tree)) + : lmdbStore(std::move(other.lmdbStore)) + , tree(std::move(other.tree)) , store(std::move(other.store)) {} @@ -47,15 +56,19 @@ template class WorldStateService { OutputStream& outputStream; std::unordered_map> trees; ThreadPool workers; + LMDBEnvironment lmdbEnvironment; public: - WorldStateService(OutputStream& out, size_t numThreads) + WorldStateService(OutputStream& out, uint32_t numThreads) : outputStream(out) , workers(numThreads) + , lmdbEnvironment(randomTempDirectory(), 1024, 50, numThreads) {} bool startTree(msgpack::object& obj); bool getTreeInfo(msgpack::object& obj); bool insertLeaves(msgpack::object& obj); + + private: }; template bool WorldStateService::startTree(msgpack::object& obj) @@ -67,16 +80,15 @@ template bool WorldStateService::startTree auto it = trees.find(startTreeRequest.value.name); if (it == trees.end()) { - auto store = std::make_unique(startTreeRequest.value.depth); + auto lmdbStore = + std::make_unique(lmdbEnvironment, startTreeRequest.value.name, false, false, IntegerKeyCmp); + auto store = std::make_unique( + startTreeRequest.value.name, startTreeRequest.value.depth, *lmdbStore); - auto tree = - std::make_unique>(*store, - startTreeRequest.value.depth, - workers, - startTreeRequest.value.preFilledSize, - startTreeRequest.value.name); + auto tree = std::make_unique>( + *store, workers, startTreeRequest.value.preFilledSize); - auto treeWithStore = std::make_unique(std::move(tree), std::move(store)); + auto treeWithStore = std::make_unique(std::move(lmdbStore), std::move(tree), std::move(store)); trees[startTreeRequest.value.name] = std::move(treeWithStore); } else { @@ -111,8 +123,8 @@ template bool WorldStateService::getTreeIn treeInfoResponse.message = ""; treeInfoResponse.success = true; treeInfoResponse.depth = static_cast(it->second->tree->depth()); - treeInfoResponse.root = it->second->tree->root(); - treeInfoResponse.size = static_cast(it->second->tree->size()); + // treeInfoResponse.root = it->second->tree->root(); + // treeInfoResponse.size = static_cast(it->second->tree->size()); } MsgHeader header(treeInfoRequest.header.messageId); @@ -155,7 +167,11 @@ template bool WorldStateService::insertLea WorldStateMsgTypes::INSERT_LEAVES_RESPONSE, header, response); outputStream.sendPackedObject(insertLeavesResponse); }; - it->second->tree->add_or_update_values(insertLeavesRequest.value.leaves, completion); + std::vector leaves(insertLeavesRequest.value.leaves.size()); + for (auto& v : insertLeavesRequest.value.leaves) { + leaves.emplace_back(v); + } + it->second->tree->add_or_update_values(leaves, completion); } return true; From ab66e668f3058c6736024f9b0a33550f300b4309 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 26 Jun 2024 16:53:09 +0100 Subject: [PATCH 25/63] WIP --- yarn-project/merkle-tree/package.json | 2 +- .../merkle-tree/src/interfaces/merkle_tree.ts | 2 +- yarn-project/merkle-tree/src/native/message.ts | 13 +++++++++++-- .../merkle-tree/src/native/native_client.ts | 2 +- .../src/standard_tree/standard_tree_native.ts | 2 +- yarn-project/merkle-tree/src/tree_base.ts | 4 ++-- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/yarn-project/merkle-tree/package.json b/yarn-project/merkle-tree/package.json index bf30b5e10e4..6541c9e320c 100644 --- a/yarn-project/merkle-tree/package.json +++ b/yarn-project/merkle-tree/package.json @@ -16,7 +16,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests" + "test": "yarn build && NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests" }, "inherits": [ "../package.common.json", diff --git a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts index 8a755a65b65..aed3d7236c2 100644 --- a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts @@ -72,7 +72,7 @@ export interface MerkleTree extends SiblingPathSo export interface SnapshottedMerkleTree extends MerkleTree { getName(): string; - getNode(level: number, index: bigint): Promise; + getNode(level: number, index: bigint): Buffer | undefined; getZeroHash(level: number): Buffer; } diff --git a/yarn-project/merkle-tree/src/native/message.ts b/yarn-project/merkle-tree/src/native/message.ts index e59315db75c..9dab2155d0d 100644 --- a/yarn-project/merkle-tree/src/native/message.ts +++ b/yarn-project/merkle-tree/src/native/message.ts @@ -13,8 +13,8 @@ export class MsgHeader { } } -export enum WorldStateMsgTypes { - START_TREE_REQUEST = FIRST_APP_MSG_TYPE, +export const enum WorldStateMsgTypes { + START_TREE_REQUEST = 100, START_TREE_RESPONSE, GET_TREE_INFO_REQUEST, GET_TREE_INFO_RESPONSE, @@ -22,6 +22,15 @@ export enum WorldStateMsgTypes { INSERT_LEAVES_RESPONSE, }; +// export const enum WorldStateMsgTypes { +// START_TREE_REQUEST = FIRST_APP_MSG_TYPE, +// START_TREE_RESPONSE = FIRST_APP_MSG_TYPE + 1, +// GET_TREE_INFO_REQUEST = FIRST_APP_MSG_TYPE + 2, +// GET_TREE_INFO_RESPONSE = FIRST_APP_MSG_TYPE + 3, +// INSERT_LEAVES_REQUEST = FIRST_APP_MSG_TYPE + 4, +// INSERT_LEAVES_RESPONSE = FIRST_APP_MSG_TYPE + 5, +// }; + export type StartTreeRequest = { name: string; depth: number; diff --git a/yarn-project/merkle-tree/src/native/native_client.ts b/yarn-project/merkle-tree/src/native/native_client.ts index 049bd921b2d..06e7a169868 100644 --- a/yarn-project/merkle-tree/src/native/native_client.ts +++ b/yarn-project/merkle-tree/src/native/native_client.ts @@ -71,7 +71,7 @@ export class NativeTreesClient { name, leaves: leaves.map(x => x.toBuffer()), }; - const { promise } = this.sendMessage(104, request); + const { promise } = this.sendMessage(WorldStateMsgTypes.INSERT_LEAVES_REQUEST, request); const response = (await promise) as InsertLeavesResponse; return response; } diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts index 0e208215d6f..267617212a5 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts @@ -46,7 +46,7 @@ export class StandardTreeNative implements Snapsh getName(): string { return this.name; } - getNode(level: number, index: bigint): Promise { + getNode(level: number, index: bigint): Buffer { throw new Error('Method not implemented.'); } getZeroHash(level: number): Buffer { diff --git a/yarn-project/merkle-tree/src/tree_base.ts b/yarn-project/merkle-tree/src/tree_base.ts index cc5edcbad2c..a842d37bb17 100644 --- a/yarn-project/merkle-tree/src/tree_base.ts +++ b/yarn-project/merkle-tree/src/tree_base.ts @@ -194,7 +194,7 @@ export abstract class TreeBase implements MerkleTree { return this.getLatestValueAtIndex(this.depth, index, includeUncommitted); } - public getNode(level: number, index: bigint): Promise { + public getNode(level: number, index: bigint): Buffer | undefined { if (level < 0 || level > this.depth) { throw Error('Invalid level: ' + level); } @@ -203,7 +203,7 @@ export abstract class TreeBase implements MerkleTree { throw Error('Invalid index: ' + index); } - return Promise.resolve(this.dbGet(indexToKeyHash(this.name, level, index))); + return this.dbGet(indexToKeyHash(this.name, level, index)); } public getZeroHash(level: number): Buffer { From 75452a00178a013293a3c2cdc6e55191ebec7403 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 27 Jun 2024 15:38:19 +0100 Subject: [PATCH 26/63] WIP --- .../append_only_tree/append_only_tree.hpp | 39 ++++++++-------- .../merkle_tree/indexed_tree/indexed_leaf.hpp | 44 +++++++++---------- .../merkle_tree/indexed_tree/indexed_tree.hpp | 16 ++++--- .../indexed_tree/indexed_tree.test.cpp | 6 +-- .../merkle_tree/lmdb_store/lmdb_store.hpp | 35 +++++++++++++-- .../merkle_tree/node_store/array_store.hpp | 4 +- .../node_store/cached_tree_store.hpp | 6 +-- 7 files changed, 90 insertions(+), 60 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 0483a68c572..67ec020fd0e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -20,11 +20,11 @@ using namespace bb; */ template class AppendOnlyTree { public: - using append_completion_callback = std::function; - using meta_data_callback = std::function; - using hash_path_callback = std::function; - using commit_callback = std::function; - using rollback_callback = std::function; + using AppendCompletionCallback = std::function; + using MetaDataCallback = std::function; + using HashPathCallback = std::function; + using CommitCallback = std::function; + using RollbackCallback = std::function; AppendOnlyTree(Store& store, ThreadPool& workers); AppendOnlyTree(AppendOnlyTree const& other) = delete; @@ -36,22 +36,22 @@ template class AppendOnlyTree { /** * @brief Adds a single value to the end of the tree */ - virtual void add_value(const fr& value, const append_completion_callback& on_completion); + virtual void add_value(const fr& value, const AppendCompletionCallback& on_completion); /** * @brief Adds the given set of values to the end of the tree */ - virtual void add_values(const std::vector& values, const append_completion_callback& on_completion); + virtual void add_values(const std::vector& values, const AppendCompletionCallback& on_completion); /** * @brief Returns the hash path from the leaf at the given index to the root */ - void get_hash_path(const index_t& index, const hash_path_callback& on_completion, bool includeUncommitted) const; + void get_hash_path(const index_t& index, const HashPathCallback& on_completion, bool includeUncommitted) const; - void get_meta_data(bool includeUncommitted, const meta_data_callback& on_completion); + void get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion); - void commit(const commit_callback& on_completion); - void rollback(const rollback_callback& on_completion); + void commit(const CommitCallback& on_completion); + void rollback(const RollbackCallback& on_completion); uint32_t depth() const { return depth_; } @@ -101,8 +101,7 @@ AppendOnlyTree::AppendOnlyTree(Store& store, ThreadPool& w } template -void AppendOnlyTree::get_meta_data(bool includeUncommitted, - const meta_data_callback& on_completion) +void AppendOnlyTree::get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion) { auto job = [=]() { index_t size = 0; @@ -118,7 +117,7 @@ void AppendOnlyTree::get_meta_data(bool includeUncommitted template void AppendOnlyTree::get_hash_path(const index_t& index, - const hash_path_callback& on_completion, + const HashPathCallback& on_completion, bool includeUncommitted) const { auto job = [=]() { @@ -143,14 +142,14 @@ void AppendOnlyTree::get_hash_path(const index_t& index, } template -void AppendOnlyTree::add_value(const fr& value, const append_completion_callback& on_completion) +void AppendOnlyTree::add_value(const fr& value, const AppendCompletionCallback& on_completion) { add_values(std::vector{ value }, on_completion); } template void AppendOnlyTree::add_values(const std::vector& values, - const append_completion_callback& on_completion) + const AppendCompletionCallback& on_completion) { uint32_t start_level = depth_; std::shared_ptr> hashes = std::make_shared>(values); @@ -160,12 +159,12 @@ void AppendOnlyTree::add_values(const std::vector& val index_t new_size = 0; { typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); - index_t start_size; + index_t start_size = 0; bb::fr root; store_.get_meta(start_size, root, *tx, true); index_t index = start_size; uint32_t level = start_level; - uint32_t number_to_insert = static_cast(values.size()); + auto number_to_insert = static_cast(values.size()); std::vector& hashes_local = *hashes; // Add the values at the leaf nodes of the tree for (uint32_t i = 0; i < number_to_insert; ++i) { @@ -243,7 +242,7 @@ std::pair AppendOnlyTree::read_node(uint32_t lev } template -void AppendOnlyTree::commit(const commit_callback& on_completion) +void AppendOnlyTree::commit(const CommitCallback& on_completion) { auto job = [=]() { store_.commit(); @@ -253,7 +252,7 @@ void AppendOnlyTree::commit(const commit_callback& on_comp } template -void AppendOnlyTree::rollback(const rollback_callback& on_completion) +void AppendOnlyTree::rollback(const RollbackCallback& on_completion) { auto job = [=]() { store_.rollback(); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp index 5f492dfcbe8..3e6015a370c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp @@ -6,18 +6,18 @@ namespace bb::crypto::merkle_tree { -struct nullifier_leaf_value { +struct NullifierLeafValue { fr value; MSGPACK_FIELDS(value) - nullifier_leaf_value() = default; - nullifier_leaf_value(const fr& v) + NullifierLeafValue() = default; + NullifierLeafValue(const fr& v) : value(v) {} - nullifier_leaf_value(const nullifier_leaf_value& other) = default; - nullifier_leaf_value(nullifier_leaf_value&& other) = default; - nullifier_leaf_value& operator=(const nullifier_leaf_value& other) + NullifierLeafValue(const NullifierLeafValue& other) = default; + NullifierLeafValue(NullifierLeafValue&& other) = default; + NullifierLeafValue& operator=(const NullifierLeafValue& other) { if (this != &other) { value = other.value; @@ -25,18 +25,18 @@ struct nullifier_leaf_value { return *this; } - nullifier_leaf_value& operator=(nullifier_leaf_value&& other) + NullifierLeafValue& operator=(NullifierLeafValue&& other) noexcept { if (this != &other) { value = other.value; } return *this; } - ~nullifier_leaf_value() = default; + ~NullifierLeafValue() = default; - bool operator==(nullifier_leaf_value const& other) const { return value == other.value; } + bool operator==(NullifierLeafValue const& other) const { return value == other.value; } - friend std::ostream& operator<<(std::ostream& os, const nullifier_leaf_value& v) + friend std::ostream& operator<<(std::ostream& os, const NullifierLeafValue& v) { os << "value = " << v.value; return os; @@ -44,34 +44,34 @@ struct nullifier_leaf_value { fr get_fr_value() const { return value; } - static nullifier_leaf_value empty() { return { 0 }; } + static NullifierLeafValue empty() { return { 0 }; } }; -template struct indexed_leaf { +template struct IndexedLeaf { LeafType value; index_t nextIndex; fr nextValue; MSGPACK_FIELDS(value, nextIndex, nextValue) - indexed_leaf() = default; + IndexedLeaf() = default; - indexed_leaf(const LeafType& val, index_t nextIdx, fr nextVal) + IndexedLeaf(const LeafType& val, index_t nextIdx, fr nextVal) : value(val) , nextIndex(nextIdx) , nextValue(nextVal) {} - indexed_leaf(const indexed_leaf& other) = default; - indexed_leaf(indexed_leaf&& other) noexcept = default; - ~indexed_leaf() = default; + IndexedLeaf(const IndexedLeaf& other) = default; + IndexedLeaf(IndexedLeaf&& other) noexcept = default; + ~IndexedLeaf() = default; - bool operator==(indexed_leaf const& other) const + bool operator==(IndexedLeaf const& other) const { return value == other.value && nextValue == other.nextValue && nextIndex == other.nextIndex; } - indexed_leaf& operator=(indexed_leaf const& other) + IndexedLeaf& operator=(IndexedLeaf const& other) { if (this != &other) { value = other.value; @@ -81,7 +81,7 @@ template struct indexed_leaf { return *this; } - indexed_leaf& operator=(indexed_leaf&& other) noexcept + IndexedLeaf& operator=(IndexedLeaf&& other) noexcept { if (this != &other) { value = other.value; @@ -91,7 +91,7 @@ template struct indexed_leaf { return *this; } - friend std::ostream& operator<<(std::ostream& os, const indexed_leaf& leaf) + friend std::ostream& operator<<(std::ostream& os, const IndexedLeaf& leaf) { os << leaf.value << "\nnextIdx = " << leaf.nextIndex << "\nnextVal = " << leaf.nextValue; return os; @@ -99,7 +99,7 @@ template struct indexed_leaf { std::vector get_hash_inputs() const { return std::vector({ value.value, nextIndex, nextValue }); } - static indexed_leaf empty() { return { LeafType::empty(), 0, 0 }; } + static IndexedLeaf empty() { return { LeafType::empty(), 0, 0 }; } }; } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 9b16cb9e645..0c387ebac6f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -61,12 +61,14 @@ class Signal { private: std::atomic signal_; -}; /** - * @brief Implements a parallelised batch insertion indexed tree - * Accepts template argument of the type of store backing the tree, the type of store containing the leaves and the - * hashing policy - * - */ +}; + +/** + * @brief Implements a parallelised batch insertion indexed tree + * Accepts template argument of the type of store backing the tree, the type of store containing the leaves and the + * hashing policy + * + */ template class IndexedTree : public AppendOnlyTree { public: using LeafValueType = typename Store::LeafType; @@ -78,6 +80,8 @@ template class IndexedTree : public App IndexedTree(IndexedTree const& other) = delete; IndexedTree(IndexedTree&& other) = delete; ~IndexedTree() = default; + IndexedTree& operator=(const IndexedTree& other) = delete; + IndexedTree& operator=(IndexedTree&& other) = delete; /** * @brief Adds or updates a single values in the tree (updates not currently supported) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index 55c65a25521..81d1057e396 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -21,11 +21,11 @@ using namespace bb::crypto::merkle_tree; using HashPolicy = Poseidon2HashPolicy; -using Store = CachedTreeStore; +using Store = CachedTreeStore; using TreeType = IndexedTree; -using IndexedLeafType = indexed_leaf; +using IndexedLeafType = IndexedLeaf; -using CompletionCallback = TreeType::add_completion_callback; +using CompletionCallback = TreeType::AddCompletionCallback; class PersistedIndexedTreeTest : public testing::Test { protected: diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp index cc0a1f04d9b..ff72889eba1 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp @@ -46,7 +46,7 @@ template bool call_lmdb_func(int (*f)(TArgs...), TArgs... ar if (error != 0 && error != MDB_NOTFOUND) { std::stringstream ss; ss << "ERROR: " << mdb_strerror(error) << std::endl; - std::cout << ss.str(); + std::cerr << ss.str(); } return error == 0; } @@ -59,6 +59,10 @@ template void call_lmdb_func(void (*f)(TArgs...), TArgs... a class LMDBEnvironment { public: LMDBEnvironment(const std::string& directory, uint64_t mapSizeMB, uint32_t maxNumDBs, uint32_t maxNumReaders); + LMDBEnvironment(const LMDBEnvironment& other) = delete; + LMDBEnvironment(LMDBEnvironment&& other) = delete; + LMDBEnvironment& operator=(const LMDBEnvironment& other) = delete; + LMDBEnvironment& operator=(LMDBEnvironment&& other) = delete; ~LMDBEnvironment() { call_lmdb_func(mdb_env_close, _mdbEnv); } @@ -85,9 +89,12 @@ class LMDBTransaction { { MDB_txn* p = nullptr; if (!call_lmdb_func(mdb_txn_begin, _environment.underlying(), p, readOnly ? MDB_RDONLY : 0U, &_transaction)) { - // throw here } } + LMDBTransaction(const LMDBTransaction& other) = delete; + LMDBTransaction(LMDBTransaction&& other) = delete; + LMDBTransaction& operator=(const LMDBTransaction& other) = delete; + LMDBTransaction& operator=(LMDBTransaction&& other) = delete; virtual ~LMDBTransaction() = default; @@ -125,6 +132,11 @@ class LMDBDatabase { } } + LMDBDatabase(const LMDBDatabase& other) = delete; + LMDBDatabase(LMDBDatabase&& other) = delete; + LMDBDatabase& operator=(const LMDBDatabase& other) = delete; + LMDBDatabase& operator=(LMDBDatabase&& other) = delete; + ~LMDBDatabase() { call_lmdb_func(mdb_dbi_close, _environment.underlying(), _dbi); } const MDB_dbi& underlying() const { return _dbi; } @@ -142,6 +154,10 @@ class LMDBReadTransaction : public LMDBTransaction { : LMDBTransaction(env, true) , _database(database) {} + LMDBReadTransaction(const LMDBReadTransaction& other) = delete; + LMDBReadTransaction(LMDBReadTransaction&& other) = delete; + LMDBReadTransaction& operator=(const LMDBReadTransaction& other) = delete; + LMDBReadTransaction& operator=(LMDBReadTransaction&& other) = delete; ~LMDBReadTransaction() override { abort(); } @@ -181,7 +197,7 @@ template bool LMDBReadTransaction::get_value_by_integer(T& key, std template bool LMDBReadTransaction::get_value_or_previous(T& key, std::vector& data) const { T keyCopy = key; - MDB_cursor* cursor; + MDB_cursor* cursor = nullptr; call_lmdb_func(mdb_cursor_open, underlying(), _database.underlying(), &cursor); MDB_val dbKey; @@ -261,7 +277,10 @@ class LMDBWriteTransaction : public LMDBTransaction { : LMDBTransaction(env) , _database(database) {} - + LMDBWriteTransaction(const LMDBWriteTransaction& other) = delete; + LMDBWriteTransaction(LMDBWriteTransaction&& other) = delete; + LMDBWriteTransaction& operator=(const LMDBWriteTransaction& other) = delete; + LMDBWriteTransaction& operator=(LMDBWriteTransaction&& other) = delete; ~LMDBWriteTransaction() override { commit(); } void put_node(uint32_t level, index_t index, std::vector& data); @@ -303,6 +322,10 @@ class LMDBDatabaseCreationTransaction : public LMDBTransaction { LMDBDatabaseCreationTransaction(LMDBEnvironment& env) : LMDBTransaction(env) {} + LMDBDatabaseCreationTransaction(const LMDBDatabaseCreationTransaction& other) = delete; + LMDBDatabaseCreationTransaction(LMDBDatabaseCreationTransaction&& other) = delete; + LMDBDatabaseCreationTransaction& operator=(const LMDBDatabaseCreationTransaction& other) = delete; + LMDBDatabaseCreationTransaction& operator=(LMDBDatabaseCreationTransaction&& other) = delete; ~LMDBDatabaseCreationTransaction() override { commit(); } @@ -324,6 +347,10 @@ class LMDBStore { , _name(std::move(name)) , _database(_environment, LMDBDatabaseCreationTransaction(_environment), _name, integerKeys, reverseKeys, cmp) {} + LMDBStore(const LMDBStore& other) = delete; + LMDBStore(LMDBStore&& other) = delete; + LMDBStore& operator=(const LMDBStore& other) = delete; + LMDBStore& operator=(LMDBStore&& other) = delete; ~LMDBStore() = default; LMDBWriteTransaction::Ptr createWriteTransaction() const diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp index cc30f7a3e43..701a1e603b7 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp @@ -26,8 +26,8 @@ class MockPersistedStore { public: using ReadTransaction = MockTransaction; using WriteTransaction = MockTransaction; - MockTransaction::Ptr createWriteTransaction() const { return std::make_unique(); } - MockTransaction::Ptr createReadTransaction() { return std::make_unique(); } + static MockTransaction::Ptr createWriteTransaction() { return std::make_unique(); } + static MockTransaction::Ptr createReadTransaction() { return std::make_unique(); } }; /** diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp index 5bab4133843..64b96118364 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp @@ -19,7 +19,7 @@ namespace bb::crypto::merkle_tree { template class CachedTreeStore { public: using LeafType = LeafValueType; - using IndexedLeafValueType = indexed_leaf; + using IndexedLeafValueType = IndexedLeaf; using ReadTransaction = typename PersistedStore::ReadTransaction; using WriteTransaction = typename PersistedStore::WriteTransaction; using ReadTransactionPtr = std::unique_ptr; @@ -101,14 +101,14 @@ std::pair CachedTreeStore::find_lo std::vector data; FrKeyType key(new_value.get_fr_value()); tx.get_value_or_previous(key, data); - index_t db_index = from_buffer(data, 0); + auto db_index = from_buffer(data, 0); uint256_t retrieved_value = key; if (!includeUncommitted || retrieved_value == new_value_as_number || indices_.empty()) { return std::make_pair(new_value_as_number == retrieved_value, db_index); } // At this stage, we have been asked to include uncommitted and the value was not exactly found in the db - std::map::const_iterator it = indices_.lower_bound(new_value_as_number); + auto it = indices_.lower_bound(new_value_as_number); if (it == indices_.end()) { // there is no element >= the requested value. // decrement the iterator to get the value preceeding the requested value From f0df0c6e1b0b42bb533ec67d9a379d0cc148daf1 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 27 Jun 2024 21:46:27 +0100 Subject: [PATCH 27/63] Refactoring --- .../merkle_tree/indexed_tree/indexed_tree.hpp | 50 ++- .../indexed_tree/indexed_tree.test.cpp | 8 +- .../merkle_tree/lmdb_store/functions.cpp | 40 ++ .../merkle_tree/lmdb_store/functions.hpp | 52 +++ .../merkle_tree/lmdb_store/lmdb_database.cpp | 36 ++ .../merkle_tree/lmdb_store/lmdb_database.hpp | 61 ++++ .../lmdb_store/lmdb_db_transaction.cpp | 12 + .../lmdb_store/lmdb_db_transaction.hpp | 20 + .../lmdb_store/lmdb_environment.cpp | 52 +++ .../lmdb_store/lmdb_environment.hpp | 31 ++ .../lmdb_store/lmdb_read_transaction.cpp | 40 ++ .../lmdb_store/lmdb_read_transaction.hpp | 120 ++++++ .../merkle_tree/lmdb_store/lmdb_store.cpp | 114 +----- .../merkle_tree/lmdb_store/lmdb_store.hpp | 345 +----------------- .../lmdb_store/lmdb_store.test.cpp | 12 + .../lmdb_store/lmdb_transaction.cpp | 23 ++ .../lmdb_store/lmdb_transaction.hpp | 31 ++ .../lmdb_store/lmdb_write_transaction.cpp | 52 +++ .../lmdb_store/lmdb_write_transaction.hpp | 44 +++ .../node_store/cached_tree_store.hpp | 5 +- .../service/world_state_service.hpp | 4 +- 21 files changed, 673 insertions(+), 479 deletions(-) create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 0c387ebac6f..a30c0ed836e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -73,8 +73,8 @@ template class IndexedTree : public App public: using LeafValueType = typename Store::LeafType; using IndexedLeafValueType = typename Store::IndexedLeafValueType; - using add_completion_callback = std::function&, fr&, index_t)>; - using leaf_callback = std::function; + using AddCompletionCallback = std::function&, fr&, index_t)>; + using LeafCallback = std::function; IndexedTree(Store& store, ThreadPool& workers, index_t initial_size); IndexedTree(IndexedTree const& other) = delete; @@ -88,7 +88,7 @@ template class IndexedTree : public App * @param value The value to be added or updated * @returns The 'previous' hash paths of all updated values */ - void add_or_update_value(const LeafValueType& value, const add_completion_callback& completion); + void add_or_update_value(const LeafValueType& value, const AddCompletionCallback& completion); /** * @brief Adds or updates the given set of values in the tree (updates not currently supported) @@ -96,14 +96,14 @@ template class IndexedTree : public App * @param no_multithreading Performs single threaded insertion, just used whilst prototyping and benchmarking * @returns The 'previous' hash paths of all updated values */ - void add_or_update_values(const std::vector& values, const add_completion_callback& completion); + void add_or_update_values(const std::vector& values, const AddCompletionCallback& completion); - void get_leaf(const index_t& index, bool includeUncommitted, const leaf_callback& completion); + void get_leaf(const index_t& index, bool includeUncommitted, const LeafCallback& completion); using AppendOnlyTree::get_hash_path; private: - using typename AppendOnlyTree::append_completion_callback; + using typename AppendOnlyTree::AppendCompletionCallback; using ReadTransaction = typename Store::ReadTransaction; using ReadTransactionPtr = typename Store::ReadTransactionPtr; @@ -119,18 +119,18 @@ template class IndexedTree : public App fr_hash_path& previous_hash_path, ReadTransaction& tx); - using insertion_generation_callback = std::function>, - std::shared_ptr>)>; + using InsertionGenerationCallback = std::function>, + std::shared_ptr>)>; void generate_insertions(const std::shared_ptr>>& values_to_be_sorted, - const insertion_generation_callback& completion); + const InsertionGenerationCallback& completion); - using insertion_completion_callback = std::function> paths)>; + using InsertionCompletionCallback = std::function> paths)>; void perform_insertions(std::shared_ptr> insertions, - const insertion_completion_callback& completion); + const InsertionCompletionCallback& completion); - using hash_generation_callback = std::function> hashes)>; + using HashGenerationCallback = std::function> hashes)>; void generate_hashes_for_appending(std::shared_ptr> leaves_to_hash, - const hash_generation_callback& completion); + const HashGenerationCallback& completion); using AppendOnlyTree::get_element_or_zero; using AppendOnlyTree::write_node; @@ -188,7 +188,7 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers } Signal signal(1); - append_completion_callback completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + AppendCompletionCallback completion = [&](fr, index_t) -> void { signal.signal_level(0); }; AppendOnlyTree::add_values(appended_hashes, completion); signal.wait_for_level(0); store_.commit(); @@ -197,7 +197,7 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers template void IndexedTree::get_leaf(const index_t& index, bool includeUncommitted, - const leaf_callback& completion) + const LeafCallback& completion) { auto job = [=]() { IndexedLeafValueType leaf; @@ -212,14 +212,14 @@ void IndexedTree::get_leaf(const index_t& index, template void IndexedTree::add_or_update_value(const LeafValueType& value, - const add_completion_callback& completion) + const AddCompletionCallback& completion) { add_or_update_values(std::vector{ value }, completion); } template void IndexedTree::add_or_update_values(const std::vector& values, - const add_completion_callback& completion) + const AddCompletionCallback& completion) { std::shared_ptr>> values_to_be_sorted = std::make_shared>>(values.size()); @@ -237,25 +237,23 @@ void IndexedTree::add_or_update_values(const std::vector results = std::make_shared(); - append_completion_callback final_completion = [=](fr root, index_t size) { - completion(*results->paths, root, size); - }; + AppendCompletionCallback final_completion = [=](fr root, index_t size) { completion(*results->paths, root, size); }; - hash_generation_callback hash_completion = [=](const std::shared_ptr>& hashes_to_append) { + HashGenerationCallback hash_completion = [=](const std::shared_ptr>& hashes_to_append) { results->hashes_to_append = hashes_to_append; if (results->count.fetch_sub(1) == 1) { AppendOnlyTree::add_values(*hashes_to_append, final_completion); } }; - insertion_completion_callback insertion_completion = [=](const std::shared_ptr>& paths) { + InsertionCompletionCallback insertion_completion = [=](const std::shared_ptr>& paths) { results->paths = paths; if (results->count.fetch_sub(1) == 1) { AppendOnlyTree::add_values(*results->hashes_to_append, final_completion); } }; - insertion_generation_callback insertion_generation_completed = + InsertionGenerationCallback insertion_generation_completed = [=](std::shared_ptr> insertions, std::shared_ptr> leaves_to_append) { workers_.enqueue([=]() { generate_hashes_for_appending(leaves_to_append, hash_completion); }); @@ -266,7 +264,7 @@ void IndexedTree::add_or_update_values(const std::vector void IndexedTree::perform_insertions(std::shared_ptr> insertions, - const insertion_completion_callback& completion) + const InsertionCompletionCallback& completion) { // We now kick off multiple workers to perform the low leaf updates // We create set of signals to coordinate the workers as the move up the tree @@ -296,7 +294,7 @@ void IndexedTree::perform_insertions(std::shared_ptr void IndexedTree::generate_hashes_for_appending( - std::shared_ptr> leaves_to_hash, const hash_generation_callback& completion) + std::shared_ptr> leaves_to_hash, const HashGenerationCallback& completion) { std::shared_ptr> hashed = std::make_shared>(leaves_to_hash->size()); std::vector& leaves = *leaves_to_hash; @@ -309,7 +307,7 @@ void IndexedTree::generate_hashes_for_appending( template void IndexedTree::generate_insertions( const std::shared_ptr>>& values_to_be_sorted, - const insertion_generation_callback& on_completion) + const InsertionGenerationCallback& on_completion) { // The first thing we do is sort the values into descending order but maintain knowledge of their orignal order struct { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index 81d1057e396..390fbdf1d29 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -125,7 +125,7 @@ void add_value(TreeType& tree, const fr& value) signal.wait_for_level(0); } -void add_values(TreeType& tree, const std::vector& values) +void add_values(TreeType& tree, const std::vector& values) { Signal signal(1); auto completion = [&](const std::vector&, fr&, index_t) -> void { signal.signal_level(0); }; @@ -302,7 +302,7 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) check_hash_path(tree2, 512, memdb.get_hash_path(512)); for (uint32_t i = 0; i < num_batches; i++) { - std::vector batch; + std::vector batch; std::vector memory_tree_hash_paths; for (uint32_t j = 0; j < batch_size; j++) { batch.emplace_back(random_engine.get_random_uint256()); @@ -367,7 +367,7 @@ bool verify_hash_path(TreeType& tree, const IndexedLeafType& leaf_value, const u IndexedLeafType create_indexed_nullifier_leaf(const fr& value, index_t nextIndex, const fr& nextValue) { - return IndexedLeafType(nullifier_leaf_value(value), nextIndex, nextValue); + return IndexedLeafType(NullifierLeafValue(value), nextIndex, nextValue); } TEST_F(PersistedIndexedTreeTest, test_indexed_memory) @@ -389,7 +389,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) * nextIdx 0 0 0 0 0 0 0 0 * nextVal 0 0 0 0 0 0 0 0 */ - IndexedLeafType zero_leaf(nullifier_leaf_value(0), 0, 0); + IndexedLeafType zero_leaf(NullifierLeafValue(0), 0, 0); check_size(tree, 1); EXPECT_EQ(get_leaf(tree, 0), zero_leaf); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp new file mode 100644 index 00000000000..8e11f658db7 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp @@ -0,0 +1,40 @@ +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include + +namespace bb::crypto::merkle_tree { +void ThrowError(const std::string& errorString, int error) +{ + std::stringstream ss; + ss << errorString << ": " << error << " - " << mdb_strerror(error) << std::endl; + throw std::runtime_error(ss.str()); +} + +int SizeCmp(const MDB_val* a, const MDB_val* b) +{ + if (a->mv_size < b->mv_size) { + return -1; + } + if (a->mv_size > b->mv_size) { + return 1; + } + return 0; +} + +int Invalid(const MDB_val*, const MDB_val*) +{ + throw std::runtime_error("Invalid comparison"); +} + +int IntegerKeyCmp(const MDB_val* a, const MDB_val* b) +{ + if (a->mv_size != b->mv_size) { + return SizeCmp(a, b); + } + + using f_type = std::function; + static std::vector functions{ + ValueCmp, ValueCmp, ValueCmp, Invalid, ValueCmp + }; + return functions[a->mv_size / 8](a, b); +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp new file mode 100644 index 00000000000..3f59104f4e5 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include "barretenberg/numeric/uint128/uint128.hpp" +#include "barretenberg/numeric/uint256/uint256.hpp" +#include + +namespace bb::crypto::merkle_tree { +using NodeKeyType = uint128_t; +using LeafIndexKeyType = uint64_t; +using FrKeyType = uint256_t; +using MetaKeyType = uint8_t; + +void ThrowError(const std::string& errorString, int error); + +int SizeCmp(const MDB_val* a, const MDB_val* b); + +int Invalid(const MDB_val*, const MDB_val*); + +template int ValueCmp(const MDB_val* a, const MDB_val* b) +{ + const T* lhs = static_cast(a->mv_data); + const T* rhs = static_cast(b->mv_data); + if (*lhs < *rhs) { + return -1; + } + if (*lhs > *rhs) { + return 1; + } + return 0; +} + +int IntegerKeyCmp(const MDB_val* a, const MDB_val* b); + +template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) +{ + int error = f(args...); + return error == 0; +} + +template void call_lmdb_func(const std::string& errorString, int (*f)(TArgs...), TArgs... args) +{ + int error = f(args...); + if (error != 0) { + ThrowError(errorString, error); + } +} + +template void call_lmdb_func(void (*f)(TArgs...), TArgs... args) +{ + f(args...); +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.cpp new file mode 100644 index 00000000000..b565b92a828 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.cpp @@ -0,0 +1,36 @@ +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" + +namespace bb::crypto::merkle_tree { +LMDBDatabase::LMDBDatabase(const LMDBEnvironment& env, + const LMDBDatabaseCreationTransaction& transaction, + const std::string& name, + bool integerKeys, + bool reverseKeys, + MDB_cmp_func* cmp) + : _environment(env) +{ + unsigned int flags = MDB_CREATE; + if (integerKeys) { + flags |= MDB_INTEGERKEY; + } + if (reverseKeys) { + flags |= MDB_REVERSEKEY; + } + call_lmdb_func("mdb_dbi_open", mdb_dbi_open, transaction.underlying(), name.c_str(), flags, &_dbi); + if (cmp != nullptr) { + call_lmdb_func("mdb_set_compare", mdb_set_compare, transaction.underlying(), _dbi, cmp); + } + transaction.commit(); +} + +LMDBDatabase::~LMDBDatabase() +{ + call_lmdb_func(mdb_dbi_close, _environment.underlying(), _dbi); +} + +const MDB_dbi& LMDBDatabase::underlying() const +{ + return _dbi; +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp new file mode 100644 index 00000000000..732476183ec --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp @@ -0,0 +1,61 @@ +#pragma once +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" + +namespace bb::crypto::merkle_tree { +class LMDBDatabase { + public: + LMDBDatabase(const LMDBEnvironment& env, + const LMDBDatabaseCreationTransaction& transaction, + const std::string& name, + bool integerKeys = false, + bool reverseKeys = false, + MDB_cmp_func* cmp = nullptr); + + LMDBDatabase(const LMDBDatabase& other) = delete; + LMDBDatabase(LMDBDatabase&& other) = delete; + LMDBDatabase& operator=(const LMDBDatabase& other) = delete; + LMDBDatabase& operator=(LMDBDatabase&& other) = delete; + + ~LMDBDatabase(); + + const MDB_dbi& underlying() const; + + private: + MDB_dbi _dbi; + const LMDBEnvironment& _environment; +}; + +// LMDBDatabase::LMDBDatabase(const LMDBEnvironment& env, +// const LMDBDatabaseCreationTransaction& transaction, +// const std::string& name, +// bool integerKeys, +// bool reverseKeys, +// MDB_cmp_func* cmp) +// : _environment(env) +// { +// unsigned int flags = MDB_CREATE; +// if (integerKeys) { +// flags |= MDB_INTEGERKEY; +// } +// if (reverseKeys) { +// flags |= MDB_REVERSEKEY; +// } +// call_lmdb_func("mdb_dbi_open", mdb_dbi_open, transaction.underlying(), name.c_str(), flags, &_dbi); +// if (cmp != nullptr) { +// call_lmdb_func("mdb_set_compare", mdb_set_compare, transaction.underlying(), _dbi, cmp); +// } +// transaction.commit(); +// } + +// LMDBDatabase::~LMDBDatabase() +// { +// call_lmdb_func(mdb_dbi_close, _environment.underlying(), _dbi); +// } + +// const MDB_dbi& LMDBDatabase::underlying() const +// { +// return _dbi; +// } +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.cpp new file mode 100644 index 00000000000..5b4204193a3 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.cpp @@ -0,0 +1,12 @@ +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" + +namespace bb::crypto::merkle_tree { +LMDBDatabaseCreationTransaction::LMDBDatabaseCreationTransaction(LMDBEnvironment& env) + : LMDBTransaction(env) +{} +void LMDBDatabaseCreationTransaction::commit() const +{ + call_lmdb_func("mdb_txn_commit", mdb_txn_commit, _transaction); +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp new file mode 100644 index 00000000000..48645059592 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp @@ -0,0 +1,20 @@ +#pragma once +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" + +namespace bb::crypto::merkle_tree { + +class LMDBDatabaseCreationTransaction : public LMDBTransaction { + public: + using Ptr = std::unique_ptr; + + LMDBDatabaseCreationTransaction(LMDBEnvironment& env); + LMDBDatabaseCreationTransaction(const LMDBDatabaseCreationTransaction& other) = delete; + LMDBDatabaseCreationTransaction(LMDBDatabaseCreationTransaction&& other) = delete; + LMDBDatabaseCreationTransaction& operator=(const LMDBDatabaseCreationTransaction& other) = delete; + LMDBDatabaseCreationTransaction& operator=(LMDBDatabaseCreationTransaction&& other) = delete; + + ~LMDBDatabaseCreationTransaction() override = default; + void commit() const; +}; + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp new file mode 100644 index 00000000000..61bc88c5216 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp @@ -0,0 +1,52 @@ +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include + +namespace bb::crypto::merkle_tree { +LMDBEnvironment::LMDBEnvironment(const std::string& directory, + uint64_t mapSizeMB, + uint32_t maxNumDBs, + uint32_t maxNumReaders) + : _maxReaders(maxNumReaders) + , _numReaders(0) +{ + call_lmdb_func("mdb_env_create", mdb_env_create, &_mdbEnv); + uint64_t kb = 1024; + uint64_t totalMapSize = kb * kb * mapSizeMB; + call_lmdb_func(mdb_env_set_mapsize, _mdbEnv, totalMapSize); + call_lmdb_func(mdb_env_set_maxdbs, _mdbEnv, static_cast(maxNumDBs)); + call_lmdb_func(mdb_env_set_maxreaders, _mdbEnv, maxNumReaders); + uint32_t flags = MDB_NOTLS; + if (!call_lmdb_func( + mdb_env_open, _mdbEnv, directory.c_str(), flags, static_cast(S_IRWXU | S_IRWXG | S_IRWXO))) { + call_lmdb_func(mdb_env_close, _mdbEnv); + // throw here + } +} + +void LMDBEnvironment::waitForReader() +{ + std::unique_lock lock(_readersLock); + if (_numReaders >= _maxReaders) { + _readersCondition.wait(lock, [&] { return _numReaders < _maxReaders; }); + } + ++_numReaders; +} + +void LMDBEnvironment::releaseReader() +{ + std::unique_lock lock(_readersLock); + --_numReaders; + _readersCondition.notify_one(); +} + +LMDBEnvironment::~LMDBEnvironment() +{ + call_lmdb_func(mdb_env_close, _mdbEnv); +} + +MDB_env* LMDBEnvironment::underlying() const +{ + return _mdbEnv; +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp new file mode 100644 index 00000000000..48eda657a2a --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include +namespace bb::crypto::merkle_tree { +class LMDBEnvironment { + public: + LMDBEnvironment(const std::string& directory, uint64_t mapSizeMB, uint32_t maxNumDBs, uint32_t maxNumReaders); + LMDBEnvironment(const LMDBEnvironment& other) = delete; + LMDBEnvironment(LMDBEnvironment&& other) = delete; + LMDBEnvironment& operator=(const LMDBEnvironment& other) = delete; + LMDBEnvironment& operator=(LMDBEnvironment&& other) = delete; + + ~LMDBEnvironment(); + + MDB_env* underlying() const; + + void waitForReader(); + + void releaseReader(); + + private: + MDB_env* _mdbEnv; + uint32_t _maxReaders; + uint32_t _numReaders; + std::mutex _readersLock; + std::condition_variable _readersCondition; +}; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp new file mode 100644 index 00000000000..c8bcab01d31 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp @@ -0,0 +1,40 @@ +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp" + +namespace bb::crypto::merkle_tree { +LMDBReadTransaction::LMDBReadTransaction(LMDBEnvironment& env, const LMDBDatabase& database) + : LMDBTransaction(env, true) + , _database(database) +{} + +LMDBReadTransaction::~LMDBReadTransaction() +{ + abort(); +} + +void LMDBReadTransaction::abort() +{ + LMDBTransaction::abort(); + _environment.releaseReader(); +} + +bool LMDBReadTransaction::get_value(std::vector& key, std::vector& data) const +{ + MDB_val dbKey; + dbKey.mv_size = key.size(); + dbKey.mv_data = (void*)key.data(); + + MDB_val dbVal; + if (!call_lmdb_func(mdb_get, underlying(), _database.underlying(), &dbKey, &dbVal)) { + return false; + } + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + return true; +} + +bool LMDBReadTransaction::get_node(uint32_t level, index_t index, std::vector& data) const +{ + NodeKeyType key = (static_cast(1 << level) + static_cast(index)) - 1; + return get_value_by_integer(key, data); +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp new file mode 100644 index 00000000000..ec68f48fe06 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp @@ -0,0 +1,120 @@ +#pragma once +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include +#include + +namespace bb::crypto::merkle_tree { + +class LMDBReadTransaction : public LMDBTransaction { + public: + using Ptr = std::unique_ptr; + + LMDBReadTransaction(LMDBEnvironment& env, const LMDBDatabase& database); + LMDBReadTransaction(const LMDBReadTransaction& other) = delete; + LMDBReadTransaction(LMDBReadTransaction&& other) = delete; + LMDBReadTransaction& operator=(const LMDBReadTransaction& other) = delete; + LMDBReadTransaction& operator=(LMDBReadTransaction&& other) = delete; + + ~LMDBReadTransaction() override; + + template bool get_value_or_previous(T& key, std::vector& data) const; + + bool get_node(uint32_t level, index_t index, std::vector& data) const; + + template bool get_value_by_integer(T& key, std::vector& data) const; + + bool get_value(std::vector& key, std::vector& data) const; + + void abort() override; + + protected: + const LMDBDatabase& _database; +}; + +template bool LMDBReadTransaction::get_value_by_integer(T& key, std::vector& data) const +{ + MDB_val dbKey; + dbKey.mv_size = sizeof(T); + dbKey.mv_data = (void*)&key; + + MDB_val dbVal; + if (!call_lmdb_func(mdb_get, underlying(), _database.underlying(), &dbKey, &dbVal)) { + return false; + } + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + return true; +} + +template bool LMDBReadTransaction::get_value_or_previous(T& key, std::vector& data) const +{ + T keyCopy = key; + MDB_cursor* cursor = nullptr; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, underlying(), _database.underlying(), &cursor); + + MDB_val dbKey; + dbKey.mv_size = sizeof(T); + dbKey.mv_data = (void*)&keyCopy; + + MDB_val dbVal; + + bool success = false; + + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + if (code == 0) { + // we found the key, now determine if it is the exact key + if (dbKey.mv_size == sizeof(T) && std::memcmp(dbKey.mv_data, &key, dbKey.mv_size) == 0) { + // we have the exact key + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + success = true; + } else { + // We have a key of the same size but larger value OR a larger size + // either way we now need to find the previous key + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); + if (code == 0) { + // We have found a previous key. It could be of the same size but smaller value, or smaller size which + // is equal to not found + if (dbKey.mv_size != sizeof(T)) { + // There is no previous key, do nothing + } else { + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + std::memcpy(&key, dbKey.mv_data, dbKey.mv_size); + success = true; + } + } else if (code == MDB_NOTFOUND) { + // There is no previous key, do nothing + } else { + ThrowError("get_value_or_previous::mdb_cursor_get", code); + } + } + } else if (code == MDB_NOTFOUND) { + // The key was not found, use the last key in the db + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); + if (code == 0) { + // We found the last key, but we need to ensure it is the same size + if (dbKey.mv_size != sizeof(T)) { + // The key is not the same size, same as not found, do nothing + } else { + data.resize(dbVal.mv_size); + std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + std::memcpy(&key, dbKey.mv_data, dbKey.mv_size); + success = true; + } + } else if (code == MDB_NOTFOUND) { + // DB is empty? + } else { + ThrowError("get_value_or_previous::mdb_cursor_get", code); + } + } else { + ThrowError("get_value_or_previous::mdb_cursor_get", code); + } + call_lmdb_func(mdb_cursor_close, cursor); + return success; +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp index fb9f5527b19..91bd5fcb727 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp @@ -9,118 +9,20 @@ #include namespace bb::crypto::merkle_tree { +LMDBStore::LMDBStore( + LMDBEnvironment& environment, std::string name, bool integerKeys, bool reverseKeys, MDB_cmp_func* cmp) + : _environment(environment) + , _name(std::move(name)) + , _database(_environment, LMDBDatabaseCreationTransaction(_environment), _name, integerKeys, reverseKeys, cmp) +{} -int SizeCmp(const MDB_val* a, const MDB_val* b) +LMDBWriteTransaction::Ptr LMDBStore::createWriteTransaction() const { - if (a->mv_size < b->mv_size) { - return -1; - } - if (a->mv_size > b->mv_size) { - return 1; - } - return 0; + return std::make_unique(_environment, _database); } - -int Invalid(const MDB_val*, const MDB_val*) -{ - throw std::runtime_error("Invalid comparison"); -} - -int IntegerKeyCmp(const MDB_val* a, const MDB_val* b) -{ - if (a->mv_size != b->mv_size) { - return SizeCmp(a, b); - } - - using f_type = std::function; - static std::vector functions{ - ValueCmp, ValueCmp, ValueCmp, Invalid, ValueCmp - }; - return functions[a->mv_size / 8](a, b); -} - -LMDBEnvironment::LMDBEnvironment(const std::string& directory, - uint64_t mapSizeMB, - uint32_t maxNumDBs, - uint32_t maxNumReaders) - : _maxReaders(maxNumReaders) - , _numReaders(0) -{ - if (!call_lmdb_func(mdb_env_create, &_mdbEnv)) { - // throw here - } - uint64_t kb = 1024; - uint64_t totalMapSize = kb * kb * mapSizeMB; - call_lmdb_func(mdb_env_set_mapsize, _mdbEnv, totalMapSize); - call_lmdb_func(mdb_env_set_maxdbs, _mdbEnv, static_cast(maxNumDBs)); - call_lmdb_func(mdb_env_set_maxreaders, _mdbEnv, maxNumReaders); - uint32_t flags = MDB_NOTLS; - if (!call_lmdb_func( - mdb_env_open, _mdbEnv, directory.c_str(), flags, static_cast(S_IRWXU | S_IRWXG | S_IRWXO))) { - call_lmdb_func(mdb_env_close, _mdbEnv); - // throw here - } -} - -void LMDBEnvironment::waitForReader() -{ - std::unique_lock lock(_readersLock); - if (_numReaders >= _maxReaders) { - _readersCondition.wait(lock, [&] { return _numReaders < _maxReaders; }); - } - ++_numReaders; -} - -void LMDBEnvironment::releaseReader() -{ - std::unique_lock lock(_readersLock); - --_numReaders; - _readersCondition.notify_one(); -} - -void LMDBWriteTransaction::put_node(uint32_t level, index_t index, std::vector& data) -{ - NodeKeyType key = (static_cast(1 << level) + static_cast(index)) - 1; - put_value_by_integer(key, data); -} - -void LMDBWriteTransaction::put_value(std::vector& key, std::vector& data) -{ - MDB_val dbKey; - dbKey.mv_size = key.size(); - dbKey.mv_data = (void*)key.data(); - - MDB_val dbVal; - dbVal.mv_size = data.size(); - dbVal.mv_data = (void*)data.data(); - call_lmdb_func(mdb_put, underlying(), _database.underlying(), &dbKey, &dbVal, 0U); -} - -bool LMDBReadTransaction::get_value(std::vector& key, std::vector& data) const -{ - MDB_val dbKey; - dbKey.mv_size = key.size(); - dbKey.mv_data = (void*)key.data(); - - MDB_val dbVal; - if (!call_lmdb_func(mdb_get, underlying(), _database.underlying(), &dbKey, &dbVal)) { - return false; - } - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); - return true; -} - -bool LMDBReadTransaction::get_node(uint32_t level, index_t index, std::vector& data) const -{ - NodeKeyType key = (static_cast(1 << level) + static_cast(index)) - 1; - return get_value_by_integer(key, data); -} - LMDBReadTransaction::Ptr LMDBStore::createReadTransaction() { _environment.waitForReader(); return std::make_unique(_environment, _database); } - } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp index ff72889eba1..e5f3490cf92 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp @@ -1,338 +1,10 @@ #pragma once -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/numeric/uint256/uint256.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp" namespace bb::crypto::merkle_tree { -using NodeKeyType = uint128_t; -using LeafIndexKeyType = uint64_t; -using FrKeyType = uint256_t; -using MetaKeyType = uint8_t; - -int SizeCmp(const MDB_val* a, const MDB_val* b); - -int Invalid(const MDB_val*, const MDB_val*); - -template int ValueCmp(const MDB_val* a, const MDB_val* b) -{ - const T* lhs = static_cast(a->mv_data); - const T* rhs = static_cast(b->mv_data); - if (*lhs < *rhs) { - return -1; - } - if (*lhs > *rhs) { - return 1; - } - return 0; -} - -int IntegerKeyCmp(const MDB_val* a, const MDB_val* b); - -template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) -{ - int error = f(args...); - if (error != 0 && error != MDB_NOTFOUND) { - std::stringstream ss; - ss << "ERROR: " << mdb_strerror(error) << std::endl; - std::cerr << ss.str(); - } - return error == 0; -} - -template void call_lmdb_func(void (*f)(TArgs...), TArgs... args) -{ - f(args...); -} - -class LMDBEnvironment { - public: - LMDBEnvironment(const std::string& directory, uint64_t mapSizeMB, uint32_t maxNumDBs, uint32_t maxNumReaders); - LMDBEnvironment(const LMDBEnvironment& other) = delete; - LMDBEnvironment(LMDBEnvironment&& other) = delete; - LMDBEnvironment& operator=(const LMDBEnvironment& other) = delete; - LMDBEnvironment& operator=(LMDBEnvironment&& other) = delete; - - ~LMDBEnvironment() { call_lmdb_func(mdb_env_close, _mdbEnv); } - - MDB_env* underlying() const { return _mdbEnv; } - - void waitForReader(); - - void releaseReader(); - - private: - MDB_env* _mdbEnv; - uint32_t _maxReaders; - uint32_t _numReaders; - std::mutex _readersLock; - std::condition_variable _readersCondition; -}; - -class LMDBDatabase; - -class LMDBTransaction { - public: - LMDBTransaction(LMDBEnvironment& env, bool readOnly = false) - : _environment(env) - { - MDB_txn* p = nullptr; - if (!call_lmdb_func(mdb_txn_begin, _environment.underlying(), p, readOnly ? MDB_RDONLY : 0U, &_transaction)) { - } - } - LMDBTransaction(const LMDBTransaction& other) = delete; - LMDBTransaction(LMDBTransaction&& other) = delete; - LMDBTransaction& operator=(const LMDBTransaction& other) = delete; - LMDBTransaction& operator=(LMDBTransaction&& other) = delete; - - virtual ~LMDBTransaction() = default; - - MDB_txn* underlying() const { return _transaction; } - - protected: - LMDBEnvironment& _environment; - MDB_txn* _transaction; -}; - -class LMDBDatabase { - public: - LMDBDatabase(const LMDBEnvironment& env, - const LMDBTransaction& transaction, - const std::string& name, - bool integerKeys = false, - bool reverseKeys = false, - MDB_cmp_func* cmp = nullptr) - : _environment(env) - { - unsigned int flags = MDB_CREATE; - if (integerKeys) { - flags |= MDB_INTEGERKEY; - } - if (reverseKeys) { - flags |= MDB_REVERSEKEY; - } - if (!call_lmdb_func(mdb_dbi_open, transaction.underlying(), name.c_str(), flags, &_dbi)) { - // throw here - } - if (cmp != nullptr) { - if (!call_lmdb_func(mdb_set_compare, transaction.underlying(), _dbi, cmp)) { - // throw here - } - } - } - - LMDBDatabase(const LMDBDatabase& other) = delete; - LMDBDatabase(LMDBDatabase&& other) = delete; - LMDBDatabase& operator=(const LMDBDatabase& other) = delete; - LMDBDatabase& operator=(LMDBDatabase&& other) = delete; - - ~LMDBDatabase() { call_lmdb_func(mdb_dbi_close, _environment.underlying(), _dbi); } - - const MDB_dbi& underlying() const { return _dbi; } - - private: - MDB_dbi _dbi; - const LMDBEnvironment& _environment; -}; - -class LMDBReadTransaction : public LMDBTransaction { - public: - using Ptr = std::unique_ptr; - - LMDBReadTransaction(LMDBEnvironment& env, const LMDBDatabase& database) - : LMDBTransaction(env, true) - , _database(database) - {} - LMDBReadTransaction(const LMDBReadTransaction& other) = delete; - LMDBReadTransaction(LMDBReadTransaction&& other) = delete; - LMDBReadTransaction& operator=(const LMDBReadTransaction& other) = delete; - LMDBReadTransaction& operator=(LMDBReadTransaction&& other) = delete; - - ~LMDBReadTransaction() override { abort(); } - - template bool get_value_or_previous(T& key, std::vector& data) const; - - bool get_node(uint32_t level, index_t index, std::vector& data) const; - - template bool get_value_by_integer(T& key, std::vector& data) const; - - bool get_value(std::vector& key, std::vector& data) const; - - protected: - const LMDBDatabase& _database; - - void abort() - { - call_lmdb_func(mdb_txn_abort, _transaction); - _environment.releaseReader(); - } -}; - -template bool LMDBReadTransaction::get_value_by_integer(T& key, std::vector& data) const -{ - MDB_val dbKey; - dbKey.mv_size = sizeof(T); - dbKey.mv_data = (void*)&key; - - MDB_val dbVal; - if (!call_lmdb_func(mdb_get, underlying(), _database.underlying(), &dbKey, &dbVal)) { - return false; - } - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); - return true; -} - -template bool LMDBReadTransaction::get_value_or_previous(T& key, std::vector& data) const -{ - T keyCopy = key; - MDB_cursor* cursor = nullptr; - call_lmdb_func(mdb_cursor_open, underlying(), _database.underlying(), &cursor); - - MDB_val dbKey; - dbKey.mv_size = sizeof(T); - dbKey.mv_data = (void*)&keyCopy; - - MDB_val dbVal; - - bool success = false; - - // Look for the key >= to that provided - int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); - if (code == 0) { - // we found the key, now determine if it is the exact key - if (dbKey.mv_size == sizeof(T) && std::memcmp(dbKey.mv_data, &key, dbKey.mv_size) == 0) { - // we have the exact key - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); - success = true; - } else { - // We have a key of the same size but larger value OR a larger size - // either way we now need to find the previous key - code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); - if (code == 0) { - // We have found a previous key. It could be of the same size but smaller value, or smaller size which - // is equal to not found - if (dbKey.mv_size != sizeof(T)) { - // There is no previous key, do nothing - } else { - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); - std::memcpy(&key, dbKey.mv_data, dbKey.mv_size); - success = true; - } - } else if (code == MDB_NOTFOUND) { - // There is no previous key, do nothing - } else { - std::stringstream ss; - ss << "ERROR: " << mdb_strerror(code) << std::endl; - std::cout << ss.str(); - } - } - } else if (code == MDB_NOTFOUND) { - // The key was not found, use the last key in the db - code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); - if (code == 0) { - // We found the last key, but we need to ensure it is the same size - if (dbKey.mv_size != sizeof(T)) { - // The key is not the same size, same as not found, do nothing - } else { - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); - std::memcpy(&key, dbKey.mv_data, dbKey.mv_size); - success = true; - } - } else if (code == MDB_NOTFOUND) { - // DB is empty? - } else { - std::stringstream ss; - ss << "ERROR: " << mdb_strerror(code) << std::endl; - std::cout << ss.str(); - } - } else { - std::stringstream ss; - ss << "ERROR: " << mdb_strerror(code) << std::endl; - std::cout << ss.str(); - } - call_lmdb_func(mdb_cursor_close, cursor); - return success; -} - -class LMDBWriteTransaction : public LMDBTransaction { - public: - using Ptr = std::unique_ptr; - - LMDBWriteTransaction(LMDBEnvironment& env, const LMDBDatabase& database) - : LMDBTransaction(env) - , _database(database) - {} - LMDBWriteTransaction(const LMDBWriteTransaction& other) = delete; - LMDBWriteTransaction(LMDBWriteTransaction&& other) = delete; - LMDBWriteTransaction& operator=(const LMDBWriteTransaction& other) = delete; - LMDBWriteTransaction& operator=(LMDBWriteTransaction&& other) = delete; - ~LMDBWriteTransaction() override { commit(); } - - void put_node(uint32_t level, index_t index, std::vector& data); - - template void put_value_by_integer(T& key, std::vector& data); - - void put_value(std::vector& key, std::vector& data); - - bool commit() - { - if (_committed) { - return true; - } - _committed = call_lmdb_func(mdb_txn_commit, _transaction); - return _committed; - } - - protected: - const LMDBDatabase& _database; - bool _committed{}; -}; - -template void LMDBWriteTransaction::put_value_by_integer(T& key, std::vector& data) -{ - MDB_val dbKey; - dbKey.mv_size = sizeof(T); - dbKey.mv_data = &key; - - MDB_val dbVal; - dbVal.mv_size = data.size(); - dbVal.mv_data = (void*)data.data(); - call_lmdb_func(mdb_put, underlying(), _database.underlying(), &dbKey, &dbVal, 0U); -} - -class LMDBDatabaseCreationTransaction : public LMDBTransaction { - public: - using Ptr = std::unique_ptr; - - LMDBDatabaseCreationTransaction(LMDBEnvironment& env) - : LMDBTransaction(env) - {} - LMDBDatabaseCreationTransaction(const LMDBDatabaseCreationTransaction& other) = delete; - LMDBDatabaseCreationTransaction(LMDBDatabaseCreationTransaction&& other) = delete; - LMDBDatabaseCreationTransaction& operator=(const LMDBDatabaseCreationTransaction& other) = delete; - LMDBDatabaseCreationTransaction& operator=(LMDBDatabaseCreationTransaction&& other) = delete; - - ~LMDBDatabaseCreationTransaction() override { commit(); } - - private: - bool commit() { return call_lmdb_func(mdb_txn_commit, _transaction); } -}; - class LMDBStore { public: @@ -342,21 +14,14 @@ class LMDBStore { std::string name, bool integerKeys = false, bool reverseKeys = false, - MDB_cmp_func* cmp = nullptr) - : _environment(environment) - , _name(std::move(name)) - , _database(_environment, LMDBDatabaseCreationTransaction(_environment), _name, integerKeys, reverseKeys, cmp) - {} + MDB_cmp_func* cmp = nullptr); LMDBStore(const LMDBStore& other) = delete; LMDBStore(LMDBStore&& other) = delete; LMDBStore& operator=(const LMDBStore& other) = delete; LMDBStore& operator=(LMDBStore&& other) = delete; ~LMDBStore() = default; - LMDBWriteTransaction::Ptr createWriteTransaction() const - { - return std::make_unique(_environment, _database); - } + LMDBWriteTransaction::Ptr createWriteTransaction() const; LMDBReadTransaction::Ptr createReadTransaction(); private: diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp index 7747d7721dc..a36709bb005 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp @@ -54,6 +54,7 @@ TEST_F(LMDBStoreTest, can_write_to_and_read_from_store) write(buf, VALUES[0]); LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); transaction->put_node(0, 0, buf); + transaction->commit(); } { @@ -75,6 +76,7 @@ TEST_F(LMDBStoreTest, can_write_to_and_read_from_store) write(value, VALUES[1]); LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); transaction->put_value(key, value); + transaction->commit(); } { @@ -100,6 +102,7 @@ TEST_F(LMDBStoreTest, reading_an_empty_key_reports_correctly) write(buf, VALUES[0]); LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); transaction->put_node(0, 0, buf); + transaction->commit(); } { @@ -120,6 +123,7 @@ TEST_F(LMDBStoreTest, reading_an_empty_key_reports_correctly) write(value, VALUES[1]); LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); transaction->put_value(key, value); + transaction->commit(); } { LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); @@ -145,6 +149,7 @@ TEST_F(LMDBStoreTest, can_write_and_read_multiple) write(buf, VALUES[i]); transaction->put_node(10, i, buf); } + transaction->commit(); } { @@ -172,6 +177,7 @@ TEST_F(LMDBStoreTest, can_write_and_read_multiple) std::vector buf; write(buf, VALUES[i + 128]); transaction->put_value(key, buf); + transaction->commit(); } } @@ -233,6 +239,7 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) transaction->put_value_by_integer(higher64, value); transaction->put_value_by_integer(lower256, value); transaction->put_value_by_integer(higher256, value); + transaction->commit(); } { @@ -315,6 +322,7 @@ TEST_F(LMDBStoreTest, can_write_and_read_at_random_keys) keys.push_back(key); transaction->put_node(0, key, buf); } + transaction->commit(); } { @@ -344,6 +352,7 @@ TEST_F(LMDBStoreTest, can_recreate_the_store_and_use_again) keys.push_back(key); transaction->put_node(0, key, buf); } + transaction->commit(); } { @@ -394,6 +403,7 @@ TEST_F(LMDBStoreTest, can_read_from_multiple_threads) std::vector buf; write(buf, VALUES[0]); transaction->put_node(0, key, buf); + transaction->commit(); } { @@ -413,6 +423,7 @@ TEST_F(LMDBStoreTest, can_read_from_multiple_threads) std::vector buf; write(buf, VALUES[0] + 1); transaction->put_node(0, key, buf); + transaction->commit(); } // now wait for all threads to exit having seen the new value for (size_t i = 0; i < num_threads; i++) { @@ -462,6 +473,7 @@ TEST_F(LMDBStoreTest, can_handle_different_key_spaces) write_values.template operator()(*transaction, 1); write_values.template operator()(*transaction, 2); write_values.template operator()(*transaction, 3); + transaction->commit(); } { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp new file mode 100644 index 00000000000..c12385a7d35 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp @@ -0,0 +1,23 @@ +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" + +namespace bb::crypto::merkle_tree { +LMDBTransaction::LMDBTransaction(LMDBEnvironment& env, bool readOnly) + : _environment(env) + , state(TransactionState::OPEN) +{ + MDB_txn* p = nullptr; + call_lmdb_func( + "mdb_txn_begin", mdb_txn_begin, _environment.underlying(), p, readOnly ? MDB_RDONLY : 0U, &_transaction); +} + +MDB_txn* LMDBTransaction::underlying() const +{ + return _transaction; +} + +void LMDBTransaction::abort() +{ + call_lmdb_func(mdb_txn_abort, _transaction); +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp new file mode 100644 index 00000000000..b8b0ff2bb19 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp @@ -0,0 +1,31 @@ +#pragma once +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" + +namespace bb::crypto::merkle_tree { + +enum TransactionState { + OPEN, + COMMITTED, + ABORTED, +}; + +class LMDBTransaction { + public: + LMDBTransaction(LMDBEnvironment& env, bool readOnly = false); + LMDBTransaction(const LMDBTransaction& other) = delete; + LMDBTransaction(LMDBTransaction&& other) = delete; + LMDBTransaction& operator=(const LMDBTransaction& other) = delete; + LMDBTransaction& operator=(LMDBTransaction&& other) = delete; + + virtual ~LMDBTransaction() = default; + + MDB_txn* underlying() const; + + virtual void abort(); + + protected: + LMDBEnvironment& _environment; + MDB_txn* _transaction; + TransactionState state; +}; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp new file mode 100644 index 00000000000..4c7cd76f6cf --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp @@ -0,0 +1,52 @@ + + +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp" + +namespace bb::crypto::merkle_tree { + +LMDBWriteTransaction::LMDBWriteTransaction(LMDBEnvironment& env, const LMDBDatabase& database) + : LMDBTransaction(env) + , _database(database) +{} + +LMDBWriteTransaction::~LMDBWriteTransaction() +{ + tryAbort(); +} + +void LMDBWriteTransaction::commit() +{ + if (state == TransactionState::ABORTED) { + throw std::runtime_error("Tried to commit reverted transaction"); + } + call_lmdb_func("mdb_txn_commit", mdb_txn_commit, _transaction); + state = TransactionState::COMMITTED; +} + +void LMDBWriteTransaction::tryAbort() +{ + if (state != TransactionState::OPEN) { + return; + } + LMDBTransaction::abort(); + state = TransactionState::ABORTED; +} + +void LMDBWriteTransaction::put_node(uint32_t level, index_t index, std::vector& data) +{ + NodeKeyType key = (static_cast(1 << level) + static_cast(index)) - 1; + put_value_by_integer(key, data); +} + +void LMDBWriteTransaction::put_value(std::vector& key, std::vector& data) +{ + MDB_val dbKey; + dbKey.mv_size = key.size(); + dbKey.mv_data = (void*)key.data(); + + MDB_val dbVal; + dbVal.mv_size = data.size(); + dbVal.mv_data = (void*)data.data(); + call_lmdb_func(mdb_put, underlying(), _database.underlying(), &dbKey, &dbVal, 0U); +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp new file mode 100644 index 00000000000..d7469e045c3 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp @@ -0,0 +1,44 @@ +#pragma once +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" + +namespace bb::crypto::merkle_tree { + +class LMDBWriteTransaction : public LMDBTransaction { + public: + using Ptr = std::unique_ptr; + + LMDBWriteTransaction(LMDBEnvironment& env, const LMDBDatabase& database); + LMDBWriteTransaction(const LMDBWriteTransaction& other) = delete; + LMDBWriteTransaction(LMDBWriteTransaction&& other) = delete; + LMDBWriteTransaction& operator=(const LMDBWriteTransaction& other) = delete; + LMDBWriteTransaction& operator=(LMDBWriteTransaction&& other) = delete; + ~LMDBWriteTransaction() override; + + void put_node(uint32_t level, index_t index, std::vector& data); + + template void put_value_by_integer(T& key, std::vector& data); + + void put_value(std::vector& key, std::vector& data); + + void commit(); + + void tryAbort(); + + protected: + const LMDBDatabase& _database; +}; + +template void LMDBWriteTransaction::put_value_by_integer(T& key, std::vector& data) +{ + MDB_val dbKey; + dbKey.mv_size = sizeof(T); + dbKey.mv_data = &key; + + MDB_val dbVal; + dbVal.mv_size = data.size(); + dbVal.mv_data = (void*)data.data(); + call_lmdb_func("mdb_put", mdb_put, underlying(), _database.underlying(), &dbKey, &dbVal, 0U); +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp index 64b96118364..a96de02dd09 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp @@ -74,7 +74,6 @@ template class CachedTreeStore void rollback(); ReadTransactionPtr createReadTransaction() const { return dataStore.createReadTransaction(); } - WriteTransactionPtr createWriteTransaction() const { return dataStore.createWriteTransaction(); } private: std::string name; @@ -90,6 +89,8 @@ template class CachedTreeStore bool readPersistedMeta(TreeMeta& m, ReadTransaction& tx) const; void persistMeta(TreeMeta& m, WriteTransaction& tx); + + WriteTransactionPtr createWriteTransaction() const { return dataStore.createWriteTransaction(); } }; template @@ -262,6 +263,7 @@ template void CachedTreeStore< tx->put_value_by_integer(key, value); } persistMeta(meta, *tx); + tx->commit(); } rollback(); } @@ -317,6 +319,7 @@ void CachedTreeStore::initialise() meta.depth = depth; WriteTransactionPtr tx = createWriteTransaction(); persistMeta(meta, *tx); + tx->commit(); } } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp index 6c55cd004dd..3f45aa76d8c 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp @@ -26,7 +26,7 @@ using namespace bb::crypto::merkle_tree; using HashPolicy = PedersenHashPolicy; struct TreeWithStore { - typedef CachedTreeStore Store; + typedef CachedTreeStore Store; std::unique_ptr lmdbStore; std::unique_ptr> tree; std::unique_ptr store; @@ -167,7 +167,7 @@ template bool WorldStateService::insertLea WorldStateMsgTypes::INSERT_LEAVES_RESPONSE, header, response); outputStream.sendPackedObject(insertLeavesResponse); }; - std::vector leaves(insertLeavesRequest.value.leaves.size()); + std::vector leaves(insertLeavesRequest.value.leaves.size()); for (auto& v : insertLeavesRequest.value.leaves) { leaves.emplace_back(v); } From 1bf0ff2e4d5bdcb8dcc491a238932d5ac093b638 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 2 Jul 2024 10:16:21 +0100 Subject: [PATCH 28/63] WIP --- .../append_only_tree.bench.cpp | 5 +- .../indexed_tree_bench/indexed_tree.bench.cpp | 17 +- .../append_only_tree/append_only_tree.hpp | 178 +++--- .../append_only_tree.test.cpp | 199 +++++-- .../merkle_tree/indexed_tree/indexed_leaf.hpp | 59 +- .../merkle_tree/indexed_tree/indexed_tree.hpp | 385 ++++++++----- .../indexed_tree/indexed_tree.test.cpp | 529 +++++++++++++----- .../merkle_tree/lmdb_store/functions.cpp | 24 +- .../merkle_tree/lmdb_store/functions.hpp | 2 +- .../lmdb_store/lmdb_environment.cpp | 23 +- .../lmdb_store/lmdb_environment.hpp | 2 +- .../lmdb_store/lmdb_store.test.cpp | 23 +- .../lmdb_store/lmdb_transaction.cpp | 4 + .../lmdb_store/lmdb_write_transaction.cpp | 7 +- .../lmdb_store/lmdb_write_transaction.hpp | 2 +- .../node_store/cached_tree_store.hpp | 62 +- .../nullifier_tree/nullifier_memory_tree.hpp | 2 +- .../crypto/merkle_tree/response.cpp | 3 + .../crypto/merkle_tree/response.hpp | 79 +++ .../crypto/merkle_tree/signal.hpp | 57 ++ .../barretenberg/crypto/merkle_tree/types.hpp | 2 - 21 files changed, 1203 insertions(+), 461 deletions(-) create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp index 8e0d4176de8..6a4f6954d83 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp @@ -7,6 +7,7 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" #include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp" #include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/numeric/random/engine.hpp" #include @@ -26,7 +27,7 @@ const size_t MAX_BATCH_SIZE = 128; template void perform_batch_insert(TreeType& tree, const std::vector& values) { Signal signal(1); - auto completion = [&](const fr&, index_t) -> void { signal.signal_level(0); }; + auto completion = [&](const TypedResponse&) -> void { signal.signal_level(0); }; tree.add_values(values, completion); signal.wait_for_level(0); @@ -49,7 +50,7 @@ template void append_only_tree_bench(State& state) noexcept std::string name = randomString(); std::filesystem::create_directories(directory); uint32_t num_threads = 16; - LMDBEnvironment environment = LMDBEnvironment(directory, 1024, 2, num_threads); + LMDBEnvironment environment = LMDBEnvironment(directory, 1024 * 1024, 2, num_threads); LMDBStore db(environment, name, false, false, IntegerKeyCmp); StoreType store(name, depth, db); diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp index ff7eff0a1ba..65ba9384a75 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp @@ -1,9 +1,10 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" #include "barretenberg/crypto/merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" #include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/numeric/random/engine.hpp" #include #include @@ -11,7 +12,7 @@ using namespace benchmark; using namespace bb::crypto::merkle_tree; -using StoreType = CachedTreeStore; +using StoreType = CachedTreeStore; using Poseidon2 = IndexedTree; using Pedersen = IndexedTree; @@ -19,10 +20,10 @@ using Pedersen = IndexedTree; const size_t TREE_DEPTH = 40; const size_t MAX_BATCH_SIZE = 128; -template void add_values(TreeType& tree, const std::vector& values) +template void add_values(TreeType& tree, const std::vector& values) { Signal signal(1); - auto completion = [&](const std::vector&, fr&, index_t) -> void { signal.signal_level(0); }; + auto completion = [&](const TypedResponse&) -> void { signal.signal_level(0); }; tree.add_or_update_values(values, completion); signal.wait_for_level(0); @@ -37,7 +38,7 @@ template void multi_thread_indexed_tree_bench(State& state) std::string name = randomString(); std::filesystem::create_directories(directory); uint32_t num_threads = 16; - LMDBEnvironment environment = LMDBEnvironment(directory, 1024, 2, num_threads); + LMDBEnvironment environment = LMDBEnvironment(directory, 1024 * 1024, 2, num_threads); LMDBStore db(environment, name, false, false, IntegerKeyCmp); StoreType store(name, depth, db); @@ -46,7 +47,7 @@ template void multi_thread_indexed_tree_bench(State& state) for (auto _ : state) { state.PauseTiming(); - std::vector values(batch_size); + std::vector values(batch_size); for (size_t i = 0; i < batch_size; ++i) { values[i] = fr(random_engine.get_random_uint256()); } @@ -64,7 +65,7 @@ template void single_thread_indexed_tree_bench(State& state) std::string name = randomString(); std::filesystem::create_directories(directory); uint32_t num_threads = 1; - LMDBEnvironment environment = LMDBEnvironment(directory, 1024, 2, num_threads); + LMDBEnvironment environment = LMDBEnvironment(directory, 1024 * 1024, 2, num_threads); LMDBStore db(environment, name, false, false, IntegerKeyCmp); StoreType store(name, depth, db); @@ -73,7 +74,7 @@ template void single_thread_indexed_tree_bench(State& state) for (auto _ : state) { state.PauseTiming(); - std::vector values(batch_size); + std::vector values(batch_size); for (size_t i = 0; i < batch_size; ++i) { values[i] = fr(random_engine.get_random_uint256()); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 67ec020fd0e..7840e18a386 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -1,12 +1,16 @@ #pragma once #include "../hash_path.hpp" #include "../node_store//tree_meta.hpp" +#include "../response.hpp" #include "../types.hpp" #include "barretenberg/common/thread_pool.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/numeric/bitop/pow.hpp" +#include #include #include #include +#include #include namespace bb::crypto::merkle_tree { @@ -20,11 +24,11 @@ using namespace bb; */ template class AppendOnlyTree { public: - using AppendCompletionCallback = std::function; - using MetaDataCallback = std::function; - using HashPathCallback = std::function; - using CommitCallback = std::function; - using RollbackCallback = std::function; + using AppendCompletionCallback = std::function&)>; + using MetaDataCallback = std::function&)>; + using HashPathCallback = std::function&)>; + using CommitCallback = std::function; + using RollbackCallback = std::function; AppendOnlyTree(Store& store, ThreadPool& workers); AppendOnlyTree(AppendOnlyTree const& other) = delete; @@ -66,9 +70,12 @@ template class AppendOnlyTree { ReadTransaction& tx, bool includeUncommitted) const; + void add_values_internal(std::shared_ptr> values, fr& new_root, index_t& new_size); + Store& store_; uint32_t depth_; std::string name_; + uint64_t max_size_; std::vector zero_hashes_; ThreadPool& workers_; }; @@ -98,19 +105,19 @@ AppendOnlyTree::AppendOnlyTree(Store& store, ThreadPool& w store_.put_meta(0, current); store_.commit(); } + max_size_ = numeric::pow64(2, depth_); } template void AppendOnlyTree::get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion) { auto job = [=]() { - index_t size = 0; - bb::fr root = fr::zero(); - { - ReadTransactionPtr tx = store_.createReadTransaction(); - store_.get_meta(size, root, *tx, includeUncommitted); - } - on_completion(name_, depth_, size, root); + ExecuteAndReport( + [=](TypedResponse& response) { + ReadTransactionPtr tx = store_.createReadTransaction(); + store_.get_meta(response.inner.size, response.inner.root, *tx, includeUncommitted); + }, + on_completion); }; workers_.enqueue(job); } @@ -121,22 +128,21 @@ void AppendOnlyTree::get_hash_path(const index_t& index, bool includeUncommitted) const { auto job = [=]() { - fr_hash_path path; - index_t current_index = index; - { - ReadTransactionPtr tx = store_.createReadTransaction(); - - for (uint32_t level = depth_; level > 0; --level) { - bool is_right = static_cast(current_index & 0x01); - fr right_value = is_right ? get_element_or_zero(level, current_index, *tx, includeUncommitted) - : get_element_or_zero(level, current_index + 1, *tx, includeUncommitted); - fr left_value = is_right ? get_element_or_zero(level, current_index - 1, *tx, includeUncommitted) - : get_element_or_zero(level, current_index, *tx, includeUncommitted); - path.emplace_back(left_value, right_value); - current_index >>= 1; - } - } - on_completion(path); + ExecuteAndReport( + [=](TypedResponse& response) { + index_t current_index = index; + ReadTransactionPtr tx = store_.createReadTransaction(); + for (uint32_t level = depth_; level > 0; --level) { + bool is_right = static_cast(current_index & 0x01); + fr right_value = is_right ? get_element_or_zero(level, current_index, *tx, includeUncommitted) + : get_element_or_zero(level, current_index + 1, *tx, includeUncommitted); + fr left_value = is_right ? get_element_or_zero(level, current_index - 1, *tx, includeUncommitted) + : get_element_or_zero(level, current_index, *tx, includeUncommitted); + response.inner.path.emplace_back(left_value, right_value); + current_index >>= 1; + } + }, + on_completion); }; workers_.enqueue(job); } @@ -151,59 +157,70 @@ template void AppendOnlyTree::add_values(const std::vector& values, const AppendCompletionCallback& on_completion) { - uint32_t start_level = depth_; std::shared_ptr> hashes = std::make_shared>(values); - auto append_op = [=]() -> void { - bb::fr new_root = fr::zero(); - index_t new_size = 0; - { - typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); - index_t start_size = 0; - bb::fr root; - store_.get_meta(start_size, root, *tx, true); - index_t index = start_size; - uint32_t level = start_level; - auto number_to_insert = static_cast(values.size()); - std::vector& hashes_local = *hashes; - // Add the values at the leaf nodes of the tree - for (uint32_t i = 0; i < number_to_insert; ++i) { - write_node(level, index + i, hashes_local[i]); - } - - // Hash the values as a sub tree and insert them - while (number_to_insert > 1) { - number_to_insert >>= 1; - index >>= 1; - --level; - for (uint32_t i = 0; i < number_to_insert; ++i) { - hashes_local[i] = HashingPolicy::hash_pair(hashes_local[i * 2], hashes_local[i * 2 + 1]); - write_node(level, index + i, hashes_local[i]); - } - } - - // Hash from the root of the sub-tree to the root of the overall tree - fr new_hash = hashes_local[0]; - while (level > 0) { - bool is_right = static_cast(index & 0x01); - fr left_hash = is_right ? get_element_or_zero(level, index - 1, *tx, true) : new_hash; - fr right_hash = is_right ? new_hash : get_element_or_zero(level, index + 1, *tx, true); - new_hash = HashingPolicy::hash_pair(left_hash, right_hash); - index >>= 1; - --level; - if (level > 0) { - write_node(level, index, new_hash); - } - } - new_size = start_size + values.size(); - new_root = new_hash; - store_.put_meta(new_size, new_root); - } - on_completion(new_root, new_size); + ExecuteAndReport( + [=](TypedResponse& response) { + add_values_internal(hashes, response.inner.root, response.inner.size); + }, + on_completion); }; workers_.enqueue(append_op); } +template +void AppendOnlyTree::add_values_internal(std::shared_ptr> values, + fr& new_root, + index_t& new_size) +{ + uint32_t start_level = depth_; + uint32_t level = start_level; + std::vector& hashes_local = *values; + auto number_to_insert = static_cast(hashes_local.size()); + index_t start_size = 0; + bb::fr root; + + typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); + store_.get_meta(start_size, root, *tx, true); + index_t index = start_size; + new_size = start_size + number_to_insert; + if (new_size > max_size_) { + throw std::runtime_error("Tree is full"); + } + + // Add the values at the leaf nodes of the tree + for (uint32_t i = 0; i < number_to_insert; ++i) { + write_node(level, index + i, hashes_local[i]); + } + + // Hash the values as a sub tree and insert them + while (number_to_insert > 1) { + number_to_insert >>= 1; + index >>= 1; + --level; + for (uint32_t i = 0; i < number_to_insert; ++i) { + hashes_local[i] = HashingPolicy::hash_pair(hashes_local[i * 2], hashes_local[i * 2 + 1]); + write_node(level, index + i, hashes_local[i]); + } + } + + // Hash from the root of the sub-tree to the root of the overall tree + fr new_hash = hashes_local[0]; + while (level > 0) { + bool is_right = static_cast(index & 0x01); + fr left_hash = is_right ? get_element_or_zero(level, index - 1, *tx, true) : new_hash; + fr right_hash = is_right ? new_hash : get_element_or_zero(level, index + 1, *tx, true); + new_hash = HashingPolicy::hash_pair(left_hash, right_hash); + index >>= 1; + --level; + if (level > 0) { + write_node(level, index, new_hash); + } + } + new_root = new_hash; + store_.put_meta(new_size, new_root); +} + template fr AppendOnlyTree::get_element_or_zero(uint32_t level, const index_t& index, @@ -214,7 +231,6 @@ fr AppendOnlyTree::get_element_or_zero(uint32_t level, if (read_data.first) { return read_data.second; } - ASSERT(level > 0 && level < zero_hashes_.size()); return zero_hashes_[level]; } @@ -244,20 +260,14 @@ std::pair AppendOnlyTree::read_node(uint32_t lev template void AppendOnlyTree::commit(const CommitCallback& on_completion) { - auto job = [=]() { - store_.commit(); - on_completion(); - }; + auto job = [=]() { ExecuteAndReport([=]() { store_.commit(); }, on_completion); }; workers_.enqueue(job); } template void AppendOnlyTree::rollback(const RollbackCallback& on_completion) { - auto job = [=]() { - store_.rollback(); - on_completion(); - }; + auto job = [=]() { ExecuteAndReport([=]() { store_.rollback(); }, on_completion); }; workers_.enqueue(job); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp index 3191d71dac0..1649b40ff45 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -6,6 +6,7 @@ #include "barretenberg/common/thread_pool.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" #include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp" #include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" @@ -29,7 +30,7 @@ class PersistedAppendOnlyTreeTest : public testing::Test { // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers _directory = randomTempDirectory(); std::filesystem::create_directories(_directory); - _environment = std::make_unique(_directory, 1, 2, 2); + _environment = std::make_unique(_directory, 1024, 2, 2); } void TearDown() override { std::filesystem::remove_all(_directory); } @@ -43,61 +44,84 @@ std::string PersistedAppendOnlyTreeTest::_directory; void check_size(TreeType& tree, index_t expected_size, bool includeUncommitted = true) { - Signal signal(1); - auto completion = [&](const std::string&, uint32_t, const index_t& size, const fr&) -> void { - EXPECT_EQ(size, expected_size); - signal.signal_level(0); + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, true); + EXPECT_EQ(response.inner.size, expected_size); + signal.signal_level(); }; tree.get_meta_data(includeUncommitted, completion); - signal.wait_for_level(0); + signal.wait_for_level(); } void check_root(TreeType& tree, fr expected_root, bool includeUncommitted = true) { - Signal signal(1); - auto completion = [&](const std::string&, uint32_t, const index_t&, const fr& root) -> void { - EXPECT_EQ(root, expected_root); - signal.signal_level(0); + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, true); + EXPECT_EQ(response.inner.root, expected_root); + signal.signal_level(); }; tree.get_meta_data(includeUncommitted, completion); - signal.wait_for_level(0); + signal.wait_for_level(); } void check_hash_path(TreeType& tree, index_t index, fr_hash_path expected_hash_path, bool includeUncommitted = true) { - Signal signal(1); - auto completion = [&](const fr_hash_path& path) -> void { - EXPECT_EQ(path, expected_hash_path); - signal.signal_level(0); + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, true); + EXPECT_EQ(response.inner.path, expected_hash_path); + signal.signal_level(); }; tree.get_hash_path(index, completion, includeUncommitted); - signal.wait_for_level(0); + signal.wait_for_level(); } void commit_tree(TreeType& tree) { - Signal signal(1); - auto completion = [&]() -> void { signal.signal_level(0); }; + Signal signal; + auto completion = [&](const Response& response) -> void { + EXPECT_EQ(response.success, true); + signal.signal_level(); + }; tree.commit(completion); - signal.wait_for_level(0); + signal.wait_for_level(); +} + +void rollback_tree(TreeType& tree) +{ + Signal signal; + auto completion = [&](const Response& response) -> void { + EXPECT_EQ(response.success, true); + signal.signal_level(); + }; + tree.rollback(completion); + signal.wait_for_level(); } void add_value(TreeType& tree, const fr& value) { - Signal signal(1); - auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, true); + signal.signal_level(); + }; tree.add_value(value, completion); - signal.wait_for_level(0); + signal.wait_for_level(); } void add_values(TreeType& tree, const std::vector& values) { - Signal signal(1); - auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, true); + signal.signal_level(); + }; tree.add_values(values, completion); - signal.wait_for_level(0); + signal.wait_for_level(); } TEST_F(PersistedAppendOnlyTreeTest, can_create) @@ -141,18 +165,125 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_value_and_get_hash_path) check_size(tree, 0); check_root(tree, memdb.root()); - Signal signal(1); - auto completion = [&](fr, index_t) -> void { signal.signal_level(0); }; - memdb.update_element(0, VALUES[0]); - tree.add_value(VALUES[0], completion); - signal.wait_for_level(0); + add_value(tree, VALUES[0]); check_size(tree, 1); check_root(tree, memdb.root()); check_hash_path(tree, 0, memdb.get_hash_path(0)); } +TEST_F(PersistedAppendOnlyTreeTest, reports_an_error_if_tree_is_overfilled) +{ + constexpr size_t depth = 4; + std::string name = randomString(); + std::string directory = randomTempDirectory(); + std::filesystem::create_directories(directory); + auto environment = std::make_unique(directory, 1024, 1, 2); + LMDBStore db(*environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + + ThreadPool pool(1); + TreeType tree(store, pool); + + std::vector values; + for (uint32_t i = 0; i < 16; i++) { + values.push_back(VALUES[i]); + } + add_values(tree, values); + + Signal signal; + auto add_completion = [&](const TypedResponse& response) { + EXPECT_EQ(response.success, false); + EXPECT_EQ(response.message, "Tree is full"); + signal.signal_level(); + }; + tree.add_value(VALUES[16], add_completion); + signal.wait_for_level(); + std::filesystem::remove_all(directory); +} + +TEST_F(PersistedAppendOnlyTreeTest, errors_are_caught_and_handled) +{ + // We use a deep tree with a small amount of storage (20 * 1024) bytes + constexpr size_t depth = 16; + std::string name = randomString(); + std::string directory = randomTempDirectory(); + std::filesystem::create_directories(directory); + auto environment = std::make_unique(directory, 300, 1, 2); + LMDBStore db(*environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + + ThreadPool pool(1); + TreeType tree(store, pool); + MemoryTree memdb(depth); + + // check the committed data only, so we read from the db + check_size(tree, 0, false); + check_root(tree, memdb.root(), false); + + fr empty_root = memdb.root(); + + // Add lots of values to the tree + uint32_t num_values_to_add = 16 * 1024; + std::vector values(num_values_to_add, VALUES[0]); + for (uint32_t i = 0; i < num_values_to_add; i++) { + memdb.update_element(i, VALUES[0]); + } + add_values(tree, values); + + // check the uncommitted data is accurate + check_size(tree, num_values_to_add, true); + check_root(tree, memdb.root(), true); + + // trying to commit that should fail + Signal signal; + auto completion = [&](const Response& response) -> void { + EXPECT_EQ(response.success, false); + signal.signal_level(); + }; + + tree.commit(completion); + signal.wait_for_level(); + + // At this stage, the tree is still in an uncommited state despite the error + // Reading both committed and uncommitted data shold be ok + + // check the uncommitted data is accurate + check_size(tree, num_values_to_add, true); + check_root(tree, memdb.root(), true); + + // Reading committed data should still work + check_size(tree, 0, false); + check_root(tree, empty_root, false); + + // Now rollback the tree + rollback_tree(tree); + + // committed and uncommitted data should be as an empty tree + check_size(tree, 0, true); + check_root(tree, empty_root, true); + + // Reading committed data should still work + check_size(tree, 0, false); + check_root(tree, empty_root, false); + + // // Now add a single value and commit it + add_value(tree, VALUES[0]); + commit_tree(tree); + + MemoryTree memdb2(depth); + memdb2.update_element(0, VALUES[0]); + + // committed and uncommitted data should be equal to the tree with 1 item + check_size(tree, 1, true); + check_root(tree, memdb2.root(), true); + + // Reading committed data should still work + check_size(tree, 1, false); + check_root(tree, memdb2.root(), false); +} + TEST_F(PersistedAppendOnlyTreeTest, can_commit_and_restore) { constexpr size_t depth = 10; @@ -332,15 +463,17 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_single_whilst_reading) Signal signal(2); - auto add_completion = [&](const fr&, index_t) { + auto add_completion = [&](const TypedResponse&) { signal.signal_level(1); - auto commit_completion = [&]() { signal.signal_level(0); }; + auto commit_completion = [&](const Response&) { signal.signal_level(0); }; tree.commit(commit_completion); }; tree.add_value(VALUES[0], add_completion); for (size_t i = 0; i < num_reads; i++) { - auto completion = [&, i](const fr_hash_path& path) { paths[i] = path; }; + auto completion = [&, i](const TypedResponse& response) { + paths[i] = response.inner.path; + }; tree.get_hash_path(0, completion, false); } signal.wait_for_level(0); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp index 3e6015a370c..db28f2f40bd 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp @@ -42,9 +42,62 @@ struct NullifierLeafValue { return os; } - fr get_fr_value() const { return value; } + fr get_key() const { return value; } - static NullifierLeafValue empty() { return { 0 }; } + bool is_empty() const { return value == fr::zero(); } + + static NullifierLeafValue empty() { return { fr::zero() }; } + + static NullifierLeafValue padding(index_t i) { return { i }; } +}; + +struct PublicDataLeafValue { + fr value; + fr slot; + + MSGPACK_FIELDS(value, slot) + + PublicDataLeafValue() = default; + PublicDataLeafValue(const fr& s, const fr& v) + : value(v) + , slot(s) + {} + PublicDataLeafValue(const PublicDataLeafValue& other) = default; + PublicDataLeafValue(PublicDataLeafValue&& other) = default; + PublicDataLeafValue& operator=(const PublicDataLeafValue& other) + { + if (this != &other) { + value = other.value; + slot = other.slot; + } + return *this; + } + + PublicDataLeafValue& operator=(PublicDataLeafValue&& other) noexcept + { + if (this != &other) { + value = other.value; + slot = other.slot; + } + return *this; + } + ~PublicDataLeafValue() = default; + + bool operator==(PublicDataLeafValue const& other) const { return value == other.value && slot == other.slot; } + + friend std::ostream& operator<<(std::ostream& os, const PublicDataLeafValue& v) + { + os << "slot = " << v.slot << " : value = " << v.value; + return os; + } + + fr get_key() const { return slot; } + + bool is_empty() const { return value == fr::zero() && slot == fr::zero(); } + + static PublicDataLeafValue empty() { return { fr::zero(), fr::zero() }; } + + static PublicDataLeafValue padding(index_t i) { return { i, fr::zero() }; } }; template struct IndexedLeaf { @@ -100,6 +153,8 @@ template struct IndexedLeaf { std::vector get_hash_inputs() const { return std::vector({ value.value, nextIndex, nextValue }); } static IndexedLeaf empty() { return { LeafType::empty(), 0, 0 }; } + + static IndexedLeaf padding(index_t i) { return { LeafType::padding(i), 0, 0 }; } }; } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index a30c0ed836e..08539e19d57 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -1,68 +1,26 @@ #pragma once -#include "../../../common/thread.hpp" #include "../append_only_tree/append_only_tree.hpp" #include "../hash.hpp" #include "../hash_path.hpp" +#include "../signal.hpp" #include "barretenberg/common/assert.hpp" #include "barretenberg/common/thread_pool.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/numeric/uint256/uint256.hpp" #include "indexed_leaf.hpp" #include #include +#include #include #include #include +#include +#include #include namespace bb::crypto::merkle_tree { -/** - * @brief Used in parallel insertions in the the IndexedTree. Workers signal to other following workes as they move up - * the level of the tree. - * - */ -class Signal { - public: - Signal(uint32_t initial_level) - : signal_(initial_level){}; - ~Signal() = default; - Signal(const Signal& other) - : signal_(other.signal_.load()) - {} - Signal(const Signal&& other) noexcept - : signal_(other.signal_.load()) - {} - Signal& operator=(const Signal& other) = delete; - Signal& operator=(const Signal&& other) = delete; - - /** - * @brief Causes the thread to wait until the required level has been signalled - * @param level The required level - * - */ - void wait_for_level(uint32_t level) - { - uint32_t current_level = signal_.load(); - while (current_level > level) { - signal_.wait(current_level); - current_level = signal_.load(); - } - } - - /** - * @brief Signals that the given level has been passed - * @param level The level to be signalled - * - */ - void signal_level(uint32_t level) - { - signal_.store(level); - signal_.notify_all(); - } - - private: - std::atomic signal_; -}; - /** * @brief Implements a parallelised batch insertion indexed tree * Accepts template argument of the type of store backing the tree, the type of store containing the leaves and the @@ -73,8 +31,8 @@ template class IndexedTree : public App public: using LeafValueType = typename Store::LeafType; using IndexedLeafValueType = typename Store::IndexedLeafValueType; - using AddCompletionCallback = std::function&, fr&, index_t)>; - using LeafCallback = std::function; + using AddCompletionCallback = std::function&)>; + using LeafCallback = std::function>&)>; IndexedTree(Store& store, ThreadPool& workers, index_t initial_size); IndexedTree(IndexedTree const& other) = delete; @@ -107,7 +65,19 @@ template class IndexedTree : public App using ReadTransaction = typename Store::ReadTransaction; using ReadTransactionPtr = typename Store::ReadTransactionPtr; - struct leaf_insertion { + struct Status { + std::atomic_bool success{ true }; + std::string message; + + void set_failure(const std::string& msg) + { + if (success.exchange(false)) { + message = msg; + } + } + }; + + struct LeafInsertion { index_t low_leaf_index; IndexedLeafValueType low_leaf; }; @@ -119,16 +89,28 @@ template class IndexedTree : public App fr_hash_path& previous_hash_path, ReadTransaction& tx); - using InsertionGenerationCallback = std::function>, - std::shared_ptr>)>; + struct InsertionGenerationResponse { + std::shared_ptr> insertions; + std::shared_ptr> indexed_leaves; + }; + + using InsertionGenerationCallback = std::function&)>; void generate_insertions(const std::shared_ptr>>& values_to_be_sorted, const InsertionGenerationCallback& completion); - using InsertionCompletionCallback = std::function> paths)>; - void perform_insertions(std::shared_ptr> insertions, + struct InsertionCompletionResponse { + std::shared_ptr> paths; + }; + + using InsertionCompletionCallback = std::function&)>; + void perform_insertions(std::shared_ptr> insertions, const InsertionCompletionCallback& completion); - using HashGenerationCallback = std::function> hashes)>; + struct HashGenerationResponse { + std::shared_ptr> hashes; + }; + + using HashGenerationCallback = std::function&)>; void generate_hashes_for_appending(std::shared_ptr> leaves_to_hash, const HashGenerationCallback& completion); @@ -144,13 +126,16 @@ template class IndexedTree : public App using AppendOnlyTree::depth_; using AppendOnlyTree::name_; using AppendOnlyTree::workers_; + using AppendOnlyTree::max_size_; }; template IndexedTree::IndexedTree(Store& store, ThreadPool& workers, index_t initial_size) : AppendOnlyTree(store, workers) { - ASSERT(initial_size > 0); + if (initial_size < 2) { + throw std::runtime_error("Indexed trees must have initial size > 1"); + } zero_hashes_.resize(depth_ + 1); // Create the zero hashes for the tree @@ -181,16 +166,24 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers for (uint32_t i = 0; i < initial_size; ++i) { // Insert the zero leaf to the `leaves` and also to the tree at index 0. bool last = i == (initial_size - 1); - IndexedLeafValueType initial_leaf = IndexedLeafValueType(LeafValueType(i), last ? 0 : i + 1, last ? 0 : i + 1); + IndexedLeafValueType initial_leaf = + IndexedLeafValueType(LeafValueType::padding(i), last ? 0 : i + 1, last ? 0 : i + 1); appended_leaves.push_back(initial_leaf); appended_hashes.push_back(HashingPolicy::hash(initial_leaf.get_hash_inputs())); store_.set_at_index(i, initial_leaf, true); } + bool success = true; Signal signal(1); - AppendCompletionCallback completion = [&](fr, index_t) -> void { signal.signal_level(0); }; + AppendCompletionCallback completion = [&](const TypedResponse& result) -> void { + success = result.success; + signal.signal_level(0); + }; AppendOnlyTree::add_values(appended_hashes, completion); signal.wait_for_level(0); + if (!success) { + throw std::runtime_error("Failed to initialise tree"); + } store_.commit(); } @@ -200,12 +193,12 @@ void IndexedTree::get_leaf(const index_t& index, const LeafCallback& completion) { auto job = [=]() { - IndexedLeafValueType leaf; - { - ReadTransactionPtr tx = store_.createReadTransaction(); - leaf = store_.get_leaf(index, *tx, includeUncommitted); - } - completion(leaf); + ExecuteAndReport>( + [=](TypedResponse>& response) { + ReadTransactionPtr tx = store_.createReadTransaction(); + response.inner.indexed_leaf = store_.get_leaf(index, *tx, includeUncommitted); + }, + completion); }; workers_.enqueue(job); } @@ -221,71 +214,144 @@ template void IndexedTree::add_or_update_values(const std::vector& values, const AddCompletionCallback& completion) { + // We first take a copy of the leaf values and their locations within the set given to us std::shared_ptr>> values_to_be_sorted = std::make_shared>>(values.size()); for (size_t i = 0; i < values.size(); ++i) { (*values_to_be_sorted)[i] = std::make_pair(values[i], i); } + // This is to collect some state from the asynchronous operations we are about to perform struct IntermediateResults { std::shared_ptr> hashes_to_append; std::shared_ptr> paths; std::atomic count; + Status status; + // We set to 2 here as we will kick off the 2 main async operations concurrently and we need to trakc thri + // completion IntermediateResults() - : count(2){}; + : count(2) + { + // Default to success, set to false on error + status.success = true; + }; }; std::shared_ptr results = std::make_shared(); - AppendCompletionCallback final_completion = [=](fr root, index_t size) { completion(*results->paths, root, size); }; + auto on_error = [=](const std::string& message) { + try { + TypedResponse response; + response.success = false; + response.message = message; + completion(response); + } catch (std::exception&) { + } + }; - HashGenerationCallback hash_completion = [=](const std::shared_ptr>& hashes_to_append) { - results->hashes_to_append = hashes_to_append; - if (results->count.fetch_sub(1) == 1) { - AppendOnlyTree::add_values(*hashes_to_append, final_completion); + // Thsi is the final callback triggered once the leaves have been appended to the tree + AppendCompletionCallback final_completion = [=](const TypedResponse& add_result) { + TypedResponse response; + response.success = add_result.success; + response.message = add_result.message; + if (add_result.success) { + response.inner.add_data_result = add_result.inner; + response.inner.paths = results->paths; } + // Trigger the client's provided callback + completion(response); }; - InsertionCompletionCallback insertion_completion = [=](const std::shared_ptr>& paths) { - results->paths = paths; + // This signals the completion of the appended hash generation + // If the low leaf updates are also completed then we will append the leaves + HashGenerationCallback hash_completion = [=](const TypedResponse& hashes_response) { + if (!hashes_response.success) { + results->status.set_failure(hashes_response.message); + } else { + results->hashes_to_append = hashes_response.inner.hashes; + } if (results->count.fetch_sub(1) == 1) { - AppendOnlyTree::add_values(*results->hashes_to_append, final_completion); + if (!results->status.success) { + on_error(results->status.message); + return; + } + AppendOnlyTree::add_values((*results->hashes_to_append), final_completion); } }; + // This signals the completion of the low leaf updates + // If the append hash generation has also copleted then the hashes can be appended + InsertionCompletionCallback insertion_completion = + [=](const TypedResponse& insertion_response) { + if (!insertion_response.success) { + results->status.set_failure(insertion_response.message); + } else { + results->paths = insertion_response.inner.paths; + } + if (results->count.fetch_sub(1) == 1) { + if (!results->status.success) { + on_error(results->status.message); + return; + } + AppendOnlyTree::add_values((*results->hashes_to_append), final_completion); + } + }; + + // This signals the completion of the insertion data generation + // Here we will enqueue both the generation of the appended hashes and the low leaf updates (insertions) InsertionGenerationCallback insertion_generation_completed = - [=](std::shared_ptr> insertions, - std::shared_ptr> leaves_to_append) { - workers_.enqueue([=]() { generate_hashes_for_appending(leaves_to_append, hash_completion); }); - perform_insertions(insertions, insertion_completion); + [=](const TypedResponse& insertion_response) { + if (!insertion_response.success) { + on_error(insertion_response.message); + return; + } + workers_.enqueue( + [=]() { generate_hashes_for_appending(insertion_response.inner.indexed_leaves, hash_completion); }); + perform_insertions(insertion_response.inner.insertions, insertion_completion); }; + + // We start by enqueueing the insertion data generation workers_.enqueue([=]() { generate_insertions(values_to_be_sorted, insertion_generation_completed); }); } template -void IndexedTree::perform_insertions(std::shared_ptr> insertions, +void IndexedTree::perform_insertions(std::shared_ptr> insertions, const InsertionCompletionCallback& completion) { // We now kick off multiple workers to perform the low leaf updates // We create set of signals to coordinate the workers as the move up the tree std::shared_ptr> paths = std::make_shared>(insertions->size()); std::shared_ptr> signals = std::make_shared>(); + std::shared_ptr status = std::make_shared(); // The first signal is set to 0. This ensures the first worker up the tree is not impeded signals->emplace_back(0); // Workers will follow their leaders up the tree, being triggered by the signal in front of them for (size_t i = 0; i < insertions->size(); ++i) { signals->emplace_back(uint32_t(1 + depth_)); } + for (uint32_t i = 0; i < insertions->size(); ++i) { auto op = [=]() { - leaf_insertion& insertion = (*insertions)[i]; - { + LeafInsertion& insertion = (*insertions)[i]; + Signal& leaderSignal = (*signals)[i]; + Signal& followerSignal = (*signals)[i + 1]; + try { ReadTransactionPtr tx = store_.createReadTransaction(); update_leaf_and_hash_to_root( - insertion.low_leaf_index, insertion.low_leaf, (*signals)[i], (*signals)[i + 1], (*paths)[i], *tx); + insertion.low_leaf_index, insertion.low_leaf, leaderSignal, followerSignal, (*paths)[i], *tx); + } catch (std::exception& e) { + status->set_failure(e.what()); + // ensure that any followers are not blocked by our failure + followerSignal.signal_level(0); } if (i == insertions->size() - 1) { - completion(paths); + TypedResponse response; + response.success = status->success; + response.message = status->message; + if (response.success) { + response.inner.paths = paths; + } + completion(response); } }; workers_.enqueue(op); @@ -296,12 +362,15 @@ template void IndexedTree::generate_hashes_for_appending( std::shared_ptr> leaves_to_hash, const HashGenerationCallback& completion) { - std::shared_ptr> hashed = std::make_shared>(leaves_to_hash->size()); - std::vector& leaves = *leaves_to_hash; - for (uint32_t i = 0; i < leaves.size(); ++i) { - (*hashed)[i] = HashingPolicy::hash(leaves[i].get_hash_inputs()); - } - completion(hashed); + ExecuteAndReport( + [=](TypedResponse& response) { + response.inner.hashes = std::make_shared>(leaves_to_hash->size()); + std::vector& leaves = *leaves_to_hash; + for (uint32_t i = 0; i < leaves.size(); ++i) { + (*response.inner.hashes)[i] = HashingPolicy::hash(leaves[i].get_hash_inputs()); + } + }, + completion); } template @@ -309,62 +378,88 @@ void IndexedTree::generate_insertions( const std::shared_ptr>>& values_to_be_sorted, const InsertionGenerationCallback& on_completion) { - // The first thing we do is sort the values into descending order but maintain knowledge of their orignal order - struct { - bool operator()(std::pair& a, std::pair& b) const - { - return uint256_t(a.first.get_fr_value()) > uint256_t(b.first.get_fr_value()); - } - } comp; - std::sort(values_to_be_sorted->begin(), values_to_be_sorted->end(), comp); - - std::vector>& values = *values_to_be_sorted; - - // Now that we have the sorted values we need to identify the leaves that need updating. - // This is performed sequentially and is stored in this 'leaf_insertion' struct - std::shared_ptr> insertions = - std::make_shared>(values.size()); - std::shared_ptr> leaves_to_append = - std::make_shared>(values.size()); - index_t old_size = 0; - { - ReadTransactionPtr tx = store_.createReadTransaction(); - bb::fr old_root = fr::zero(); - store_.get_meta(old_size, old_root, *tx, true); - for (size_t i = 0; i < values.size(); ++i) { - fr value = values[i].first.get_fr_value(); - size_t index_into_appended_leaves = values[i].second; - index_t index_of_new_leaf = static_cast(index_into_appended_leaves) + old_size; - - // This gives us the leaf that need updating - index_t current = 0; - bool is_already_present = false; - std::tie(is_already_present, current) = store_.find_low_value(values[i].first, true, *tx); - IndexedLeafValueType current_leaf = store_.get_leaf(current, *tx, true); - - IndexedLeafValueType new_leaf = - IndexedLeafValueType(values[i].first, current_leaf.nextIndex, current_leaf.nextValue); - - // We only handle new values being added. We don't yet handle values being updated - if (!is_already_present) { - // Update the current leaf to point it to the new leaf - current_leaf.nextIndex = index_of_new_leaf; - current_leaf.nextValue = value; - store_.set_at_index(current, current_leaf, false); - store_.set_at_index(index_of_new_leaf, new_leaf, true); - } else { + ExecuteAndReport( + [=](TypedResponse& response) { + // The first thing we do is sort the values into descending order but maintain knowledge of their orignal + // order + struct { + bool operator()(std::pair& a, std::pair& b) const + { + return uint256_t(a.first.get_key()) > uint256_t(b.first.get_key()); + } + } comp; + std::sort(values_to_be_sorted->begin(), values_to_be_sorted->end(), comp); + + std::vector>& values = *values_to_be_sorted; + + // Now that we have the sorted values we need to identify the leaves that need updating. + // This is performed sequentially and is stored in this 'leaf_insertion' struct + response.inner.insertions = std::make_shared>(values.size()); + response.inner.indexed_leaves = + std::make_shared>(values.size(), IndexedLeafValueType::empty()); + index_t old_size = 0; + index_t num_leaves_to_be_inserted = values.size(); + std::set unique_values; + { + ReadTransactionPtr tx = store_.createReadTransaction(); + bb::fr old_root = fr::zero(); + store_.get_meta(old_size, old_root, *tx, true); + // Ensure that the tree is not going to be overfilled + index_t new_total_size = num_leaves_to_be_inserted + old_size; + if (new_total_size > max_size_) { + throw std::runtime_error("Tree is full"); + } + for (size_t i = 0; i < values.size(); ++i) { + std::pair& value_pair = values[i]; + size_t index_into_appended_leaves = value_pair.second; + index_t index_of_new_leaf = static_cast(index_into_appended_leaves) + old_size; + if (value_pair.first.is_empty()) { + continue; + } + fr value = value_pair.first.get_key(); + auto it = unique_values.insert(value); + if (!it.second) { + throw std::runtime_error("Duplicate key not allowed in same batch"); + } + + // This gives us the leaf that need updating + index_t current = 0; + bool is_already_present = false; + std::tie(is_already_present, current) = store_.find_low_value(value_pair.first, true, *tx); + IndexedLeafValueType current_leaf = store_.get_leaf(current, *tx, true); + + // We only handle new values being added. We don't yet handle values being updated + if (!is_already_present) { + // Update the current leaf to point it to the new leaf + IndexedLeafValueType new_leaf = + IndexedLeafValueType(value_pair.first, current_leaf.nextIndex, current_leaf.nextValue); + current_leaf.nextIndex = index_of_new_leaf; + current_leaf.nextValue = value; + store_.set_at_index(current, current_leaf, false); + store_.set_at_index(index_of_new_leaf, new_leaf, true); + + // Update the set of leaves to append + (*response.inner.indexed_leaves)[index_into_appended_leaves] = new_leaf; + } else { + // Update the current leaf's value, don't change it's link + IndexedLeafValueType replacement_leaf = + IndexedLeafValueType(value_pair.first, current_leaf.nextIndex, current_leaf.nextValue); + store_.set_at_index(current, replacement_leaf, false); + IndexedLeafValueType empty_leaf = IndexedLeafValueType::empty(); + store_.set_at_index(index_of_new_leaf, empty_leaf, true); + // The set of appended leaves already has an empty leaf in the slot at index + // 'index_into_appended_leaves' + } + + // Capture the index and value of the updated 'low' leaf as well as the new leaf to be appended + LeafInsertion& insertion = (*response.inner.insertions)[i]; + insertion.low_leaf_index = current; + insertion.low_leaf = + IndexedLeafValueType(current_leaf.value, current_leaf.nextIndex, current_leaf.nextValue); + } } - - (*leaves_to_append)[index_into_appended_leaves] = new_leaf; - - // Capture the index and value of the updated 'low' leaf as well as the new leaf to be appended - leaf_insertion& insertion = (*insertions)[i]; - insertion.low_leaf_index = current; - insertion.low_leaf = - IndexedLeafValueType(current_leaf.value, current_leaf.nextIndex, current_leaf.nextValue); - } - } - on_completion(insertions, leaves_to_append); + }, + on_completion); } template diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index 390fbdf1d29..eff2d65e692 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -10,10 +10,15 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" #include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/numeric/random/engine.hpp" #include #include +#include +#include #include +#include #include using namespace bb; @@ -23,7 +28,8 @@ using HashPolicy = Poseidon2HashPolicy; using Store = CachedTreeStore; using TreeType = IndexedTree; -using IndexedLeafType = IndexedLeaf; +using IndexedNullifierLeafType = IndexedLeaf; +using IndexedPublicDataLeafType = IndexedLeaf; using CompletionCallback = TreeType::AddCompletionCallback; @@ -34,7 +40,7 @@ class PersistedIndexedTreeTest : public testing::Test { // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers _directory = randomTempDirectory(); std::filesystem::create_directories(_directory); - _environment = std::make_unique(_directory, 1, 2, 2); + _environment = std::make_unique(_directory, 1024, 2, 2); } void TearDown() override { std::filesystem::remove_all(_directory); } @@ -46,92 +52,119 @@ class PersistedIndexedTreeTest : public testing::Test { std::string PersistedIndexedTreeTest::_directory; -void check_size(TreeType& tree, index_t expected_size, bool includeUncommitted = true) +template +void check_size(IndexedTree, Poseidon2HashPolicy>& tree, + index_t expected_size, + bool includeUncommitted = true) { - Signal signal(1); - auto completion = [&](const std::string&, uint32_t, const index_t& size, const fr&) -> void { - EXPECT_EQ(size, expected_size); - signal.signal_level(0); + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, true); + EXPECT_EQ(response.inner.size, expected_size); + signal.signal_level(); }; tree.get_meta_data(includeUncommitted, completion); - signal.wait_for_level(0); + signal.wait_for_level(); } -fr get_root(TreeType& tree, bool includeUncommitted = true) +template +fr get_root(IndexedTree, Poseidon2HashPolicy>& tree, + bool includeUncommitted = true) { fr r; - Signal signal(1); - auto completion = [&](const std::string&, uint32_t, const index_t&, const fr& root) -> void { - r = root; - signal.signal_level(0); + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + r = response.inner.root; + signal.signal_level(); }; tree.get_meta_data(includeUncommitted, completion); - signal.wait_for_level(0); + signal.wait_for_level(); return r; } -void check_root(TreeType& tree, fr expected_root, bool includeUncommitted = true) +template +void check_root(IndexedTree, Poseidon2HashPolicy>& tree, + fr expected_root, + bool includeUncommitted = true) { fr root = get_root(tree, includeUncommitted); EXPECT_EQ(root, expected_root); } -fr_hash_path get_hash_path(TreeType& tree, index_t index, bool includeUncommitted = true) +template +fr_hash_path get_hash_path(IndexedTree, Poseidon2HashPolicy>& tree, + index_t index, + bool includeUncommitted = true) { fr_hash_path h; - Signal signal(1); - auto completion = [&](const fr_hash_path& path) -> void { - h = path; - signal.signal_level(0); + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + h = response.inner.path; + signal.signal_level(); }; tree.get_hash_path(index, completion, includeUncommitted); - signal.wait_for_level(0); + signal.wait_for_level(); return h; } -IndexedLeafType get_leaf(TreeType& tree, index_t index, bool includeUncommitted = true) +template +IndexedLeaf get_leaf(IndexedTree, Poseidon2HashPolicy>& tree, + index_t index, + bool includeUncommitted = true) { - IndexedLeafType l; - Signal signal(1); - auto completion = [&](const IndexedLeafType& leaf) -> void { - l = leaf; - signal.signal_level(0); + IndexedLeaf l; + Signal signal; + auto completion = [&](const TypedResponse>& leaf) -> void { + l = leaf.inner.indexed_leaf; + signal.signal_level(); }; tree.get_leaf(index, includeUncommitted, completion); - signal.wait_for_level(0); + signal.wait_for_level(); return l; } -void check_hash_path(TreeType& tree, index_t index, fr_hash_path expected_hash_path, bool includeUncommitted = true) +template +void check_hash_path(IndexedTree, Poseidon2HashPolicy>& tree, + index_t index, + fr_hash_path expected_hash_path, + bool includeUncommitted = true) { fr_hash_path path = get_hash_path(tree, index, includeUncommitted); EXPECT_EQ(path, expected_hash_path); } -void commit_tree(TreeType& tree) +template +void commit_tree(IndexedTree, Poseidon2HashPolicy>& tree) { - Signal signal(1); - auto completion = [&]() -> void { signal.signal_level(0); }; + Signal signal; + auto completion = [&](const Response& response) -> void { + EXPECT_EQ(response.success, true); + signal.signal_level(); + }; tree.commit(completion); - signal.wait_for_level(0); + signal.wait_for_level(); } -void add_value(TreeType& tree, const fr& value) +template +void add_value(IndexedTree, Poseidon2HashPolicy>& tree, + const LeafValueType& value) { - Signal signal(1); - auto completion = [&](const std::vector&, fr&, index_t) -> void { signal.signal_level(0); }; + Signal signal; + auto completion = [&](const TypedResponse&) -> void { signal.signal_level(); }; tree.add_or_update_value(value, completion); - signal.wait_for_level(0); + signal.wait_for_level(); } -void add_values(TreeType& tree, const std::vector& values) +template +void add_values(IndexedTree, Poseidon2HashPolicy>& tree, + const std::vector& values) { - Signal signal(1); - auto completion = [&](const std::vector&, fr&, index_t) -> void { signal.signal_level(0); }; + Signal signal; + auto completion = [&](const TypedResponse&) -> void { signal.signal_level(); }; tree.add_or_update_values(values, completion); - signal.wait_for_level(0); + signal.wait_for_level(); } TEST_F(PersistedIndexedTreeTest, can_create) @@ -142,8 +175,8 @@ TEST_F(PersistedIndexedTreeTest, can_create) EXPECT_NO_THROW(Store store(name, depth, db)); Store store(name, depth, db); ThreadPool workers(1); - TreeType tree = TreeType(store, workers, 1); - check_size(tree, 1); + TreeType tree = TreeType(store, workers, 2); + check_size(tree, 2); NullifierMemoryTree memdb(10); check_root(tree, memdb.root()); @@ -162,24 +195,63 @@ TEST_F(PersistedIndexedTreeTest, can_only_recreate_with_same_name_and_depth) TEST_F(PersistedIndexedTreeTest, test_size) { + index_t current_size = 2; ThreadPool workers(1); constexpr size_t depth = 10; std::string name = randomString(); LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); Store store(name, depth, db); - auto tree = TreeType(store, workers, 1); + auto tree = TreeType(store, workers, current_size); - check_size(tree, 1); + check_size(tree, current_size); // We assume that the first leaf is already filled with (0, 0, 0). for (uint32_t i = 0; i < 4; i++) { - add_value(tree, VALUES[i]); - check_size(tree, i + 2); + add_value(tree, NullifierLeafValue(VALUES[i])); + check_size(tree, ++current_size); } } +TEST_F(PersistedIndexedTreeTest, indexed_tree_must_have_at_least_2_initial_size) +{ + index_t current_size = 1; + ThreadPool workers(1); + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + EXPECT_THROW(TreeType(store, workers, current_size), std::runtime_error); +} + +TEST_F(PersistedIndexedTreeTest, reports_an_error_if_tree_is_overfilled) +{ + index_t current_size = 2; + ThreadPool workers(1); + constexpr size_t depth = 4; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, current_size); + + std::vector values; + for (uint32_t i = 0; i < 14; i++) { + values.emplace_back(VALUES[i]); + } + add_values(tree, values); + + Signal signal; + auto add_completion = [&](const TypedResponse& response) { + EXPECT_EQ(response.success, false); + EXPECT_EQ(response.message, "Tree is full"); + signal.signal_level(); + }; + tree.add_or_update_value(NullifierLeafValue(VALUES[16]), add_completion); + signal.wait_for_level(); +} + TEST_F(PersistedIndexedTreeTest, test_get_hash_path) { + index_t current_size = 2; NullifierMemoryTree memdb(10); ThreadPool workers(1); @@ -187,16 +259,16 @@ TEST_F(PersistedIndexedTreeTest, test_get_hash_path) std::string name = randomString(); LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); Store store(name, depth, db); - auto tree = TreeType(store, workers, 1); + auto tree = TreeType(store, workers, current_size); - check_size(tree, 1); + check_size(tree, current_size); check_root(tree, memdb.root()); check_hash_path(tree, 0, memdb.get_hash_path(0)); memdb.update_element(VALUES[512]); - add_value(tree, VALUES[512]); + add_value(tree, NullifierLeafValue(VALUES[512])); - check_size(tree, 2); + check_size(tree, ++current_size); check_hash_path(tree, 0, memdb.get_hash_path(0)); check_hash_path(tree, 1, memdb.get_hash_path(1)); @@ -204,9 +276,9 @@ TEST_F(PersistedIndexedTreeTest, test_get_hash_path) for (uint32_t i = 0; i < num_to_append; ++i) { memdb.update_element(VALUES[i]); - add_value(tree, VALUES[i]); + add_value(tree, NullifierLeafValue(VALUES[i])); } - check_size(tree, num_to_append + 2); + check_size(tree, num_to_append + current_size); check_hash_path(tree, 0, memdb.get_hash_path(0)); check_hash_path(tree, 512, memdb.get_hash_path(512)); } @@ -214,7 +286,7 @@ TEST_F(PersistedIndexedTreeTest, test_get_hash_path) TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) { NullifierMemoryTree memdb(10); - + index_t current_size = 2; ThreadPool workers(1); constexpr size_t depth = 10; std::string name = randomString(); @@ -222,16 +294,16 @@ TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) { LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); Store store(name, depth, db); - auto tree = TreeType(store, workers, 1); + auto tree = TreeType(store, workers, current_size); - check_size(tree, 1); + check_size(tree, current_size); check_root(tree, memdb.root()); check_hash_path(tree, 0, memdb.get_hash_path(0)); - add_value(tree, VALUES[512]); + add_value(tree, NullifierLeafValue(VALUES[512])); // Committed data should not have changed - check_size(tree, 1, false); + check_size(tree, current_size, false); check_root(tree, memdb.root(), false); check_hash_path(tree, 0, memdb.get_hash_path(0), false); check_hash_path(tree, 1, memdb.get_hash_path(1), false); @@ -239,7 +311,7 @@ TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) memdb.update_element(VALUES[512]); // Uncommitted data should have changed - check_size(tree, 2, true); + check_size(tree, current_size + 1, true); check_root(tree, memdb.root(), true); check_hash_path(tree, 0, memdb.get_hash_path(0), true); check_hash_path(tree, 1, memdb.get_hash_path(1), true); @@ -248,7 +320,7 @@ TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) commit_tree(tree); // Now committed data should have changed - check_size(tree, 2, false); + check_size(tree, ++current_size, false); check_root(tree, memdb.root(), false); check_hash_path(tree, 0, memdb.get_hash_path(0), false); check_hash_path(tree, 1, memdb.get_hash_path(1), false); @@ -258,15 +330,15 @@ TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) { LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); Store store(name, depth, db); - auto tree = TreeType(store, workers, 1); + auto tree = TreeType(store, workers, current_size); // check uncommitted state - check_size(tree, 2); + check_size(tree, current_size); check_root(tree, memdb.root()); check_hash_path(tree, 0, memdb.get_hash_path(0)); // check committed state - check_size(tree, 2, false); + check_size(tree, current_size, false); check_root(tree, memdb.root(), false); check_hash_path(tree, 0, memdb.get_hash_path(0), false); } @@ -309,25 +381,25 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) fr_hash_path path = memdb.update_element(batch[j].value); memory_tree_hash_paths.push_back(path); } - std::vector tree1_hash_paths; - std::vector tree2_hash_paths; + std::shared_ptr> tree1_hash_paths; + std::shared_ptr> tree2_hash_paths; { - Signal signal(1); - CompletionCallback completion = [&](std::vector& hash_paths, fr&, index_t) { - tree1_hash_paths = hash_paths; - signal.signal_level(0); + Signal signal; + CompletionCallback completion = [&](const TypedResponse& response) { + tree1_hash_paths = response.inner.paths; + signal.signal_level(); }; tree1.add_or_update_values(batch, completion); - signal.wait_for_level(0); + signal.wait_for_level(); } { - Signal signal(1); - CompletionCallback completion = [&](std::vector& hash_paths, fr&, index_t) { - tree2_hash_paths = hash_paths; - signal.signal_level(0); + Signal signal; + CompletionCallback completion = [&](const TypedResponse& response) { + tree2_hash_paths = response.inner.paths; + signal.signal_level(); }; tree2.add_or_update_values(batch, completion); - signal.wait_for_level(0); + signal.wait_for_level(); } check_root(tree1, memdb.root()); check_root(tree2, memdb.root()); @@ -339,17 +411,43 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) check_hash_path(tree2, 512, memdb.get_hash_path(512)); for (uint32_t j = 0; j < batch_size; j++) { - EXPECT_EQ(tree1_hash_paths[j], tree2_hash_paths[j]); + EXPECT_EQ(tree1_hash_paths->at(j), tree2_hash_paths->at(j)); } } } -fr hash_leaf(const IndexedLeafType& leaf) +TEST_F(PersistedIndexedTreeTest, reports_an_error_if_batch_contains_duplicate) +{ + index_t current_size = 2; + ThreadPool workers(1); + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, current_size); + + std::vector values; + for (uint32_t i = 0; i < 16; i++) { + values.emplace_back(VALUES[i]); + } + values[8] = values[0]; + + Signal signal; + auto add_completion = [&](const TypedResponse& response) { + EXPECT_EQ(response.success, false); + EXPECT_EQ(response.message, "Duplicate key not allowed in same batch"); + signal.signal_level(); + }; + tree.add_or_update_values(values, add_completion); + signal.wait_for_level(); +} + +template fr hash_leaf(const IndexedLeaf& leaf) { return HashPolicy::hash(leaf.get_hash_inputs()); } -bool verify_hash_path(TreeType& tree, const IndexedLeafType& leaf_value, const uint32_t idx) +bool verify_hash_path(TreeType& tree, const IndexedNullifierLeafType& leaf_value, const uint32_t idx) { fr root = get_root(tree, true); fr_hash_path path = get_hash_path(tree, idx, true); @@ -365,78 +463,92 @@ bool verify_hash_path(TreeType& tree, const IndexedLeafType& leaf_value, const u return current == root; } -IndexedLeafType create_indexed_nullifier_leaf(const fr& value, index_t nextIndex, const fr& nextValue) +IndexedNullifierLeafType create_indexed_nullifier_leaf(const fr& value, index_t nextIndex, const fr& nextValue) { - return IndexedLeafType(NullifierLeafValue(value), nextIndex, nextValue); + return IndexedNullifierLeafType{ NullifierLeafValue(value), nextIndex, nextValue }; +} + +IndexedPublicDataLeafType create_indexed_public_data_leaf(const fr& slot, + const fr& value, + index_t nextIndex, + const fr& nextValue) +{ + return IndexedPublicDataLeafType{ PublicDataLeafValue(slot, value), nextIndex, nextValue }; } TEST_F(PersistedIndexedTreeTest, test_indexed_memory) { + index_t current_size = 2; ThreadPool workers(8); // Create a depth-3 indexed merkle tree constexpr size_t depth = 3; std::string name = randomString(); LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); Store store(name, depth, db); - auto tree = TreeType(store, workers, 1); + auto tree = TreeType(store, workers, current_size); /** * Intial state: * * index 0 1 2 3 4 5 6 7 * --------------------------------------------------------------------- - * val 0 0 0 0 0 0 0 0 - * nextIdx 0 0 0 0 0 0 0 0 + * val 1 1 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 * nextVal 0 0 0 0 0 0 0 0 */ - IndexedLeafType zero_leaf(NullifierLeafValue(0), 0, 0); - check_size(tree, 1); + IndexedNullifierLeafType zero_leaf(NullifierLeafValue(0), 1, 1); + IndexedNullifierLeafType one_leaf(NullifierLeafValue(1), 0, 0); + check_size(tree, current_size); EXPECT_EQ(get_leaf(tree, 0), zero_leaf); + EXPECT_EQ(get_leaf(tree, 1), one_leaf); /** * Add new value 30: * * index 0 1 2 3 4 5 6 7 * --------------------------------------------------------------------- - * val 0 30 0 0 0 0 0 0 - * nextIdx 1 0 0 0 0 0 0 0 - * nextVal 30 0 0 0 0 0 0 0 + * val 0 1 30 0 0 0 0 0 + * nextIdx 1 2 0 0 0 0 0 0 + * nextVal 1 30 0 0 0 0 0 0 */ - add_value(tree, 30); - check_size(tree, 2); - EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(create_indexed_nullifier_leaf(0, 1, 30))); - EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf(create_indexed_nullifier_leaf(30, 0, 0))); + add_value(tree, NullifierLeafValue(30)); + check_size(tree, ++current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_nullifier_leaf(0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_nullifier_leaf(1, 2, 30)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_nullifier_leaf(30, 0, 0)); /** * Add new value 10: * * index 0 1 2 3 4 5 6 7 * --------------------------------------------------------------------- - * val 0 30 10 0 0 0 0 0 - * nextIdx 2 0 1 0 0 0 0 0 - * nextVal 10 0 30 0 0 0 0 0 + * val 0 1 30 10 0 0 0 0 + * nextIdx 1 3 0 2 0 0 0 0 + * nextVal 1 10 0 30 0 0 0 0 */ - add_value(tree, 10); - check_size(tree, 3); - EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(create_indexed_nullifier_leaf(0, 2, 10))); - EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf(create_indexed_nullifier_leaf(30, 0, 0))); - EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf(create_indexed_nullifier_leaf(10, 1, 30))); + add_value(tree, NullifierLeafValue(10)); + check_size(tree, ++current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_nullifier_leaf(0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_nullifier_leaf(1, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_nullifier_leaf(30, 0, 0)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_nullifier_leaf(10, 2, 30)); /** * Add new value 20: * * index 0 1 2 3 4 5 6 7 * --------------------------------------------------------------------- - * val 0 30 10 20 0 0 0 0 - * nextIdx 2 0 3 1 0 0 0 0 - * nextVal 10 0 20 30 0 0 0 0 + * val 0 1 30 10 20 0 0 0 + * nextIdx 1 3 0 4 2 0 0 0 + * nextVal 1 10 0 20 30 0 0 0 */ - add_value(tree, 20); - check_size(tree, 4); - EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(create_indexed_nullifier_leaf(0, 2, 10))); - EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf(create_indexed_nullifier_leaf(30, 0, 0))); - EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf(create_indexed_nullifier_leaf(10, 3, 20))); - EXPECT_EQ(hash_leaf(get_leaf(tree, 3)), hash_leaf(create_indexed_nullifier_leaf(20, 1, 30))); + add_value(tree, NullifierLeafValue(20)); + check_size(tree, ++current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_nullifier_leaf(0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_nullifier_leaf(1, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_nullifier_leaf(30, 0, 0)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_nullifier_leaf(10, 4, 20)); + EXPECT_EQ(get_leaf(tree, 4), create_indexed_nullifier_leaf(20, 2, 30)); // Adding the same value must not affect anything // tree.update_element(20); @@ -451,17 +563,18 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) * * index 0 1 2 3 4 5 6 7 * --------------------------------------------------------------------- - * val 0 30 10 20 50 0 0 0 - * nextIdx 2 4 3 1 0 0 0 0 - * nextVal 10 50 20 30 0 0 0 0 + * val 0 1 30 10 20 50 0 0 + * nextIdx 1 3 5 4 2 0 0 0 + * nextVal 1 10 50 20 30 0 0 0 */ - add_value(tree, 50); - check_size(tree, 5); - EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(create_indexed_nullifier_leaf(0, 2, 10))); - EXPECT_EQ(hash_leaf(get_leaf(tree, 1)), hash_leaf(create_indexed_nullifier_leaf(30, 4, 50))); - EXPECT_EQ(hash_leaf(get_leaf(tree, 2)), hash_leaf(create_indexed_nullifier_leaf(10, 3, 20))); - EXPECT_EQ(hash_leaf(get_leaf(tree, 3)), hash_leaf(create_indexed_nullifier_leaf(20, 1, 30))); - EXPECT_EQ(hash_leaf(get_leaf(tree, 4)), hash_leaf(create_indexed_nullifier_leaf(50, 0, 0))); + add_value(tree, NullifierLeafValue(50)); + check_size(tree, ++current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_nullifier_leaf(0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_nullifier_leaf(1, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_nullifier_leaf(30, 5, 50)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_nullifier_leaf(10, 4, 20)); + EXPECT_EQ(get_leaf(tree, 4), create_indexed_nullifier_leaf(20, 2, 30)); + EXPECT_EQ(get_leaf(tree, 5), create_indexed_nullifier_leaf(50, 0, 0)); // Manually compute the node values auto e000 = hash_leaf(get_leaf(tree, 0)); @@ -469,7 +582,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) auto e010 = hash_leaf(get_leaf(tree, 2)); auto e011 = hash_leaf(get_leaf(tree, 3)); auto e100 = hash_leaf(get_leaf(tree, 4)); - auto e101 = fr::zero(); + auto e101 = hash_leaf(get_leaf(tree, 5)); auto e110 = fr::zero(); auto e111 = fr::zero(); @@ -512,22 +625,24 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) TEST_F(PersistedIndexedTreeTest, test_indexed_tree) { + index_t current_size = 2; ThreadPool workers(1); // Create a depth-8 indexed merkle tree constexpr uint32_t depth = 8; std::string name = randomString(); LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); Store store(name, depth, db); - auto tree = TreeType(store, workers, 1); + auto tree = TreeType(store, workers, current_size); - IndexedLeafType zero_leaf = create_indexed_nullifier_leaf(0, 0, 0); - check_size(tree, 1); + IndexedNullifierLeafType zero_leaf = create_indexed_nullifier_leaf(0, 1, 1); + check_size(tree, current_size); EXPECT_EQ(hash_leaf(get_leaf(tree, 0)), hash_leaf(zero_leaf)); // Add 20 random values to the tree for (uint32_t i = 0; i < 20; i++) { auto value = fr::random_element(); - add_value(tree, value); + add_value(tree, NullifierLeafValue(value)); + ++current_size; } auto abs_diff = [](uint256_t a, uint256_t b) { @@ -538,14 +653,14 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_tree) } }; - check_size(tree, 21); + check_size(tree, current_size); // Check if a new random value is not a member of this tree. fr new_member = fr::random_element(); std::vector differences; for (uint32_t i = 0; i < uint32_t(21); i++) { - uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(get_leaf(tree, i).value.get_fr_value())); - uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(get_leaf(tree, i).value.get_fr_value())); + uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(get_leaf(tree, i).value.get_key())); + uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(get_leaf(tree, i).value.get_key())); differences.push_back(diff_hi + diff_lo); } auto it = std::min_element(differences.begin(), differences.end()); @@ -571,27 +686,175 @@ TEST_F(PersistedIndexedTreeTest, can_add_single_whilst_reading) LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); Store store(name, depth, db); ThreadPool pool(8); - TreeType tree(store, pool, 1); + TreeType tree(store, pool, 2); - check_size(tree, 1); + check_size(tree, 2); Signal signal(2); - auto add_completion = [&](const std::vector&, fr&, index_t) { + auto add_completion = [&](const TypedResponse&) { signal.signal_level(1); - auto commit_completion = [&]() { signal.signal_level(0); }; + auto commit_completion = [&](const Response&) { signal.signal_level(); }; tree.commit(commit_completion); }; tree.add_or_update_value(VALUES[0], add_completion); for (size_t i = 0; i < num_reads; i++) { - auto completion = [&, i](const fr_hash_path& path) { paths[i] = path; }; + auto completion = [&, i](const TypedResponse& response) { + paths[i] = response.inner.path; + }; tree.get_hash_path(0, completion, false); } - signal.wait_for_level(0); + signal.wait_for_level(); } for (auto& path : paths) { EXPECT_TRUE(path == initial_path || path == final_hash_path); } +} + +TEST_F(PersistedIndexedTreeTest, test_indexed_memory_with_public_data_writes) +{ + index_t current_size = 2; + ThreadPool workers(8); + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + CachedTreeStore store(name, depth, db); + auto tree = + IndexedTree, Poseidon2HashPolicy>(store, workers, current_size); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 0 0 0 0 0 0 + * val 0 0 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 1 0 0 0 0 0 0 0 + */ + IndexedPublicDataLeafType zero_leaf = create_indexed_public_data_leaf(0, 0, 1, 1); + IndexedPublicDataLeafType one_leaf = create_indexed_public_data_leaf(1, 0, 0, 0); + check_size(tree, current_size); + EXPECT_EQ(get_leaf(tree, 0), zero_leaf); + EXPECT_EQ(get_leaf(tree, 1), one_leaf); + + /** + * Add new slot:value 30:5: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 0 0 0 0 0 + * val 0 0 5 0 0 0 0 0 + * nextIdx 1 2 0 0 0 0 0 0 + * nextVal 1 30 0 0 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(30, 5)); + check_size(tree, ++current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 2, 30)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 5, 0, 0)); + + /** + * Add new slot:value 10:20: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 10 0 0 0 0 + * val 0 0 5 20 0 0 0 0 + * nextIdx 1 3 0 2 0 0 0 0 + * nextVal 1 10 0 30 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(10, 20)); + check_size(tree, ++current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 5, 0, 0)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_public_data_leaf(10, 20, 2, 30)); + + /** + * Update value at slot 30 to 6: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 10 0 0 0 0 + * val 0 0 6 20 0 0 0 0 + * nextIdx 1 3 0 2 0 0 0 0 + * nextVal 1 10 0 30 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(30, 6)); + // The size still increases as we pad with an empty leaf + check_size(tree, ++current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 6, 0, 0)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_public_data_leaf(10, 20, 2, 30)); + EXPECT_EQ(get_leaf(tree, 4), create_indexed_public_data_leaf(0, 0, 0, 0)); + + /** + * Add new value slot:value 50:8: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 10 0 50 0 0 + * val 0 0 6 20 0 8 0 0 + * nextIdx 1 3 5 2 0 0 0 0 + * nextVal 1 10 50 30 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(50, 8)); + check_size(tree, ++current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 6, 5, 50)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_public_data_leaf(10, 20, 2, 30)); + EXPECT_EQ(get_leaf(tree, 4), create_indexed_public_data_leaf(0, 0, 0, 0)); + EXPECT_EQ(get_leaf(tree, 5), create_indexed_public_data_leaf(50, 8, 0, 0)); + + // Manually compute the node values + auto e000 = hash_leaf(get_leaf(tree, 0)); + auto e001 = hash_leaf(get_leaf(tree, 1)); + auto e010 = hash_leaf(get_leaf(tree, 2)); + auto e011 = hash_leaf(get_leaf(tree, 3)); + auto e100 = hash_leaf(get_leaf(tree, 4)); + auto e101 = hash_leaf(get_leaf(tree, 5)); + auto e110 = fr::zero(); + auto e111 = fr::zero(); + + auto e00 = HashPolicy::hash_pair(e000, e001); + auto e01 = HashPolicy::hash_pair(e010, e011); + auto e10 = HashPolicy::hash_pair(e100, e101); + auto e11 = HashPolicy::hash_pair(e110, e111); + + auto e0 = HashPolicy::hash_pair(e00, e01); + auto e1 = HashPolicy::hash_pair(e10, e11); + auto root = HashPolicy::hash_pair(e0, e1); + + // Check the hash path at index 2 and 3 + // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) + fr_hash_path expected = { + std::make_pair(e000, e001), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + check_hash_path(tree, 0, expected); + check_hash_path(tree, 1, expected); + expected = { + std::make_pair(e010, e011), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + check_hash_path(tree, 2, expected); + check_hash_path(tree, 3, expected); + check_root(tree, root); + + // Check the hash path at index 6 and 7 + expected = { + std::make_pair(e110, e111), + std::make_pair(e10, e11), + std::make_pair(e0, e1), + }; + check_hash_path(tree, 6, expected); + check_hash_path(tree, 7, expected); } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp index 8e11f658db7..5f586ea67d8 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp @@ -1,4 +1,7 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/numeric/uint256/uint256.hpp" +#include +#include #include namespace bb::crypto::merkle_tree { @@ -20,9 +23,9 @@ int SizeCmp(const MDB_val* a, const MDB_val* b) return 0; } -int Invalid(const MDB_val*, const MDB_val*) +int MemCmp(const MDB_val* a, const MDB_val* b) { - throw std::runtime_error("Invalid comparison"); + return std::memcmp(static_cast(a->mv_data), static_cast(b->mv_data), a->mv_size); } int IntegerKeyCmp(const MDB_val* a, const MDB_val* b) @@ -30,11 +33,20 @@ int IntegerKeyCmp(const MDB_val* a, const MDB_val* b) if (a->mv_size != b->mv_size) { return SizeCmp(a, b); } + uint64_t factor = a->mv_size / sizeof(uint64_t); + uint64_t remainder = a->mv_size % sizeof(uint64_t); - using f_type = std::function; - static std::vector functions{ - ValueCmp, ValueCmp, ValueCmp, Invalid, ValueCmp + if (a->mv_size > sizeof(uint256_t)) { + return MemCmp(a, b); + } + if (a->mv_size > 1 && remainder != 0) { + return MemCmp(a, b); + } + + using f = std::function; + static std::vector functions{ + ValueCmp, ValueCmp, ValueCmp, MemCmp, ValueCmp }; - return functions[a->mv_size / 8](a, b); + return functions[factor](a, b); } } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp index 3f59104f4e5..eb8f6fb4a8c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp @@ -14,7 +14,7 @@ void ThrowError(const std::string& errorString, int error); int SizeCmp(const MDB_val* a, const MDB_val* b); -int Invalid(const MDB_val*, const MDB_val*); +int MemCmp(const MDB_val*, const MDB_val*); template int ValueCmp(const MDB_val* a, const MDB_val* b) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp index 61bc88c5216..437ca66a926 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp @@ -1,10 +1,11 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include #include namespace bb::crypto::merkle_tree { LMDBEnvironment::LMDBEnvironment(const std::string& directory, - uint64_t mapSizeMB, + uint64_t mapSizeKB, uint32_t maxNumDBs, uint32_t maxNumReaders) : _maxReaders(maxNumReaders) @@ -12,15 +13,21 @@ LMDBEnvironment::LMDBEnvironment(const std::string& directory, { call_lmdb_func("mdb_env_create", mdb_env_create, &_mdbEnv); uint64_t kb = 1024; - uint64_t totalMapSize = kb * kb * mapSizeMB; - call_lmdb_func(mdb_env_set_mapsize, _mdbEnv, totalMapSize); - call_lmdb_func(mdb_env_set_maxdbs, _mdbEnv, static_cast(maxNumDBs)); - call_lmdb_func(mdb_env_set_maxreaders, _mdbEnv, maxNumReaders); + uint64_t totalMapSize = kb * mapSizeKB; uint32_t flags = MDB_NOTLS; - if (!call_lmdb_func( - mdb_env_open, _mdbEnv, directory.c_str(), flags, static_cast(S_IRWXU | S_IRWXG | S_IRWXO))) { + try { + call_lmdb_func("mdb_env_set_mapsize", mdb_env_set_mapsize, _mdbEnv, totalMapSize); + call_lmdb_func("mdb_env_set_maxdbs", mdb_env_set_maxdbs, _mdbEnv, static_cast(maxNumDBs)); + call_lmdb_func("mdb_env_set_maxreaders", mdb_env_set_maxreaders, _mdbEnv, maxNumReaders); + call_lmdb_func("mdb_env_open", + mdb_env_open, + _mdbEnv, + directory.c_str(), + flags, + static_cast(S_IRWXU | S_IRWXG | S_IRWXO)); + } catch (std::runtime_error& error) { call_lmdb_func(mdb_env_close, _mdbEnv); - // throw here + throw error; } } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp index 48eda657a2a..f4c1f826fe9 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp @@ -7,7 +7,7 @@ namespace bb::crypto::merkle_tree { class LMDBEnvironment { public: - LMDBEnvironment(const std::string& directory, uint64_t mapSizeMB, uint32_t maxNumDBs, uint32_t maxNumReaders); + LMDBEnvironment(const std::string& directory, uint64_t mapSizeKb, uint32_t maxNumDBs, uint32_t maxNumReaders); LMDBEnvironment(const LMDBEnvironment& other) = delete; LMDBEnvironment(LMDBEnvironment&& other) = delete; LMDBEnvironment& operator=(const LMDBEnvironment& other) = delete; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp index a36709bb005..5c8a492f48b 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "barretenberg/common/serialize.hpp" @@ -33,7 +34,7 @@ class LMDBStoreTest : public testing::Test { // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers _directory = randomTempDirectory(); std::filesystem::create_directories(_directory); - _environment = std::make_unique(_directory, 1, 2, 2); + _environment = std::make_unique(_directory, 1024, 2, 2); } void TearDown() override { std::filesystem::remove_all(_directory); } @@ -169,16 +170,15 @@ TEST_F(LMDBStoreTest, can_write_and_read_multiple) uint32_t num_reads = 128; { - + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); for (size_t i = 0; i < num_reads; i++) { - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); std::vector key; write(key, VALUES[i]); std::vector buf; write(buf, VALUES[i + 128]); transaction->put_value(key, buf); - transaction->commit(); } + transaction->commit(); } { @@ -196,6 +196,21 @@ TEST_F(LMDBStoreTest, can_write_and_read_multiple) } } +TEST_F(LMDBStoreTest, throws_if_write_transaction_is_reused) +{ + { + LMDBStore store(*_environment, "DB1"); + { + std::vector buf; + write(buf, VALUES[0]); + LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + transaction->put_node(0, 0, buf); + transaction->commit(); + EXPECT_THROW(transaction->put_node(0, 1, buf), std::runtime_error); + } + } +} + TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp index c12385a7d35..dc29f8baaa8 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp @@ -18,6 +18,10 @@ MDB_txn* LMDBTransaction::underlying() const void LMDBTransaction::abort() { + if (state != TransactionState::OPEN) { + return; + } call_lmdb_func(mdb_txn_abort, _transaction); + state = TransactionState::ABORTED; } } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp index 4c7cd76f6cf..5fead3690cf 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp @@ -11,7 +11,7 @@ LMDBWriteTransaction::LMDBWriteTransaction(LMDBEnvironment& env, const LMDBDatab LMDBWriteTransaction::~LMDBWriteTransaction() { - tryAbort(); + try_abort(); } void LMDBWriteTransaction::commit() @@ -23,13 +23,12 @@ void LMDBWriteTransaction::commit() state = TransactionState::COMMITTED; } -void LMDBWriteTransaction::tryAbort() +void LMDBWriteTransaction::try_abort() { if (state != TransactionState::OPEN) { return; } LMDBTransaction::abort(); - state = TransactionState::ABORTED; } void LMDBWriteTransaction::put_node(uint32_t level, index_t index, std::vector& data) @@ -47,6 +46,6 @@ void LMDBWriteTransaction::put_value(std::vector& key, std::vector CachedTreeStore::find_lo bool includeUncommitted, ReadTransaction& tx) const { - uint256_t new_value_as_number = uint256_t(new_value.get_fr_value()); + uint256_t new_value_as_number = uint256_t(new_value.get_key()); std::vector data; - FrKeyType key(new_value.get_fr_value()); + FrKeyType key(new_value.get_key()); tx.get_value_or_previous(key, data); auto db_index = from_buffer(data, 0); uint256_t retrieved_value = key; @@ -164,7 +164,7 @@ void CachedTreeStore::set_at_index(const index_t& { leaves_[index] = leaf; if (add_to_index) { - indices_[uint256_t(leaf.value.get_fr_value())] = index; + indices_[uint256_t(leaf.value.get_key())] = index; } } @@ -241,29 +241,34 @@ template void CachedTreeStore< { { WriteTransactionPtr tx = createWriteTransaction(); - for (uint32_t i = 1; i < nodes.size(); i++) { - auto& level = nodes[i]; - for (auto& item : level) { - index_t index = item.first; - std::vector& data = item.second; - tx->put_node(i, index, data); + try { + for (uint32_t i = 1; i < nodes.size(); i++) { + auto& level = nodes[i]; + for (auto& item : level) { + index_t index = item.first; + std::vector& data = item.second; + tx->put_node(i, index, data); + } } + for (auto& idx : indices_) { + std::vector value; + write(value, idx.second); + FrKeyType key = idx.first; + tx->put_value_by_integer(key, value); + } + for (const auto& leaf : leaves_) { + msgpack::sbuffer buffer; + msgpack::pack(buffer, leaf.second); + std::vector value(buffer.data(), buffer.data() + buffer.size()); + LeafIndexKeyType key = leaf.first; + tx->put_value_by_integer(key, value); + } + persistMeta(meta, *tx); + tx->commit(); + } catch (std::exception& e) { + tx->try_abort(); + throw; } - for (auto& idx : indices_) { - std::vector value; - write(value, idx.second); - FrKeyType key = idx.first; - tx->put_value_by_integer(key, value); - } - for (const auto& leaf : leaves_) { - msgpack::sbuffer buffer; - msgpack::pack(buffer, leaf.second); - std::vector value(buffer.data(), buffer.data() + buffer.size()); - LeafIndexKeyType key = leaf.first; - tx->put_value_by_integer(key, value); - } - persistMeta(meta, *tx); - tx->commit(); } rollback(); } @@ -318,8 +323,13 @@ void CachedTreeStore::initialise() meta.size = 0; meta.depth = depth; WriteTransactionPtr tx = createWriteTransaction(); - persistMeta(meta, *tx); - tx->commit(); + try { + persistMeta(meta, *tx); + tx->commit(); + } catch (std::exception& e) { + tx->try_abort(); + throw e; + } } } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp index b8f44e91463..e2b4b1b370c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -66,7 +66,7 @@ namespace bb::crypto::merkle_tree { template class NullifierMemoryTree : public MemoryTree { public: - NullifierMemoryTree(size_t depth, size_t initial_size = 1); + NullifierMemoryTree(size_t depth, size_t initial_size = 2); using MemoryTree::get_hash_path; using MemoryTree::root; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.cpp new file mode 100644 index 00000000000..5640b000f75 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.cpp @@ -0,0 +1,3 @@ +#include "barretenberg/crypto/merkle_tree/response.hpp" + +namespace bb::crypto::merkle_tree {} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp new file mode 100644 index 00000000000..8cd6fafce96 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include +#include +#include + +namespace bb::crypto::merkle_tree { +struct TreeMetaResponse { + std::string name; + uint32_t depth; + index_t size; + fr root; +}; + +struct AddDataResponse { + index_t size; + fr root; +}; + +struct GetHashPathResponse { + fr_hash_path path; +}; + +struct AddIndexedDataResponse { + AddDataResponse add_data_result; + std::shared_ptr> paths; +}; + +template struct GetIndexedLeafResponse { + IndexedLeaf indexed_leaf; +}; + +template struct TypedResponse { + ResponseType inner; + bool success{ true }; + std::string message; +}; + +struct Response { + bool success; + std::string message; +}; + +template +void ExecuteAndReport(const std::function&)>& f, + const std::function&)>& on_completion) +{ + TypedResponse response; + try { + f(response); + } catch (std::exception& e) { + response.success = false; + response.message = e.what(); + } + try { + on_completion(response); + } catch (std::exception&) { + } +} + +inline void ExecuteAndReport(const std::function& f, const std::function& on_completion) +{ + Response response; + try { + f(); + } catch (std::exception& e) { + response.success = false; + response.message = e.what(); + } + try { + on_completion(response); + } catch (std::exception&) { + } +} +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp new file mode 100644 index 00000000000..97afff9e869 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp @@ -0,0 +1,57 @@ +#pragma once +#include +#include + +namespace bb::crypto::merkle_tree { +/** + * @brief Used in parallel insertions in the the IndexedTree. Workers signal to other following workes as they move up + * the level of the tree. + * + */ +class Signal { + public: + Signal(uint32_t initial_level = 1) + : signal_(initial_level){}; + ~Signal() = default; + Signal(const Signal& other) + : signal_(other.signal_.load()) + {} + Signal(const Signal&& other) = delete; + Signal& operator=(const Signal& other) + { + if (this != &other) { + signal_.store(other.signal_.load()); + } + return *this; + } + Signal& operator=(const Signal&& other) = delete; + + /** + * @brief Causes the thread to wait until the required level has been signalled + * @param level The required level + * + */ + void wait_for_level(uint32_t level = 0) + { + uint32_t current_level = signal_.load(); + while (current_level > level) { + signal_.wait(current_level); + current_level = signal_.load(); + } + } + + /** + * @brief Signals that the given level has been passed + * @param level The level to be signalled + * + */ + void signal_level(uint32_t level = 0) + { + signal_.store(level); + signal_.notify_all(); + } + + private: + std::atomic signal_; +}; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp index 0c93cfd2954..9cf32fafff0 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -1,8 +1,6 @@ #pragma once -#include "barretenberg/ecc/curves/bn254/fr.hpp" #include -#include namespace bb::crypto::merkle_tree { using index_t = uint64_t; } // namespace bb::crypto::merkle_tree \ No newline at end of file From 1e425a55a83d73826a95e23e5bee214aecc76380 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 3 Jul 2024 17:28:38 +0100 Subject: [PATCH 29/63] Use sibling paths --- .../append_only_tree.bench.cpp | 61 +++++- .../src/barretenberg/common/thread_pool.hpp | 1 + .../append_only_tree/append_only_tree.hpp | 24 +-- .../append_only_tree.test.cpp | 55 ++--- .../merkle_tree/indexed_tree/indexed_tree.hpp | 24 +-- .../indexed_tree/indexed_tree.test.cpp | 204 ++++++++++-------- .../nullifier_tree/nullifier_memory_tree.hpp | 12 +- .../crypto/merkle_tree/response.cpp | 3 - .../crypto/merkle_tree/response.hpp | 6 +- 9 files changed, 233 insertions(+), 157 deletions(-) delete mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.cpp diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp index 6a4f6954d83..811eeecaea0 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp @@ -11,6 +11,7 @@ #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/numeric/random/engine.hpp" #include +#include #include using namespace benchmark; @@ -22,7 +23,7 @@ using Pedersen = AppendOnlyTree; using Poseidon2 = AppendOnlyTree; const size_t TREE_DEPTH = 32; -const size_t MAX_BATCH_SIZE = 128; +// const size_t MAX_BATCH_SIZE = 128; template void perform_batch_insert(TreeType& tree, const std::vector& values) { @@ -69,15 +70,63 @@ template void append_only_tree_bench(State& state) noexcept std::filesystem::remove_all(directory); } -BENCHMARK(append_only_tree_bench) + +template void hash_subtree(std::vector& values, uint32_t offset, uint32_t size) noexcept +{ + uint32_t current = offset; + while (current < offset + size) { + HashPolicy::hash_pair(values[current], values[current + 1]); + current += 2; + } +} + +template void hash_values(State& state) noexcept +{ + + const size_t batch_size = size_t(1024); + size_t num_threads = static_cast(state.range(0)); + uint32_t per_thread = batch_size / num_threads; + for (auto _ : state) { + state.PauseTiming(); + std::vector values(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + ThreadPool pool(num_threads); + state.ResumeTiming(); + if (num_threads > 1) { + for (uint32_t i = 0; i < num_threads; i++) { + uint32_t current = 0; + pool.enqueue([&]() { hash_subtree(values, current, per_thread); }); + current += per_thread; + } + pool.wait(); + } else { + + hash_subtree(values, 0, batch_size); + } + } +} +// BENCHMARK(append_only_tree_bench) +// ->Unit(benchmark::kMillisecond) +// ->RangeMultiplier(2) +// ->Range(2, MAX_BATCH_SIZE) +// ->Iterations(100); +// BENCHMARK(append_only_tree_bench) +// ->Unit(benchmark::kMillisecond) +// ->RangeMultiplier(2) +// ->Range(2, MAX_BATCH_SIZE) +// ->Iterations(1000); + +BENCHMARK(hash_values) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) - ->Range(2, MAX_BATCH_SIZE) + ->Range(1, 8) ->Iterations(100); -BENCHMARK(append_only_tree_bench) +BENCHMARK(hash_values) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) - ->Range(2, MAX_BATCH_SIZE) - ->Iterations(1000); + ->Range(1, 8) + ->Iterations(100); BENCHMARK_MAIN(); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp b/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp index bc8660238b4..017c4ab1f6f 100644 --- a/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp +++ b/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp @@ -19,6 +19,7 @@ class ThreadPool { void enqueue(const std::function& task); void wait(); + size_t num_threads() { return workers.size(); }; private: std::vector workers; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 7840e18a386..f21adb3b6f6 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -26,7 +26,7 @@ template class AppendOnlyTree { public: using AppendCompletionCallback = std::function&)>; using MetaDataCallback = std::function&)>; - using HashPathCallback = std::function&)>; + using HashPathCallback = std::function&)>; using CommitCallback = std::function; using RollbackCallback = std::function; @@ -48,9 +48,9 @@ template class AppendOnlyTree { virtual void add_values(const std::vector& values, const AppendCompletionCallback& on_completion); /** - * @brief Returns the hash path from the leaf at the given index to the root + * @brief Returns the sibling path from the leaf at the given index to the root */ - void get_hash_path(const index_t& index, const HashPathCallback& on_completion, bool includeUncommitted) const; + void get_sibling_path(const index_t& index, const HashPathCallback& on_completion, bool includeUncommitted) const; void get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion); @@ -123,22 +123,20 @@ void AppendOnlyTree::get_meta_data(bool includeUncommitted } template -void AppendOnlyTree::get_hash_path(const index_t& index, - const HashPathCallback& on_completion, - bool includeUncommitted) const +void AppendOnlyTree::get_sibling_path(const index_t& index, + const HashPathCallback& on_completion, + bool includeUncommitted) const { auto job = [=]() { - ExecuteAndReport( - [=](TypedResponse& response) { + ExecuteAndReport( + [=](TypedResponse& response) { index_t current_index = index; ReadTransactionPtr tx = store_.createReadTransaction(); for (uint32_t level = depth_; level > 0; --level) { bool is_right = static_cast(current_index & 0x01); - fr right_value = is_right ? get_element_or_zero(level, current_index, *tx, includeUncommitted) - : get_element_or_zero(level, current_index + 1, *tx, includeUncommitted); - fr left_value = is_right ? get_element_or_zero(level, current_index - 1, *tx, includeUncommitted) - : get_element_or_zero(level, current_index, *tx, includeUncommitted); - response.inner.path.emplace_back(left_value, right_value); + fr sibling = is_right ? get_element_or_zero(level, current_index - 1, *tx, includeUncommitted) + : get_element_or_zero(level, current_index + 1, *tx, includeUncommitted); + response.inner.path.emplace_back(sibling); current_index >>= 1; } }, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp index 1649b40ff45..f28d3b11ad6 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -66,15 +66,18 @@ void check_root(TreeType& tree, fr expected_root, bool includeUncommitted = true signal.wait_for_level(); } -void check_hash_path(TreeType& tree, index_t index, fr_hash_path expected_hash_path, bool includeUncommitted = true) +void check_sibling_path(TreeType& tree, + index_t index, + fr_sibling_path expected_sibling_path, + bool includeUncommitted = true) { Signal signal; - auto completion = [&](const TypedResponse& response) -> void { + auto completion = [&](const TypedResponse& response) -> void { EXPECT_EQ(response.success, true); - EXPECT_EQ(response.inner.path, expected_hash_path); + EXPECT_EQ(response.inner.path, expected_sibling_path); signal.signal_level(); }; - tree.get_hash_path(index, completion, includeUncommitted); + tree.get_sibling_path(index, completion, includeUncommitted); signal.wait_for_level(); } @@ -151,7 +154,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_only_recreate_with_same_name_and_depth) EXPECT_ANY_THROW(Store store_wrong_depth(name, depth + 1, db)); } -TEST_F(PersistedAppendOnlyTreeTest, can_add_value_and_get_hash_path) +TEST_F(PersistedAppendOnlyTreeTest, can_add_value_and_get_sibling_path) { constexpr size_t depth = 10; std::string name = randomString(); @@ -170,7 +173,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_value_and_get_hash_path) check_size(tree, 1); check_root(tree, memdb.root()); - check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); } TEST_F(PersistedAppendOnlyTreeTest, reports_an_error_if_tree_is_overfilled) @@ -298,22 +301,22 @@ TEST_F(PersistedAppendOnlyTreeTest, can_commit_and_restore) check_size(tree, 0); check_root(tree, memdb.root()); - check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); bb::fr initial_root = memdb.root(); - fr_hash_path initial_hash_path = memdb.get_hash_path(0); + fr_sibling_path initial_sibling_path = memdb.get_sibling_path(0); memdb.update_element(0, VALUES[0]); add_value(tree, VALUES[0]); // check uncommitted state check_size(tree, 1); check_root(tree, memdb.root()); - check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); // check committed state check_size(tree, 0, false); check_root(tree, initial_root, false); - check_hash_path(tree, 0, initial_hash_path, false); + check_sibling_path(tree, 0, initial_sibling_path, false); // commit the changes commit_tree(tree); @@ -322,12 +325,12 @@ TEST_F(PersistedAppendOnlyTreeTest, can_commit_and_restore) // check uncommitted state check_size(tree, 1); check_root(tree, memdb.root()); - check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); // check committed state check_size(tree, 1, false); check_root(tree, memdb.root(), false); - check_hash_path(tree, 0, memdb.get_hash_path(0), false); + check_sibling_path(tree, 0, memdb.get_sibling_path(0), false); } // Re-create the store and tree, it should be the same as how we left it @@ -341,12 +344,12 @@ TEST_F(PersistedAppendOnlyTreeTest, can_commit_and_restore) // check uncommitted state check_size(tree, 1); check_root(tree, memdb.root()); - check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); // check committed state check_size(tree, 1, false); check_root(tree, memdb.root(), false); - check_hash_path(tree, 0, memdb.get_hash_path(0), false); + check_sibling_path(tree, 0, memdb.get_sibling_path(0), false); } } @@ -393,8 +396,8 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values) add_value(tree, VALUES[i]); check_root(tree, mock_root); - check_hash_path(tree, 0, memdb.get_hash_path(0)); - check_hash_path(tree, i, memdb.get_hash_path(i)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); + check_sibling_path(tree, i, memdb.get_sibling_path(i)); } } @@ -414,8 +417,8 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values_in_a_batch) add_values(tree, VALUES); check_size(tree, NUM_VALUES); check_root(tree, memdb.root()); - check_hash_path(tree, 0, memdb.get_hash_path(0)); - check_hash_path(tree, NUM_VALUES - 1, memdb.get_hash_path(NUM_VALUES - 1)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); + check_sibling_path(tree, NUM_VALUES - 1, memdb.get_sibling_path(NUM_VALUES - 1)); } TEST_F(PersistedAppendOnlyTreeTest, can_be_filled) @@ -437,20 +440,20 @@ TEST_F(PersistedAppendOnlyTreeTest, can_be_filled) } check_root(tree, memdb.root()); - check_hash_path(tree, 0, memdb.get_hash_path(0)); - check_hash_path(tree, 7, memdb.get_hash_path(7)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); + check_sibling_path(tree, 7, memdb.get_sibling_path(7)); } TEST_F(PersistedAppendOnlyTreeTest, can_add_single_whilst_reading) { constexpr size_t depth = 10; MemoryTree memdb(depth); - fr_hash_path initial_path = memdb.get_hash_path(0); + fr_sibling_path initial_path = memdb.get_sibling_path(0); memdb.update_element(0, VALUES[0]); - fr_hash_path final_hash_path = memdb.get_hash_path(0); + fr_sibling_path final_sibling_path = memdb.get_sibling_path(0); uint32_t num_reads = 16 * 1024; - std::vector paths(num_reads); + std::vector paths(num_reads); { std::string name = randomString(); @@ -471,15 +474,15 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_single_whilst_reading) tree.add_value(VALUES[0], add_completion); for (size_t i = 0; i < num_reads; i++) { - auto completion = [&, i](const TypedResponse& response) { + auto completion = [&, i](const TypedResponse& response) { paths[i] = response.inner.path; }; - tree.get_hash_path(0, completion, false); + tree.get_sibling_path(0, completion, false); } signal.wait_for_level(0); } for (auto& path : paths) { - EXPECT_TRUE(path == initial_path || path == final_hash_path); + EXPECT_TRUE(path == initial_path || path == final_sibling_path); } } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 08539e19d57..d695f04c82e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -58,7 +58,7 @@ template class IndexedTree : public App void get_leaf(const index_t& index, bool includeUncommitted, const LeafCallback& completion); - using AppendOnlyTree::get_hash_path; + using AppendOnlyTree::get_sibling_path; private: using typename AppendOnlyTree::AppendCompletionCallback; @@ -86,7 +86,7 @@ template class IndexedTree : public App const IndexedLeafValueType& leaf, Signal& leader, Signal& follower, - fr_hash_path& previous_hash_path, + fr_sibling_path& previous_sibling_path, ReadTransaction& tx); struct InsertionGenerationResponse { @@ -99,7 +99,7 @@ template class IndexedTree : public App const InsertionGenerationCallback& completion); struct InsertionCompletionResponse { - std::shared_ptr> paths; + std::shared_ptr> paths; }; using InsertionCompletionCallback = std::function&)>; @@ -224,7 +224,7 @@ void IndexedTree::add_or_update_values(const std::vector> hashes_to_append; - std::shared_ptr> paths; + std::shared_ptr> paths; std::atomic count; Status status; @@ -320,7 +320,8 @@ void IndexedTree::perform_insertions(std::shared_ptr> paths = std::make_shared>(insertions->size()); + std::shared_ptr> paths = + std::make_shared>(insertions->size()); std::shared_ptr> signals = std::make_shared>(); std::shared_ptr status = std::make_shared(); // The first signal is set to 0. This ensures the first worker up the tree is not impeded @@ -467,7 +468,7 @@ void IndexedTree::update_leaf_and_hash_to_root(const index const IndexedLeafValueType& leaf, Signal& leader, Signal& follower, - fr_hash_path& previous_hash_path, + fr_sibling_path& previous_sibling_path, ReadTransaction& tx) { auto get_node = [&](uint32_t level, index_t index) -> fr { return get_element_or_zero(level, index, tx, true); }; @@ -488,9 +489,8 @@ void IndexedTree::update_leaf_and_hash_to_root(const index // Extract the value of the leaf node and it's sibling bool is_right = static_cast(index & 0x01); // extract the current leaf hash values for the previous hash path - fr current_right_value = get_node(level, index + (is_right ? 0 : 1)); - fr current_left_value = get_node(level, is_right ? (index - 1) : index); - previous_hash_path.emplace_back(current_left_value, current_right_value); + fr sibling = get_node(level, is_right ? index - 1 : index + 1); + previous_sibling_path.emplace_back(sibling); // Write the new leaf hash in place write_node(level, index, new_hash); @@ -508,10 +508,8 @@ void IndexedTree::update_leaf_and_hash_to_root(const index // Now read the node and it's sibling index_t index_of_node_above = index >> 1; bool node_above_is_right = static_cast(index_of_node_above & 0x01); - fr above_right_value = get_node(level_to_read, index_of_node_above + (node_above_is_right ? 0 : 1)); - fr above_left_value = - get_node(level_to_read, node_above_is_right ? (index_of_node_above - 1) : index_of_node_above); - previous_hash_path.emplace_back(above_left_value, above_right_value); + fr above_sibling = get_node(level, node_above_is_right ? index_of_node_above - 1 : index_of_node_above + 1); + previous_sibling_path.emplace_back(above_sibling); } // Now that we have extracted the hash path from the row above diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index eff2d65e692..bd88b943210 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -92,17 +92,17 @@ void check_root(IndexedTree, Poseidon2 } template -fr_hash_path get_hash_path(IndexedTree, Poseidon2HashPolicy>& tree, - index_t index, - bool includeUncommitted = true) +fr_sibling_path get_sibling_path(IndexedTree, Poseidon2HashPolicy>& tree, + index_t index, + bool includeUncommitted = true) { - fr_hash_path h; + fr_sibling_path h; Signal signal; - auto completion = [&](const TypedResponse& response) -> void { + auto completion = [&](const TypedResponse& response) -> void { h = response.inner.path; signal.signal_level(); }; - tree.get_hash_path(index, completion, includeUncommitted); + tree.get_sibling_path(index, completion, includeUncommitted); signal.wait_for_level(); return h; } @@ -124,13 +124,13 @@ IndexedLeaf get_leaf(IndexedTree -void check_hash_path(IndexedTree, Poseidon2HashPolicy>& tree, - index_t index, - fr_hash_path expected_hash_path, - bool includeUncommitted = true) +void check_sibling_path(IndexedTree, Poseidon2HashPolicy>& tree, + index_t index, + fr_sibling_path expected_sibling_path, + bool includeUncommitted = true) { - fr_hash_path path = get_hash_path(tree, index, includeUncommitted); - EXPECT_EQ(path, expected_hash_path); + fr_sibling_path path = get_sibling_path(tree, index, includeUncommitted); + EXPECT_EQ(path, expected_sibling_path); } template @@ -249,7 +249,7 @@ TEST_F(PersistedIndexedTreeTest, reports_an_error_if_tree_is_overfilled) signal.wait_for_level(); } -TEST_F(PersistedIndexedTreeTest, test_get_hash_path) +TEST_F(PersistedIndexedTreeTest, test_get_sibling_path) { index_t current_size = 2; NullifierMemoryTree memdb(10); @@ -263,14 +263,14 @@ TEST_F(PersistedIndexedTreeTest, test_get_hash_path) check_size(tree, current_size); check_root(tree, memdb.root()); - check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); memdb.update_element(VALUES[512]); add_value(tree, NullifierLeafValue(VALUES[512])); check_size(tree, ++current_size); - check_hash_path(tree, 0, memdb.get_hash_path(0)); - check_hash_path(tree, 1, memdb.get_hash_path(1)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); + check_sibling_path(tree, 1, memdb.get_sibling_path(1)); uint32_t num_to_append = 512; @@ -279,8 +279,8 @@ TEST_F(PersistedIndexedTreeTest, test_get_hash_path) add_value(tree, NullifierLeafValue(VALUES[i])); } check_size(tree, num_to_append + current_size); - check_hash_path(tree, 0, memdb.get_hash_path(0)); - check_hash_path(tree, 512, memdb.get_hash_path(512)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); + check_sibling_path(tree, 512, memdb.get_sibling_path(512)); } TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) @@ -298,23 +298,23 @@ TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) check_size(tree, current_size); check_root(tree, memdb.root()); - check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); add_value(tree, NullifierLeafValue(VALUES[512])); // Committed data should not have changed check_size(tree, current_size, false); check_root(tree, memdb.root(), false); - check_hash_path(tree, 0, memdb.get_hash_path(0), false); - check_hash_path(tree, 1, memdb.get_hash_path(1), false); + check_sibling_path(tree, 0, memdb.get_sibling_path(0), false); + check_sibling_path(tree, 1, memdb.get_sibling_path(1), false); memdb.update_element(VALUES[512]); // Uncommitted data should have changed check_size(tree, current_size + 1, true); check_root(tree, memdb.root(), true); - check_hash_path(tree, 0, memdb.get_hash_path(0), true); - check_hash_path(tree, 1, memdb.get_hash_path(1), true); + check_sibling_path(tree, 0, memdb.get_sibling_path(0), true); + check_sibling_path(tree, 1, memdb.get_sibling_path(1), true); // Now commit commit_tree(tree); @@ -322,8 +322,8 @@ TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) // Now committed data should have changed check_size(tree, ++current_size, false); check_root(tree, memdb.root(), false); - check_hash_path(tree, 0, memdb.get_hash_path(0), false); - check_hash_path(tree, 1, memdb.get_hash_path(1), false); + check_sibling_path(tree, 0, memdb.get_sibling_path(0), false); + check_sibling_path(tree, 1, memdb.get_sibling_path(1), false); } // Now restore and it should continue from where we left off @@ -335,12 +335,12 @@ TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) // check uncommitted state check_size(tree, current_size); check_root(tree, memdb.root()); - check_hash_path(tree, 0, memdb.get_hash_path(0)); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); // check committed state check_size(tree, current_size, false); check_root(tree, memdb.root(), false); - check_hash_path(tree, 0, memdb.get_hash_path(0), false); + check_sibling_path(tree, 0, memdb.get_sibling_path(0), false); } } @@ -367,26 +367,26 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) check_root(tree1, memdb.root()); check_root(tree2, memdb.root()); - check_hash_path(tree1, 0, memdb.get_hash_path(0)); - check_hash_path(tree2, 0, memdb.get_hash_path(0)); + check_sibling_path(tree1, 0, memdb.get_sibling_path(0)); + check_sibling_path(tree2, 0, memdb.get_sibling_path(0)); - check_hash_path(tree1, 512, memdb.get_hash_path(512)); - check_hash_path(tree2, 512, memdb.get_hash_path(512)); + check_sibling_path(tree1, 512, memdb.get_sibling_path(512)); + check_sibling_path(tree2, 512, memdb.get_sibling_path(512)); for (uint32_t i = 0; i < num_batches; i++) { std::vector batch; - std::vector memory_tree_hash_paths; + std::vector memory_tree_sibling_paths; for (uint32_t j = 0; j < batch_size; j++) { batch.emplace_back(random_engine.get_random_uint256()); - fr_hash_path path = memdb.update_element(batch[j].value); - memory_tree_hash_paths.push_back(path); + fr_sibling_path path = memdb.update_element(batch[j].value); + memory_tree_sibling_paths.push_back(path); } - std::shared_ptr> tree1_hash_paths; - std::shared_ptr> tree2_hash_paths; + std::shared_ptr> tree1_sibling_paths; + std::shared_ptr> tree2_sibling_paths; { Signal signal; CompletionCallback completion = [&](const TypedResponse& response) { - tree1_hash_paths = response.inner.paths; + tree1_sibling_paths = response.inner.paths; signal.signal_level(); }; tree1.add_or_update_values(batch, completion); @@ -395,7 +395,7 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) { Signal signal; CompletionCallback completion = [&](const TypedResponse& response) { - tree2_hash_paths = response.inner.paths; + tree2_sibling_paths = response.inner.paths; signal.signal_level(); }; tree2.add_or_update_values(batch, completion); @@ -404,14 +404,14 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) check_root(tree1, memdb.root()); check_root(tree2, memdb.root()); - check_hash_path(tree1, 0, memdb.get_hash_path(0)); - check_hash_path(tree2, 0, memdb.get_hash_path(0)); + check_sibling_path(tree1, 0, memdb.get_sibling_path(0)); + check_sibling_path(tree2, 0, memdb.get_sibling_path(0)); - check_hash_path(tree1, 512, memdb.get_hash_path(512)); - check_hash_path(tree2, 512, memdb.get_hash_path(512)); + check_sibling_path(tree1, 512, memdb.get_sibling_path(512)); + check_sibling_path(tree2, 512, memdb.get_sibling_path(512)); for (uint32_t j = 0; j < batch_size; j++) { - EXPECT_EQ(tree1_hash_paths->at(j), tree2_hash_paths->at(j)); + EXPECT_EQ(tree1_sibling_paths->at(j), tree2_sibling_paths->at(j)); } } } @@ -447,16 +447,16 @@ template fr hash_leaf(const IndexedLeaf& return HashPolicy::hash(leaf.get_hash_inputs()); } -bool verify_hash_path(TreeType& tree, const IndexedNullifierLeafType& leaf_value, const uint32_t idx) +bool verify_sibling_path(TreeType& tree, const IndexedNullifierLeafType& leaf_value, const uint32_t idx) { fr root = get_root(tree, true); - fr_hash_path path = get_hash_path(tree, idx, true); + fr_sibling_path path = get_sibling_path(tree, idx, true); auto current = hash_leaf(leaf_value); uint32_t depth_ = static_cast(path.size()); uint32_t index = idx; for (uint32_t i = 0; i < depth_; ++i) { - fr left = (index & 1) ? path[i].first : current; - fr right = (index & 1) ? current : path[i].second; + fr left = (index & 1) ? path[i] : current; + fr right = (index & 1) ? current : path[i]; current = HashPolicy::hash_pair(left, right); index >>= 1; } @@ -597,30 +597,45 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) // Check the hash path at index 2 and 3 // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) - fr_hash_path expected = { - std::make_pair(e000, e001), - std::make_pair(e00, e01), - std::make_pair(e0, e1), + fr_sibling_path expected = { + e001, + e01, + e1, }; - check_hash_path(tree, 0, expected); - check_hash_path(tree, 1, expected); + check_sibling_path(tree, 0, expected); expected = { - std::make_pair(e010, e011), - std::make_pair(e00, e01), - std::make_pair(e0, e1), + e000, + e01, + e1, }; - check_hash_path(tree, 2, expected); - check_hash_path(tree, 3, expected); + check_sibling_path(tree, 1, expected); + expected = { + e011, + e00, + e1, + }; + check_sibling_path(tree, 2, expected); + expected = { + e010, + e00, + e1, + }; + check_sibling_path(tree, 3, expected); check_root(tree, root); // Check the hash path at index 6 and 7 expected = { - std::make_pair(e110, e111), - std::make_pair(e10, e11), - std::make_pair(e0, e1), + e111, + e10, + e0, }; - check_hash_path(tree, 6, expected); - check_hash_path(tree, 7, expected); + check_sibling_path(tree, 6, expected); + expected = { + e110, + e10, + e0, + }; + check_sibling_path(tree, 7, expected); } TEST_F(PersistedIndexedTreeTest, test_indexed_tree) @@ -667,19 +682,19 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_tree) auto index = static_cast(it - differences.begin()); // Merkle proof at `index` proves non-membership of `new_member` - EXPECT_TRUE(verify_hash_path(tree, get_leaf(tree, index), index)); + EXPECT_TRUE(verify_sibling_path(tree, get_leaf(tree, index), index)); } TEST_F(PersistedIndexedTreeTest, can_add_single_whilst_reading) { constexpr size_t depth = 10; NullifierMemoryTree memdb(10); - fr_hash_path initial_path = memdb.get_hash_path(0); + fr_sibling_path initial_path = memdb.get_sibling_path(0); memdb.update_element(VALUES[0]); - fr_hash_path final_hash_path = memdb.get_hash_path(0); + fr_sibling_path final_sibling_path = memdb.get_sibling_path(0); uint32_t num_reads = 16 * 1024; - std::vector paths(num_reads); + std::vector paths(num_reads); { std::string name = randomString(); @@ -700,16 +715,16 @@ TEST_F(PersistedIndexedTreeTest, can_add_single_whilst_reading) tree.add_or_update_value(VALUES[0], add_completion); for (size_t i = 0; i < num_reads; i++) { - auto completion = [&, i](const TypedResponse& response) { + auto completion = [&, i](const TypedResponse& response) { paths[i] = response.inner.path; }; - tree.get_hash_path(0, completion, false); + tree.get_sibling_path(0, completion, false); } signal.wait_for_level(); } for (auto& path : paths) { - EXPECT_TRUE(path == initial_path || path == final_hash_path); + EXPECT_TRUE(path == initial_path || path == final_sibling_path); } } @@ -831,30 +846,43 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory_with_public_data_writes) auto e1 = HashPolicy::hash_pair(e10, e11); auto root = HashPolicy::hash_pair(e0, e1); - // Check the hash path at index 2 and 3 - // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) - fr_hash_path expected = { - std::make_pair(e000, e001), - std::make_pair(e00, e01), - std::make_pair(e0, e1), + fr_sibling_path expected = { + e001, + e01, + e1, + }; + check_sibling_path(tree, 0, expected); + expected = { + e000, + e01, + e1, }; - check_hash_path(tree, 0, expected); - check_hash_path(tree, 1, expected); + check_sibling_path(tree, 1, expected); expected = { - std::make_pair(e010, e011), - std::make_pair(e00, e01), - std::make_pair(e0, e1), + e011, + e00, + e1, }; - check_hash_path(tree, 2, expected); - check_hash_path(tree, 3, expected); + check_sibling_path(tree, 2, expected); + expected = { + e010, + e00, + e1, + }; + check_sibling_path(tree, 3, expected); check_root(tree, root); // Check the hash path at index 6 and 7 expected = { - std::make_pair(e110, e111), - std::make_pair(e10, e11), - std::make_pair(e0, e1), + e111, + e10, + e0, + }; + check_sibling_path(tree, 6, expected); + expected = { + e110, + e10, + e0, }; - check_hash_path(tree, 6, expected); - check_hash_path(tree, 7, expected); + check_sibling_path(tree, 7, expected); } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp index e2b4b1b370c..ce662250045 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -1,6 +1,7 @@ #pragma once #include "../hash.hpp" #include "../memory_tree.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "nullifier_leaf.hpp" namespace bb::crypto::merkle_tree { @@ -69,10 +70,11 @@ template class NullifierMemoryTree : public MemoryTree< NullifierMemoryTree(size_t depth, size_t initial_size = 2); using MemoryTree::get_hash_path; + using MemoryTree::get_sibling_path; using MemoryTree::root; using MemoryTree::update_element; - fr_hash_path update_element(fr const& value); + fr_sibling_path update_element(fr const& value); const std::vector& get_hashes() { return hashes_; } const WrappedNullifierLeaf get_leaf(size_t index) @@ -125,15 +127,15 @@ NullifierMemoryTree::NullifierMemoryTree(size_t depth, size_t ini } } -template fr_hash_path NullifierMemoryTree::update_element(fr const& value) +template fr_sibling_path NullifierMemoryTree::update_element(fr const& value) { // Find the leaf with the value closest and less than `value` // If value is 0 we simply append 0 a null NullifierLeaf to the tree - fr_hash_path hash_path; + fr_sibling_path hash_path; if (value == 0) { auto zero_leaf = WrappedNullifierLeaf::zero(); - hash_path = get_hash_path(leaves_.size() - 1); + hash_path = get_sibling_path(leaves_.size() - 1); leaves_.push_back(zero_leaf); update_element(leaves_.size() - 1, zero_leaf.hash()); return hash_path; @@ -159,7 +161,7 @@ template fr_hash_path NullifierMemoryTree> paths; + std::shared_ptr> paths; }; template struct GetIndexedLeafResponse { From e6663b89ebea21495db73c2ecb57733f85d089fe Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 4 Jul 2024 16:15:01 +0000 Subject: [PATCH 30/63] fix: tree lambda capture this --- .../append_only_tree/append_only_tree.hpp | 18 +++++++-------- .../merkle_tree/indexed_tree/indexed_tree.hpp | 23 ++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index f21adb3b6f6..2bb1d22dcd1 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -111,9 +111,9 @@ AppendOnlyTree::AppendOnlyTree(Store& store, ThreadPool& w template void AppendOnlyTree::get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion) { - auto job = [=]() { + auto job = [=, this]() { ExecuteAndReport( - [=](TypedResponse& response) { + [=, this](TypedResponse& response) { ReadTransactionPtr tx = store_.createReadTransaction(); store_.get_meta(response.inner.size, response.inner.root, *tx, includeUncommitted); }, @@ -127,9 +127,9 @@ void AppendOnlyTree::get_sibling_path(const index_t& index const HashPathCallback& on_completion, bool includeUncommitted) const { - auto job = [=]() { + auto job = [=, this]() { ExecuteAndReport( - [=](TypedResponse& response) { + [=, this](TypedResponse& response) { index_t current_index = index; ReadTransactionPtr tx = store_.createReadTransaction(); for (uint32_t level = depth_; level > 0; --level) { @@ -156,9 +156,9 @@ void AppendOnlyTree::add_values(const std::vector& val const AppendCompletionCallback& on_completion) { std::shared_ptr> hashes = std::make_shared>(values); - auto append_op = [=]() -> void { + auto append_op = [=, this]() -> void { ExecuteAndReport( - [=](TypedResponse& response) { + [=, this](TypedResponse& response) { add_values_internal(hashes, response.inner.root, response.inner.size); }, on_completion); @@ -258,15 +258,15 @@ std::pair AppendOnlyTree::read_node(uint32_t lev template void AppendOnlyTree::commit(const CommitCallback& on_completion) { - auto job = [=]() { ExecuteAndReport([=]() { store_.commit(); }, on_completion); }; + auto job = [=, this]() { ExecuteAndReport([=, this]() { store_.commit(); }, on_completion); }; workers_.enqueue(job); } template void AppendOnlyTree::rollback(const RollbackCallback& on_completion) { - auto job = [=]() { ExecuteAndReport([=]() { store_.rollback(); }, on_completion); }; + auto job = [=, this]() { ExecuteAndReport([=, this]() { store_.rollback(); }, on_completion); }; workers_.enqueue(job); } -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index d695f04c82e..9806ff20450 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -192,9 +192,9 @@ void IndexedTree::get_leaf(const index_t& index, bool includeUncommitted, const LeafCallback& completion) { - auto job = [=]() { + auto job = [=, this]() { ExecuteAndReport>( - [=](TypedResponse>& response) { + [=, this](TypedResponse>& response) { ReadTransactionPtr tx = store_.createReadTransaction(); response.inner.indexed_leaf = store_.get_leaf(index, *tx, includeUncommitted); }, @@ -264,7 +264,7 @@ void IndexedTree::add_or_update_values(const std::vector& hashes_response) { + HashGenerationCallback hash_completion = [=, this](const TypedResponse& hashes_response) { if (!hashes_response.success) { results->status.set_failure(hashes_response.message); } else { @@ -282,7 +282,7 @@ void IndexedTree::add_or_update_values(const std::vector& insertion_response) { + [=, this](const TypedResponse& insertion_response) { if (!insertion_response.success) { results->status.set_failure(insertion_response.message); } else { @@ -300,18 +300,19 @@ void IndexedTree::add_or_update_values(const std::vector& insertion_response) { + [=, this](const TypedResponse& insertion_response) { if (!insertion_response.success) { on_error(insertion_response.message); return; } - workers_.enqueue( - [=]() { generate_hashes_for_appending(insertion_response.inner.indexed_leaves, hash_completion); }); + workers_.enqueue([=, this]() { + generate_hashes_for_appending(insertion_response.inner.indexed_leaves, hash_completion); + }); perform_insertions(insertion_response.inner.insertions, insertion_completion); }; // We start by enqueueing the insertion data generation - workers_.enqueue([=]() { generate_insertions(values_to_be_sorted, insertion_generation_completed); }); + workers_.enqueue([=, this]() { generate_insertions(values_to_be_sorted, insertion_generation_completed); }); } template @@ -332,7 +333,7 @@ void IndexedTree::perform_insertions(std::shared_ptrsize(); ++i) { - auto op = [=]() { + auto op = [=, this]() { LeafInsertion& insertion = (*insertions)[i]; Signal& leaderSignal = (*signals)[i]; Signal& followerSignal = (*signals)[i + 1]; @@ -380,7 +381,7 @@ void IndexedTree::generate_insertions( const InsertionGenerationCallback& on_completion) { ExecuteAndReport( - [=](TypedResponse& response) { + [=, this](TypedResponse& response) { // The first thing we do is sort the values into descending order but maintain knowledge of their orignal // order struct { @@ -533,4 +534,4 @@ void IndexedTree::update_leaf_and_hash_to_root(const index } } -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree From f41ff3c98da985744e8336b45756c6d23c6c5d06 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 4 Jul 2024 16:47:54 +0000 Subject: [PATCH 31/63] fix: init response object --- .../crypto/merkle_tree/node_store/cached_tree_store.hpp | 2 +- .../cpp/src/barretenberg/crypto/merkle_tree/response.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp index 232f6d0f65a..412e2dee072 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp @@ -332,4 +332,4 @@ void CachedTreeStore::initialise() } } -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 7b37fb38b04..9da2265db07 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -64,7 +64,7 @@ void ExecuteAndReport(const std::function&)>& f inline void ExecuteAndReport(const std::function& f, const std::function& on_completion) { - Response response; + Response response{ true, "" }; try { f(); } catch (std::exception& e) { @@ -76,4 +76,4 @@ inline void ExecuteAndReport(const std::function& f, const std::function } catch (std::exception&) { } } -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree From 2f6fb9cf4f748f523e861fdc777bc83d59afb6c2 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 5 Jul 2024 02:28:58 +0100 Subject: [PATCH 32/63] Added find leaf index api --- .../append_only_tree/append_only_tree.hpp | 78 +++++++++- .../append_only_tree.test.cpp | 123 ++++++++++++++++ .../merkle_tree/indexed_tree/indexed_leaf.hpp | 4 + .../merkle_tree/indexed_tree/indexed_tree.hpp | 41 +++++- .../indexed_tree/indexed_tree.test.cpp | 86 ++++++++++- .../node_store/cached_tree_store.hpp | 133 ++++++++++++++++-- .../crypto/merkle_tree/response.hpp | 4 + 7 files changed, 447 insertions(+), 22 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index f21adb3b6f6..9596f9b157c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -27,6 +27,7 @@ template class AppendOnlyTree { using AppendCompletionCallback = std::function&)>; using MetaDataCallback = std::function&)>; using HashPathCallback = std::function&)>; + using FindLeafCallback = std::function&)>; using CommitCallback = std::function; using RollbackCallback = std::function; @@ -52,7 +53,14 @@ template class AppendOnlyTree { */ void get_sibling_path(const index_t& index, const HashPathCallback& on_completion, bool includeUncommitted) const; - void get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion); + void get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion) const; + + void find_leaf_index(const fr& leaf, bool includeUncommitted, const FindLeafCallback& on_completion) const; + + void find_leaf_index_from(const fr& leaf, + index_t start_index, + bool includeUncommitted, + const FindLeafCallback& on_completion) const; void commit(const CommitCallback& on_completion); void rollback(const RollbackCallback& on_completion); @@ -70,7 +78,14 @@ template class AppendOnlyTree { ReadTransaction& tx, bool includeUncommitted) const; - void add_values_internal(std::shared_ptr> values, fr& new_root, index_t& new_size); + void add_values_internal(std::shared_ptr> values, + fr& new_root, + index_t& new_size, + bool update_index); + + void add_values_internal(const std::vector& values, + const AppendCompletionCallback& on_completion, + bool update_index); Store& store_; uint32_t depth_; @@ -109,7 +124,8 @@ AppendOnlyTree::AppendOnlyTree(Store& store, ThreadPool& w } template -void AppendOnlyTree::get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion) +void AppendOnlyTree::get_meta_data(bool includeUncommitted, + const MetaDataCallback& on_completion) const { auto job = [=]() { ExecuteAndReport( @@ -145,6 +161,36 @@ void AppendOnlyTree::get_sibling_path(const index_t& index workers_.enqueue(job); } +template +void AppendOnlyTree::find_leaf_index(const fr& leaf, + bool includeUncommitted, + const FindLeafCallback& on_completion) const +{ + return find_leaf_index_from(leaf, 0, includeUncommitted, on_completion); +} + +template +void AppendOnlyTree::find_leaf_index_from(const fr& leaf, + index_t start_index, + bool includeUncommitted, + const FindLeafCallback& on_completion) const +{ + auto job = [=]() -> void { + ExecuteAndReport( + [=](TypedResponse& response) { + typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); + std::optional leaf_index = + store_.find_leaf_index_from(leaf, start_index, *tx, includeUncommitted); + response.success = leaf_index.has_value(); + if (response.success) { + response.inner.leaf_index = leaf_index.value(); + } + }, + on_completion); + }; + workers_.enqueue(job); +} + template void AppendOnlyTree::add_value(const fr& value, const AppendCompletionCallback& on_completion) { @@ -159,7 +205,23 @@ void AppendOnlyTree::add_values(const std::vector& val auto append_op = [=]() -> void { ExecuteAndReport( [=](TypedResponse& response) { - add_values_internal(hashes, response.inner.root, response.inner.size); + add_values_internal(hashes, response.inner.root, response.inner.size, true); + }, + on_completion); + }; + workers_.enqueue(append_op); +} + +template +void AppendOnlyTree::add_values_internal(const std::vector& values, + const AppendCompletionCallback& on_completion, + bool update_index) +{ + std::shared_ptr> hashes = std::make_shared>(values); + auto append_op = [=]() -> void { + ExecuteAndReport( + [=](TypedResponse& response) { + add_values_internal(hashes, response.inner.root, response.inner.size, update_index); }, on_completion); }; @@ -169,7 +231,8 @@ void AppendOnlyTree::add_values(const std::vector& val template void AppendOnlyTree::add_values_internal(std::shared_ptr> values, fr& new_root, - index_t& new_size) + index_t& new_size, + bool update_index) { uint32_t start_level = depth_; uint32_t level = start_level; @@ -190,6 +253,11 @@ void AppendOnlyTree::add_values_internal(std::shared_ptr 1) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp index f28d3b11ad6..a97758c3f50 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -10,6 +10,7 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" #include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp" #include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "gtest/gtest.h" #include @@ -127,6 +128,42 @@ void add_values(TreeType& tree, const std::vector& values) signal.wait_for_level(); } +void check_find_leaf_index( + TreeType& tree, const fr& leaf, index_t expected_index, bool expected_success, bool includeUncommitted = true) +{ + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, expected_success); + if (expected_success) { + EXPECT_EQ(response.inner.leaf_index, expected_index); + } + signal.signal_level(); + }; + + tree.find_leaf_index(leaf, includeUncommitted, completion); + signal.wait_for_level(); +} + +void check_find_leaf_index_from(TreeType& tree, + const fr& leaf, + index_t start_index, + index_t expected_index, + bool expected_success, + bool includeUncommitted = true) +{ + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, expected_success); + if (expected_success) { + EXPECT_EQ(response.inner.leaf_index, expected_index); + } + signal.signal_level(); + }; + + tree.find_leaf_index_from(leaf, start_index, includeUncommitted, completion); + signal.wait_for_level(); +} + TEST_F(PersistedAppendOnlyTreeTest, can_create) { constexpr size_t depth = 10; @@ -381,6 +418,92 @@ TEST_F(PersistedAppendOnlyTreeTest, test_size) check_size(tree, 4); } +TEST_F(PersistedAppendOnlyTreeTest, test_find_leaf_index) +{ + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + ThreadPool pool(1); + TreeType tree(store, pool); + + add_value(tree, 30); + add_value(tree, 10); + add_value(tree, 20); + add_value(tree, 40); + + // check the committed state and that the uncommitted state is empty + check_find_leaf_index(tree, 10, 1, true, true); + check_find_leaf_index(tree, 10, 0, false, false); + + check_find_leaf_index(tree, 15, 0, false, true); + check_find_leaf_index(tree, 15, 0, false, false); + + check_find_leaf_index(tree, 40, 3, true, true); + check_find_leaf_index(tree, 30, 0, true, true); + check_find_leaf_index(tree, 20, 2, true, true); + + check_find_leaf_index(tree, 40, 0, false, false); + check_find_leaf_index(tree, 30, 0, false, false); + check_find_leaf_index(tree, 20, 0, false, false); + + commit_tree(tree); + + std::vector values{ 15, 18, 26, 2, 48 }; + add_values(tree, values); + + // check the now committed state + check_find_leaf_index(tree, 40, 3, true, false); + check_find_leaf_index(tree, 30, 0, true, false); + check_find_leaf_index(tree, 20, 2, true, false); + + // check the new uncommitted state + check_find_leaf_index(tree, 18, 5, true, true); + check_find_leaf_index(tree, 18, 0, false, false); + + commit_tree(tree); + + values = { 16, 4, 18, 22, 101 }; + add_values(tree, values); + + // we now have duplicate leaf 18, one committed the other not + check_find_leaf_index(tree, 18, 5, true, true); + check_find_leaf_index(tree, 18, 5, true, false); + + // verify the find index from api + check_find_leaf_index_from(tree, 18, 0, 5, true, true); + check_find_leaf_index_from(tree, 18, 6, 11, true, true); + check_find_leaf_index_from(tree, 18, 6, 0, false, false); + + commit_tree(tree); + + // add another leaf 18 + add_value(tree, 18); + + // should return the first index + check_find_leaf_index_from(tree, 18, 0, 5, true, false); + check_find_leaf_index_from(tree, 18, 0, 5, true, true); + + add_value(tree, 88); + // and another uncommitted 18 + add_value(tree, 18); + + add_value(tree, 32); + + // should return the first uncommitted + check_find_leaf_index_from(tree, 18, 12, 14, true, true); + check_find_leaf_index_from(tree, 18, 15, 16, true, true); + + // look past the last instance of this leaf + check_find_leaf_index_from(tree, 18, 17, 0, false, true); + + // look beyond the end of uncommitted + check_find_leaf_index_from(tree, 18, 18, 0, false, true); + + // look beyond the end of committed and don't include uncomitted + check_find_leaf_index_from(tree, 18, 14, 0, false, false); +} + TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values) { constexpr size_t depth = 10; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp index db28f2f40bd..59b2b7e4563 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp @@ -46,6 +46,8 @@ struct NullifierLeafValue { bool is_empty() const { return value == fr::zero(); } + operator uint256_t() const { return get_key(); } + static NullifierLeafValue empty() { return { fr::zero() }; } static NullifierLeafValue padding(index_t i) { return { i }; } @@ -95,6 +97,8 @@ struct PublicDataLeafValue { bool is_empty() const { return value == fr::zero() && slot == fr::zero(); } + operator uint256_t() const { return get_key(); } + static PublicDataLeafValue empty() { return { fr::zero(), fr::zero() }; } static PublicDataLeafValue padding(index_t i) { return { i, fr::zero() }; } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index d695f04c82e..b0322ae7fe5 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -58,9 +58,14 @@ template class IndexedTree : public App void get_leaf(const index_t& index, bool includeUncommitted, const LeafCallback& completion); + void find_leaf_index(const LeafValueType& leaf, + bool includeUncommitted, + const AppendOnlyTree::FindLeafCallback& on_completion) const; + using AppendOnlyTree::get_sibling_path; private: + using AppendOnlyTree::find_leaf_index_from; using typename AppendOnlyTree::AppendCompletionCallback; using ReadTransaction = typename Store::ReadTransaction; using ReadTransactionPtr = typename Store::ReadTransactionPtr; @@ -120,6 +125,7 @@ template class IndexedTree : public App using AppendOnlyTree::add_value; using AppendOnlyTree::add_values; + using AppendOnlyTree::add_values_internal; using AppendOnlyTree::store_; using AppendOnlyTree::zero_hashes_; @@ -179,7 +185,7 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers success = result.success; signal.signal_level(0); }; - AppendOnlyTree::add_values(appended_hashes, completion); + AppendOnlyTree::add_values_internal(appended_hashes, completion, false); signal.wait_for_level(0); if (!success) { throw std::runtime_error("Failed to initialise tree"); @@ -203,6 +209,27 @@ void IndexedTree::get_leaf(const index_t& index, workers_.enqueue(job); } +template +void IndexedTree::find_leaf_index( + const LeafValueType& leaf, + bool includeUncommitted, + const AppendOnlyTree::FindLeafCallback& on_completion) const +{ + auto job = [=]() -> void { + ExecuteAndReport( + [=](TypedResponse& response) { + typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); + std::optional leaf_index = store_.find_leaf_index(leaf, *tx, includeUncommitted); + response.success = leaf_index.has_value(); + if (response.success) { + response.inner.leaf_index = leaf_index.value(); + } + }, + on_completion); + }; + workers_.enqueue(job); +} + template void IndexedTree::add_or_update_value(const LeafValueType& value, const AddCompletionCallback& completion) @@ -275,7 +302,8 @@ void IndexedTree::add_or_update_values(const std::vectorstatus.message); return; } - AppendOnlyTree::add_values((*results->hashes_to_append), final_completion); + AppendOnlyTree::add_values_internal( + (*results->hashes_to_append), final_completion, false); } }; @@ -293,7 +321,8 @@ void IndexedTree::add_or_update_values(const std::vectorstatus.message); return; } - AppendOnlyTree::add_values((*results->hashes_to_append), final_completion); + AppendOnlyTree::add_values_internal( + (*results->hashes_to_append), final_completion, false); } }; @@ -447,7 +476,8 @@ void IndexedTree::generate_insertions( IndexedLeafValueType(value_pair.first, current_leaf.nextIndex, current_leaf.nextValue); store_.set_at_index(current, replacement_leaf, false); IndexedLeafValueType empty_leaf = IndexedLeafValueType::empty(); - store_.set_at_index(index_of_new_leaf, empty_leaf, true); + // don't update the index for this empty leaf + store_.set_at_index(index_of_new_leaf, empty_leaf, false); // The set of appended leaves already has an empty leaf in the slot at index // 'index_into_appended_leaves' } @@ -501,8 +531,7 @@ void IndexedTree::update_leaf_and_hash_to_root(const index if (level > 1) { // Level is > 1. Therefore we need to wait for our leader to have written to the level above meaning we can // read from it - uint32_t level_to_read = level - 1; - leader_level = level_to_read; + leader_level = level - 1; leader.wait_for_level(leader_level); // Now read the node and it's sibling diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index bd88b943210..34f91d42864 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -123,6 +123,26 @@ IndexedLeaf get_leaf(IndexedTree +void check_find_leaf_index(TreeType& tree, + const LeafValueType& leaf, + index_t expected_index, + bool expected_success, + bool includeUncommitted = true) +{ + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, expected_success); + if (expected_success) { + EXPECT_EQ(response.inner.leaf_index, expected_index); + } + signal.signal_level(); + }; + + tree.find_leaf_index(leaf, includeUncommitted, completion); + signal.wait_for_level(); +} + template void check_sibling_path(IndexedTree, Poseidon2HashPolicy>& tree, index_t index, @@ -252,7 +272,7 @@ TEST_F(PersistedIndexedTreeTest, reports_an_error_if_tree_is_overfilled) TEST_F(PersistedIndexedTreeTest, test_get_sibling_path) { index_t current_size = 2; - NullifierMemoryTree memdb(10); + NullifierMemoryTree memdb(10, current_size); ThreadPool workers(1); constexpr size_t depth = 10; @@ -268,6 +288,12 @@ TEST_F(PersistedIndexedTreeTest, test_get_sibling_path) memdb.update_element(VALUES[512]); add_value(tree, NullifierLeafValue(VALUES[512])); + // std::cout << memdb.get_sibling_path(0) << std::endl; + // std::cout << memdb.get_hash_path(0) << std::endl; + + // std::cout << get_sibling_path(tree, 0, true) << std::endl; + // std::cout << get_sibling_path(tree, 1, true) << std::endl; + check_size(tree, ++current_size); check_sibling_path(tree, 0, memdb.get_sibling_path(0)); check_sibling_path(tree, 1, memdb.get_sibling_path(1)); @@ -283,6 +309,64 @@ TEST_F(PersistedIndexedTreeTest, test_get_sibling_path) check_sibling_path(tree, 512, memdb.get_sibling_path(512)); } +TEST_F(PersistedIndexedTreeTest, test_find_leaf_index) +{ + index_t initial_size = 2; + ThreadPool workers(1); + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, initial_size); + + add_value(tree, NullifierLeafValue(30)); + add_value(tree, NullifierLeafValue(10)); + add_value(tree, NullifierLeafValue(20)); + add_value(tree, NullifierLeafValue(40)); + + // check the committed state and that the uncommitted state is empty + check_find_leaf_index(tree, NullifierLeafValue(10), 1 + initial_size, true, true); + check_find_leaf_index(tree, NullifierLeafValue(10), 0, false, false); + + check_find_leaf_index(tree, NullifierLeafValue(15), 0, false, true); + check_find_leaf_index(tree, NullifierLeafValue(15), 0, false, false); + + check_find_leaf_index(tree, NullifierLeafValue(40), 3 + initial_size, true, true); + check_find_leaf_index(tree, NullifierLeafValue(30), 0 + initial_size, true, true); + check_find_leaf_index(tree, NullifierLeafValue(20), 2 + initial_size, true, true); + + check_find_leaf_index(tree, NullifierLeafValue(40), 0, false, false); + check_find_leaf_index(tree, NullifierLeafValue(30), 0, false, false); + check_find_leaf_index(tree, NullifierLeafValue(20), 0, false, false); + + commit_tree(tree); + + std::vector values{ NullifierLeafValue(15), + NullifierLeafValue(18), + NullifierLeafValue(26), + NullifierLeafValue(2), + NullifierLeafValue(48) }; + add_values(tree, values); + + // check the now committed state + check_find_leaf_index(tree, NullifierLeafValue(40), 3 + initial_size, true, false); + check_find_leaf_index(tree, NullifierLeafValue(30), 0 + initial_size, true, false); + check_find_leaf_index(tree, NullifierLeafValue(20), 2 + initial_size, true, false); + + // check the new uncommitted state + check_find_leaf_index(tree, NullifierLeafValue(18), 5 + initial_size, true, true); + check_find_leaf_index(tree, NullifierLeafValue(18), 0, false, false); + + commit_tree(tree); + + values = { NullifierLeafValue(16), NullifierLeafValue(4), NullifierLeafValue(22), NullifierLeafValue(101) }; + add_values(tree, values); + + // we now have duplicate leaf 18, one committed the other not + check_find_leaf_index(tree, NullifierLeafValue(18), 5 + initial_size, true, true); + check_find_leaf_index(tree, NullifierLeafValue(18), 5 + initial_size, true, false); +} + TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) { NullifierMemoryTree memdb(10); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp index 232f6d0f65a..c7873165f7a 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp @@ -3,11 +3,14 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/numeric/uint256/uint256.hpp" +#include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" #include "msgpack/assert.hpp" #include #include #include +#include #include #include @@ -50,6 +53,8 @@ template class CachedTreeStore void set_at_index(const index_t& index, const IndexedLeafValueType& leaf, bool add_to_index); + void update_index(const index_t& index, const fr& leaf); + void put_node(uint32_t level, index_t index, const std::vector& data); bool get_node(uint32_t level, @@ -69,6 +74,15 @@ template class CachedTreeStore ReadTransaction& tx, bool includeUncommitted) const; + std::optional find_leaf_index(const LeafValueType& leaf, + ReadTransaction& tx, + bool includeUncommitted) const; + + std::optional find_leaf_index_from(const LeafValueType& leaf, + index_t start_index, + ReadTransaction& tx, + bool includeUncommitted) const; + void commit(); void rollback(); @@ -76,10 +90,20 @@ template class CachedTreeStore ReadTransactionPtr createReadTransaction() const { return dataStore.createReadTransaction(); } private: + struct Indices { + std::vector indices; + + // Indices(index_t index) + // : indices{ index } + // {} + + MSGPACK_FIELDS(indices); + }; + std::string name; uint32_t depth; std::vector>> nodes; - std::map indices_; + std::map indices_; std::unordered_map leaves_; PersistedStore& dataStore; TreeMeta meta; @@ -102,7 +126,9 @@ std::pair CachedTreeStore::find_lo std::vector data; FrKeyType key(new_value.get_key()); tx.get_value_or_previous(key, data); - auto db_index = from_buffer(data, 0); + Indices committed; + msgpack::unpack((const char*)data.data(), data.size()).get().convert(committed); + auto db_index = committed.indices[0]; uint256_t retrieved_value = key; if (!includeUncommitted || retrieved_value == new_value_as_number || indices_.empty()) { return std::make_pair(new_value_as_number == retrieved_value, db_index); @@ -116,12 +142,12 @@ std::pair CachedTreeStore::find_lo --it; // we need to return the larger of the db value or the cached value - return std::make_pair(false, it->first > retrieved_value ? it->second : db_index); + return std::make_pair(false, it->first > retrieved_value ? it->second.indices[0] : db_index); } if (it->first == uint256_t(new_value_as_number)) { // the value is already present and the iterator points to it - return std::make_pair(true, it->second); + return std::make_pair(true, it->second.indices[0]); } // the iterator points to the element immediately larger than the requested value // We need to return the highest value from @@ -133,7 +159,7 @@ std::pair CachedTreeStore::find_lo } --it; // it now points to the value less than that requested - return std::make_pair(false, it->first > retrieved_value ? it->second : db_index); + return std::make_pair(false, it->first > retrieved_value ? it->second.indices[0] : db_index); } template @@ -164,8 +190,80 @@ void CachedTreeStore::set_at_index(const index_t& { leaves_[index] = leaf; if (add_to_index) { - indices_[uint256_t(leaf.value.get_key())] = index; + auto it = indices_.find(uint256_t(leaf.value.get_key())); + if (it == indices_.end()) { + Indices indices; + indices.indices.push_back(index); + indices_[uint256_t(leaf.value.get_key())] = indices; + return; + } + it->second.indices.push_back(index); + } +} + +template +void CachedTreeStore::update_index(const index_t& index, const fr& leaf) +{ + auto it = indices_.find(uint256_t(leaf)); + if (it == indices_.end()) { + Indices indices; + indices.indices.push_back(index); + indices_[uint256_t(leaf)] = indices; + return; } + it->second.indices.push_back(index); +} + +template +std::optional CachedTreeStore::find_leaf_index(const LeafValueType& leaf, + ReadTransaction& tx, + bool includeUncommitted) const +{ + return find_leaf_index_from(leaf, 0, tx, includeUncommitted); +} + +template +std::optional CachedTreeStore::find_leaf_index_from( + const LeafValueType& leaf, index_t start_index, ReadTransaction& tx, bool includeUncommitted) const +{ + Indices committed; + std::optional result = std::nullopt; + FrKeyType key = leaf; + std::vector value; + bool success = tx.get_value_by_integer(key, value); + if (success) { + msgpack::unpack((const char*)value.data(), value.size()).get().convert(committed); + if (!committed.indices.empty()) { + for (size_t i = 0; i < committed.indices.size(); ++i) { + index_t ind = committed.indices[i]; + if (ind < start_index) { + continue; + } + if (!result.has_value()) { + result = ind; + continue; + } + result = std::min(ind, result.value()); + } + } + } + if (includeUncommitted) { + auto it = indices_.find(uint256_t(leaf)); + if (it != indices_.end() && !it->second.indices.empty()) { + for (size_t i = 0; i < it->second.indices.size(); ++i) { + index_t ind = it->second.indices[i]; + if (ind < start_index) { + continue; + } + if (!result.has_value()) { + result = ind; + continue; + } + result = std::min(ind, result.value()); + } + } + } + return result; } template @@ -240,6 +338,20 @@ void CachedTreeStore::get_full_meta( template void CachedTreeStore::commit() { { + { + ReadTransactionPtr tx = createReadTransaction(); + for (auto& idx : indices_) { + std::vector value; + FrKeyType key = idx.first; + bool success = tx->get_value_by_integer(key, value); + if (success) { + Indices indices; + msgpack::unpack((const char*)value.data(), value.size()).get().convert(indices); + idx.second.indices.insert( + idx.second.indices.begin(), indices.indices.begin(), indices.indices.end()); + } + } + } WriteTransactionPtr tx = createWriteTransaction(); try { for (uint32_t i = 1; i < nodes.size(); i++) { @@ -251,10 +363,11 @@ template void CachedTreeStore< } } for (auto& idx : indices_) { - std::vector value; - write(value, idx.second); + msgpack::sbuffer buffer; + msgpack::pack(buffer, idx.second); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); FrKeyType key = idx.first; - tx->put_value_by_integer(key, value); + tx->put_value_by_integer(key, encoded); } for (const auto& leaf : leaves_) { msgpack::sbuffer buffer; @@ -278,7 +391,7 @@ void CachedTreeStore::rollback() { nodes = std::vector>>( depth + 1, std::unordered_map>()); - indices_ = std::map(); + indices_ = std::map(); leaves_ = std::unordered_map(); ReadTransactionPtr tx = createReadTransaction(); readPersistedMeta(meta, *tx); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 7b37fb38b04..603fc9205f7 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -30,6 +30,10 @@ struct AddIndexedDataResponse { std::shared_ptr> paths; }; +struct FindLeafIndexResponse { + index_t leaf_index; +}; + template struct GetIndexedLeafResponse { IndexedLeaf indexed_leaf; }; From 927b5f294d75b5c4c54d7c6951228df1b591929b Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 5 Jul 2024 02:34:21 +0100 Subject: [PATCH 33/63] Minor change --- .../merkle_tree/append_only_tree/append_only_tree.hpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index c2bbf35eaa7..3d4bb16c6c3 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -201,15 +201,7 @@ template void AppendOnlyTree::add_values(const std::vector& values, const AppendCompletionCallback& on_completion) { - std::shared_ptr> hashes = std::make_shared>(values); - auto append_op = [=, this]() -> void { - ExecuteAndReport( - [=](TypedResponse& response) { - add_values_internal(hashes, response.inner.root, response.inner.size, true); - }, - on_completion); - }; - workers_.enqueue(append_op); + add_values_internal(values, on_completion, true); } template From 95d8daf9a84b079607f13c4e716336d960f2b291 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Tue, 9 Jul 2024 08:51:03 +0000 Subject: [PATCH 34/63] fix: upcast when building lmdb key --- .../crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp | 4 ++-- .../crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp index c8bcab01d31..63f3bd6bcd8 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp @@ -34,7 +34,7 @@ bool LMDBReadTransaction::get_value(std::vector& key, std::vector& data) const { - NodeKeyType key = (static_cast(1 << level) + static_cast(index)) - 1; + NodeKeyType key = ((static_cast(1) << level) + static_cast(index)) - 1; return get_value_by_integer(key, data); } -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp index 5fead3690cf..aae18f49eaa 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp @@ -33,7 +33,7 @@ void LMDBWriteTransaction::try_abort() void LMDBWriteTransaction::put_node(uint32_t level, index_t index, std::vector& data) { - NodeKeyType key = (static_cast(1 << level) + static_cast(index)) - 1; + NodeKeyType key = ((static_cast(1) << level) + static_cast(index)) - 1; put_value_by_integer(key, data); } @@ -48,4 +48,4 @@ void LMDBWriteTransaction::put_value(std::vector& key, std::vector Date: Tue, 9 Jul 2024 08:51:41 +0000 Subject: [PATCH 35/63] feat: add signal_decrement --- .../cpp/src/barretenberg/crypto/merkle_tree/signal.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp index 97afff9e869..b9ff1e7a201 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp @@ -51,7 +51,13 @@ class Signal { signal_.notify_all(); } + void signal_decrement(uint32_t delta = 1) + { + signal_.fetch_sub(delta); + signal_.notify_all(); + } + private: std::atomic signal_; }; -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree From 5c92de4211d986a89b1bf488abadece0523d58da Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Tue, 9 Jul 2024 09:11:17 +0000 Subject: [PATCH 36/63] fix: return tree depth --- .../crypto/merkle_tree/append_only_tree/append_only_tree.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 3d4bb16c6c3..0406e1521c8 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -132,6 +132,7 @@ void AppendOnlyTree::get_meta_data(bool includeUncommitted [=, this](TypedResponse& response) { ReadTransactionPtr tx = store_.createReadTransaction(); store_.get_meta(response.inner.size, response.inner.root, *tx, includeUncommitted); + response.inner.depth = depth_; }, on_completion); }; From c19b6db1767dad12d212b9a37e079f71f280630f Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Tue, 9 Jul 2024 09:12:47 +0000 Subject: [PATCH 37/63] feat: get_leaf returns optional --- .../append_only_tree/append_only_tree.hpp | 32 +++++++++++++++--- .../append_only_tree.test.cpp | 33 +++++++++++++++++++ .../merkle_tree/indexed_tree/indexed_tree.hpp | 10 +++--- .../indexed_tree/indexed_tree.test.cpp | 6 ++-- .../node_store/cached_tree_store.hpp | 12 ++++--- .../crypto/merkle_tree/response.hpp | 7 +++- 6 files changed, 84 insertions(+), 16 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 0406e1521c8..4cea1ca9b96 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -24,10 +24,12 @@ using namespace bb; */ template class AppendOnlyTree { public: + using StoreType = Store; using AppendCompletionCallback = std::function&)>; using MetaDataCallback = std::function&)>; using HashPathCallback = std::function&)>; using FindLeafCallback = std::function&)>; + using GetLeafCallback = std::function&)>; using CommitCallback = std::function; using RollbackCallback = std::function; @@ -55,6 +57,8 @@ template class AppendOnlyTree { void get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion) const; + void get_leaf(const index_t& index, bool includeUncommitted, const GetLeafCallback& completion) const; + void find_leaf_index(const fr& leaf, bool includeUncommitted, const FindLeafCallback& on_completion) const; void find_leaf_index_from(const fr& leaf, @@ -162,6 +166,26 @@ void AppendOnlyTree::get_sibling_path(const index_t& index workers_.enqueue(job); } +template +void AppendOnlyTree::get_leaf(const index_t& index, + bool includeUncommitted, + const GetLeafCallback& on_completion) const +{ + auto job = [=, this]() { + ExecuteAndReport( + [=, this](TypedResponse& response) { + ReadTransactionPtr tx = store_.createReadTransaction(); + auto leaf = read_node(depth_, index, *tx, includeUncommitted); + response.success = leaf.first; + if (leaf.first) { + response.inner.leaf = leaf.second; + } + }, + on_completion); + }; + workers_.enqueue(job); +} + template void AppendOnlyTree::find_leaf_index(const fr& leaf, bool includeUncommitted, @@ -176,9 +200,9 @@ void AppendOnlyTree::find_leaf_index_from(const fr& leaf, bool includeUncommitted, const FindLeafCallback& on_completion) const { - auto job = [=]() -> void { + auto job = [=, this]() -> void { ExecuteAndReport( - [=](TypedResponse& response) { + [=, this](TypedResponse& response) { typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); std::optional leaf_index = store_.find_leaf_index_from(leaf, start_index, *tx, includeUncommitted); @@ -211,9 +235,9 @@ void AppendOnlyTree::add_values_internal(const std::vector bool update_index) { std::shared_ptr> hashes = std::make_shared>(values); - auto append_op = [=]() -> void { + auto append_op = [=, this]() -> void { ExecuteAndReport( - [=](TypedResponse& response) { + [=, this](TypedResponse& response) { add_values_internal(hashes, response.inner.root, response.inner.size, update_index); }, on_completion); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp index a97758c3f50..36e0af2ee34 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -164,6 +164,20 @@ void check_find_leaf_index_from(TreeType& tree, signal.wait_for_level(); } +void check_leaf( + TreeType& tree, const fr& leaf, index_t leaf_index, bool expected_success, bool includeUncommitted = true) +{ + Signal signal; + tree.get_leaf(leaf_index, includeUncommitted, [&](const TypedResponse& response) { + EXPECT_EQ(response.success, expected_success); + if (expected_success) { + EXPECT_EQ(response.inner.leaf, leaf); + } + signal.signal_level(); + }); + signal.wait_for_level(); +} + TEST_F(PersistedAppendOnlyTreeTest, can_create) { constexpr size_t depth = 10; @@ -609,3 +623,22 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_single_whilst_reading) EXPECT_TRUE(path == initial_path || path == final_sibling_path); } } + +TEST_F(PersistedAppendOnlyTreeTest, can_get_inserted_leaves) +{ + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + ThreadPool pool(1); + TreeType tree(store, pool); + + add_values(tree, { 30, 10, 20, 40 }); + + check_leaf(tree, 30, 0, true); + check_leaf(tree, 10, 1, true); + check_leaf(tree, 20, 2, true); + check_leaf(tree, 40, 3, true); + + check_leaf(tree, 0, 4, false); +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 88fd6ff5399..da8d31f0214 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ namespace bb::crypto::merkle_tree { */ template class IndexedTree : public AppendOnlyTree { public: + using StoreType = Store; using LeafValueType = typename Store::LeafType; using IndexedLeafValueType = typename Store::IndexedLeafValueType; using AddCompletionCallback = std::function&)>; @@ -56,7 +58,7 @@ template class IndexedTree : public App */ void add_or_update_values(const std::vector& values, const AddCompletionCallback& completion); - void get_leaf(const index_t& index, bool includeUncommitted, const LeafCallback& completion); + void get_leaf(const index_t& index, bool includeUncommitted, const LeafCallback& completion) const; void find_leaf_index(const LeafValueType& leaf, bool includeUncommitted, @@ -196,7 +198,7 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers template void IndexedTree::get_leaf(const index_t& index, bool includeUncommitted, - const LeafCallback& completion) + const LeafCallback& completion) const { auto job = [=, this]() { ExecuteAndReport>( @@ -457,9 +459,9 @@ void IndexedTree::generate_insertions( index_t current = 0; bool is_already_present = false; std::tie(is_already_present, current) = store_.find_low_value(value_pair.first, true, *tx); - IndexedLeafValueType current_leaf = store_.get_leaf(current, *tx, true); + // .value() throws if the low leaf does not exist + IndexedLeafValueType current_leaf = store_.get_leaf(current, *tx, true).value(); - // We only handle new values being added. We don't yet handle values being updated if (!is_already_present) { // Update the current leaf to point it to the new leaf IndexedLeafValueType new_leaf = diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index 34f91d42864..c39bb7a306e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -112,7 +112,7 @@ IndexedLeaf get_leaf(IndexedTree l; + std::optional> l; Signal signal; auto completion = [&](const TypedResponse>& leaf) -> void { l = leaf.inner.indexed_leaf; @@ -120,7 +120,7 @@ IndexedLeaf get_leaf(IndexedTree @@ -969,4 +969,4 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory_with_public_data_writes) e0, }; check_sibling_path(tree, 7, expected); -} \ No newline at end of file +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp index 9627b63a19c..566ac6d52e9 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp @@ -21,6 +21,7 @@ namespace bb::crypto::merkle_tree { */ template class CachedTreeStore { public: + using PersistedStoreType = PersistedStore; using LeafType = LeafValueType; using IndexedLeafValueType = IndexedLeaf; using ReadTransaction = typename PersistedStore::ReadTransaction; @@ -49,7 +50,9 @@ template class CachedTreeStore bool includeUncommitted, ReadTransaction& tx) const; - IndexedLeafValueType get_leaf(const index_t& index, ReadTransaction& tx, bool includeUncommitted) const; + std::optional get_leaf(const index_t& index, + ReadTransaction& tx, + bool includeUncommitted) const; void set_at_index(const index_t& index, const IndexedLeafValueType& leaf, bool add_to_index); @@ -163,7 +166,7 @@ std::pair CachedTreeStore::find_lo } template -typename CachedTreeStore::IndexedLeafValueType CachedTreeStore< +std::optional::IndexedLeafValueType> CachedTreeStore< PersistedStore, LeafValueType>::get_leaf(const index_t& index, ReadTransaction& tx, bool includeUncommitted) const { @@ -173,14 +176,15 @@ typename CachedTreeStore::IndexedLeafValueType Ca return it->second; } } - IndexedLeafValueType return_value; LeafIndexKeyType key = index; std::vector data; bool success = tx.get_value_by_integer(key, data); if (success) { + IndexedLeafValueType return_value; msgpack::unpack((const char*)data.data(), data.size()).get().convert(return_value); + return return_value; } - return return_value; + return std::nullopt; } template diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 6cdc2a2336c..c37730b4d42 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -6,6 +6,7 @@ #include "barretenberg/ecc/curves/bn254/fr.hpp" #include #include +#include #include namespace bb::crypto::merkle_tree { @@ -34,8 +35,12 @@ struct FindLeafIndexResponse { index_t leaf_index; }; +struct GetLeafResponse { + std::optional leaf; +}; + template struct GetIndexedLeafResponse { - IndexedLeaf indexed_leaf; + std::optional> indexed_leaf; }; template struct TypedResponse { From e9117e02591531890f210ae438dae5737106e05c Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Tue, 9 Jul 2024 09:31:59 +0000 Subject: [PATCH 38/63] feat: get_low_leaf --- .../merkle_tree/indexed_tree/indexed_tree.hpp | 21 +++++++-- .../indexed_tree/indexed_tree.test.cpp | 43 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index da8d31f0214..76935eea550 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -64,10 +64,11 @@ template class IndexedTree : public App bool includeUncommitted, const AppendOnlyTree::FindLeafCallback& on_completion) const; + void find_low_leaf(const LeafValueType& leaf, bool includeUncommitted, const LeafCallback& on_completion) const; + using AppendOnlyTree::get_sibling_path; private: - using AppendOnlyTree::find_leaf_index_from; using typename AppendOnlyTree::AppendCompletionCallback; using ReadTransaction = typename Store::ReadTransaction; using ReadTransactionPtr = typename Store::ReadTransactionPtr; @@ -217,9 +218,9 @@ void IndexedTree::find_leaf_index( bool includeUncommitted, const AppendOnlyTree::FindLeafCallback& on_completion) const { - auto job = [=]() -> void { + auto job = [=, this]() -> void { ExecuteAndReport( - [=](TypedResponse& response) { + [=, this](TypedResponse& response) { typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); std::optional leaf_index = store_.find_leaf_index(leaf, *tx, includeUncommitted); response.success = leaf_index.has_value(); @@ -232,6 +233,20 @@ void IndexedTree::find_leaf_index( workers_.enqueue(job); } +template +void IndexedTree::find_low_leaf(const LeafValueType& leaf, + bool includeUncommitted, + const LeafCallback& on_completion) const +{ + auto job = [=, &leaf, this]() { + typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); + auto result = store_.find_low_value(leaf, includeUncommitted, *tx); + get_leaf(result.second, includeUncommitted, on_completion); + }; + + workers_.enqueue(job); +} + template void IndexedTree::add_or_update_value(const LeafValueType& value, const AddCompletionCallback& completion) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index c39bb7a306e..a185609f8a8 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -123,6 +123,23 @@ IndexedLeaf get_leaf(IndexedTree +IndexedLeaf get_low_leaf( + IndexedTree, Poseidon2HashPolicy>& tree, + const LeafValueType& leaf, + bool includeUncommitted = true) +{ + std::optional> l; + Signal signal; + auto completion = [&](const TypedResponse>& leaf) -> void { + l = leaf.inner.indexed_leaf; + signal.signal_level(); + }; + tree.find_low_leaf(leaf, includeUncommitted, completion); + signal.wait_for_level(); + return l.value(); +} + template void check_find_leaf_index(TreeType& tree, const LeafValueType& leaf, @@ -970,3 +987,29 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory_with_public_data_writes) }; check_sibling_path(tree, 7, expected); } + +TEST_F(PersistedIndexedTreeTest, returns_low_leaves) +{ + // Create a depth-8 indexed merkle tree + constexpr uint32_t depth = 8; + + ThreadPool workers(1); + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, 2); + + auto predecessor = get_low_leaf(tree, NullifierLeafValue(42)); + + EXPECT_EQ(predecessor.value, NullifierLeafValue(1)); + EXPECT_EQ(predecessor.nextIndex, 0); + EXPECT_EQ(predecessor.nextValue, 0); + + add_value(tree, NullifierLeafValue(42)); + + predecessor = get_low_leaf(tree, NullifierLeafValue(42)); + // returns the current leaf since it exists already. Inserting 42 again would modify the existing leaf + EXPECT_EQ(predecessor.value, NullifierLeafValue(42)); + EXPECT_EQ(predecessor.nextIndex, 0); + EXPECT_EQ(predecessor.nextValue, 0); +} From 61e430917267855e06a61dc324a87b8c74b5e2d5 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 10 Jul 2024 08:08:46 +0100 Subject: [PATCH 39/63] feat: define find_leaf_index_from on indexed tree --- .../merkle_tree/indexed_tree/indexed_tree.hpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 76935eea550..122f1b5cba4 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -64,6 +64,11 @@ template class IndexedTree : public App bool includeUncommitted, const AppendOnlyTree::FindLeafCallback& on_completion) const; + void find_leaf_index_from(const LeafValueType& leaf, + index_t start_index, + bool includeUncommitted, + const AppendOnlyTree::FindLeafCallback& on_completion) const; + void find_low_leaf(const LeafValueType& leaf, bool includeUncommitted, const LeafCallback& on_completion) const; using AppendOnlyTree::get_sibling_path; @@ -217,12 +222,23 @@ void IndexedTree::find_leaf_index( const LeafValueType& leaf, bool includeUncommitted, const AppendOnlyTree::FindLeafCallback& on_completion) const +{ + find_leaf_index_from(leaf, 0, includeUncommitted, on_completion); +} + +template +void IndexedTree::find_leaf_index_from( + const LeafValueType& leaf, + index_t start_index, + bool includeUncommitted, + const AppendOnlyTree::FindLeafCallback& on_completion) const { auto job = [=, this]() -> void { ExecuteAndReport( [=, this](TypedResponse& response) { typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); - std::optional leaf_index = store_.find_leaf_index(leaf, *tx, includeUncommitted); + std::optional leaf_index = + store_.find_leaf_index_from(leaf, start_index, *tx, includeUncommitted); response.success = leaf_index.has_value(); if (response.success) { response.inner.leaf_index = leaf_index.value(); From b29f33bccabdd123c783cc16622e7acb52723141 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 10 Jul 2024 10:08:39 +0000 Subject: [PATCH 40/63] feat: return low leaf witness data after insertion --- .../merkle_tree/indexed_tree/indexed_tree.hpp | 59 ++++++++++++------- .../indexed_tree/indexed_tree.test.cpp | 40 ++++++++----- .../crypto/merkle_tree/response.hpp | 12 +++- 3 files changed, 71 insertions(+), 40 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 122f1b5cba4..5e5e56a94a3 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -33,7 +33,7 @@ template class IndexedTree : public App using StoreType = Store; using LeafValueType = typename Store::LeafType; using IndexedLeafValueType = typename Store::IndexedLeafValueType; - using AddCompletionCallback = std::function&)>; + using AddCompletionCallback = std::function>&)>; using LeafCallback = std::function>&)>; IndexedTree(Store& store, ThreadPool& workers, index_t initial_size); @@ -92,7 +92,7 @@ template class IndexedTree : public App struct LeafInsertion { index_t low_leaf_index; - IndexedLeafValueType low_leaf; + IndexedLeafValueType low_leaf, original_low_leaf; }; void update_leaf_and_hash_to_root(const index_t& index, @@ -112,7 +112,7 @@ template class IndexedTree : public App const InsertionGenerationCallback& completion); struct InsertionCompletionResponse { - std::shared_ptr> paths; + std::shared_ptr>> low_leaf_witness_data; }; using InsertionCompletionCallback = std::function&)>; @@ -283,8 +283,10 @@ void IndexedTree::add_or_update_values(const std::vector> hashes_to_append; - std::shared_ptr> paths; + // info about the low leaves that have been updated + std::shared_ptr>> low_leaf_witness_data; std::atomic count; Status status; @@ -301,7 +303,7 @@ void IndexedTree::add_or_update_values(const std::vector response; + TypedResponse> response; response.success = false; response.message = message; completion(response); @@ -311,12 +313,13 @@ void IndexedTree::add_or_update_values(const std::vector& add_result) { - TypedResponse response; + TypedResponse> response; response.success = add_result.success; response.message = add_result.message; if (add_result.success) { response.inner.add_data_result = add_result.inner; - response.inner.paths = results->paths; + response.inner.sorted_leaves = values_to_be_sorted; + response.inner.low_leaf_witness_data = results->low_leaf_witness_data; } // Trigger the client's provided callback completion(response); @@ -347,7 +350,7 @@ void IndexedTree::add_or_update_values(const std::vectorstatus.set_failure(insertion_response.message); } else { - results->paths = insertion_response.inner.paths; + results->low_leaf_witness_data = insertion_response.inner.low_leaf_witness_data; } if (results->count.fetch_sub(1) == 1) { if (!results->status.success) { @@ -383,8 +386,9 @@ void IndexedTree::perform_insertions(std::shared_ptr> paths = - std::make_shared>(insertions->size()); + auto low_leaf_witness_data = std::make_shared>>( + insertions->size(), LowLeafWitnessData{ IndexedLeafValueType::empty(), 0, {} }); + std::shared_ptr> signals = std::make_shared>(); std::shared_ptr status = std::make_shared(); // The first signal is set to 0. This ensures the first worker up the tree is not impeded @@ -401,8 +405,15 @@ void IndexedTree::perform_insertions(std::shared_ptrat(i); + current_witness_data.leaf = insertion.original_low_leaf; + current_witness_data.index = insertion.low_leaf_index; + update_leaf_and_hash_to_root(insertion.low_leaf_index, + insertion.low_leaf, + leaderSignal, + followerSignal, + current_witness_data.path, + *tx); } catch (std::exception& e) { status->set_failure(e.what()); // ensure that any followers are not blocked by our failure @@ -413,7 +424,7 @@ void IndexedTree::perform_insertions(std::shared_ptrsuccess; response.message = status->message; if (response.success) { - response.inner.paths = paths; + response.inner.low_leaf_witness_data = low_leaf_witness_data; } completion(response); } @@ -444,8 +455,8 @@ void IndexedTree::generate_insertions( { ExecuteAndReport( [=, this](TypedResponse& response) { - // The first thing we do is sort the values into descending order but maintain knowledge of their orignal - // order + // The first thing we do is sort the values into descending order but maintain knowledge of their + // orignal order struct { bool operator()(std::pair& a, std::pair& b) const { @@ -493,10 +504,16 @@ void IndexedTree::generate_insertions( // .value() throws if the low leaf does not exist IndexedLeafValueType current_leaf = store_.get_leaf(current, *tx, true).value(); + LeafInsertion& insertion = (*response.inner.insertions)[i]; + // Capture the index and original value of the 'low' leaf + insertion.low_leaf_index = current; + insertion.original_low_leaf = current_leaf; + if (!is_already_present) { // Update the current leaf to point it to the new leaf IndexedLeafValueType new_leaf = IndexedLeafValueType(value_pair.first, current_leaf.nextIndex, current_leaf.nextValue); + current_leaf.nextIndex = index_of_new_leaf; current_leaf.nextValue = value; store_.set_at_index(current, current_leaf, false); @@ -516,9 +533,7 @@ void IndexedTree::generate_insertions( // 'index_into_appended_leaves' } - // Capture the index and value of the updated 'low' leaf as well as the new leaf to be appended - LeafInsertion& insertion = (*response.inner.insertions)[i]; - insertion.low_leaf_index = current; + // capture new low leaf insertion.low_leaf = IndexedLeafValueType(current_leaf.value, current_leaf.nextIndex, current_leaf.nextValue); } @@ -545,8 +560,8 @@ void IndexedTree::update_leaf_and_hash_to_root(const index uint32_t level = depth_; fr new_hash = HashingPolicy::hash(leaf.get_hash_inputs()); - // Wait until we see that our leader has cleared 'depth_ - 1' (i.e. the level above the leaves that we are about to - // write into) this ensures that our leader is not still reading the leaves + // Wait until we see that our leader has cleared 'depth_ - 1' (i.e. the level above the leaves that we are about + // to write into) this ensures that our leader is not still reading the leaves uint32_t leader_level = depth_ - 1; leader.wait_for_level(leader_level); @@ -563,8 +578,8 @@ void IndexedTree::update_leaf_and_hash_to_root(const index while (level > 0) { if (level > 1) { - // Level is > 1. Therefore we need to wait for our leader to have written to the level above meaning we can - // read from it + // Level is > 1. Therefore we need to wait for our leader to have written to the level above meaning we + // can read from it leader_level = level - 1; leader.wait_for_level(leader_level); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index a185609f8a8..e4c8d0d5473 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -187,7 +187,9 @@ void add_value(IndexedTree, Poseidon2H const LeafValueType& value) { Signal signal; - auto completion = [&](const TypedResponse&) -> void { signal.signal_level(); }; + auto completion = [&](const TypedResponse>&) -> void { + signal.signal_level(); + }; tree.add_or_update_value(value, completion); signal.wait_for_level(); @@ -198,7 +200,9 @@ void add_values(IndexedTree, Poseidon2 const std::vector& values) { Signal signal; - auto completion = [&](const TypedResponse&) -> void { signal.signal_level(); }; + auto completion = [&](const TypedResponse>&) -> void { + signal.signal_level(); + }; tree.add_or_update_values(values, completion); signal.wait_for_level(); @@ -277,7 +281,7 @@ TEST_F(PersistedIndexedTreeTest, reports_an_error_if_tree_is_overfilled) add_values(tree, values); Signal signal; - auto add_completion = [&](const TypedResponse& response) { + auto add_completion = [&](const TypedResponse>& response) { EXPECT_EQ(response.success, false); EXPECT_EQ(response.message, "Tree is full"); signal.signal_level(); @@ -482,23 +486,25 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) fr_sibling_path path = memdb.update_element(batch[j].value); memory_tree_sibling_paths.push_back(path); } - std::shared_ptr> tree1_sibling_paths; - std::shared_ptr> tree2_sibling_paths; + std::shared_ptr>> tree1_low_leaf_witness_data; + std::shared_ptr>> tree2_low_leaf_witness_data; { Signal signal; - CompletionCallback completion = [&](const TypedResponse& response) { - tree1_sibling_paths = response.inner.paths; - signal.signal_level(); - }; + CompletionCallback completion = + [&](const TypedResponse>& response) { + tree1_low_leaf_witness_data = response.inner.low_leaf_witness_data; + signal.signal_level(); + }; tree1.add_or_update_values(batch, completion); signal.wait_for_level(); } { Signal signal; - CompletionCallback completion = [&](const TypedResponse& response) { - tree2_sibling_paths = response.inner.paths; - signal.signal_level(); - }; + CompletionCallback completion = + [&](const TypedResponse>& response) { + tree2_low_leaf_witness_data = response.inner.low_leaf_witness_data; + signal.signal_level(); + }; tree2.add_or_update_values(batch, completion); signal.wait_for_level(); } @@ -512,7 +518,9 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) check_sibling_path(tree2, 512, memdb.get_sibling_path(512)); for (uint32_t j = 0; j < batch_size; j++) { - EXPECT_EQ(tree1_sibling_paths->at(j), tree2_sibling_paths->at(j)); + EXPECT_EQ(tree1_low_leaf_witness_data->at(j).leaf, tree2_low_leaf_witness_data->at(j).leaf); + EXPECT_EQ(tree1_low_leaf_witness_data->at(j).index, tree2_low_leaf_witness_data->at(j).index); + EXPECT_EQ(tree1_low_leaf_witness_data->at(j).path, tree2_low_leaf_witness_data->at(j).path); } } } @@ -534,7 +542,7 @@ TEST_F(PersistedIndexedTreeTest, reports_an_error_if_batch_contains_duplicate) values[8] = values[0]; Signal signal; - auto add_completion = [&](const TypedResponse& response) { + auto add_completion = [&](const TypedResponse>& response) { EXPECT_EQ(response.success, false); EXPECT_EQ(response.message, "Duplicate key not allowed in same batch"); signal.signal_level(); @@ -808,7 +816,7 @@ TEST_F(PersistedIndexedTreeTest, can_add_single_whilst_reading) Signal signal(2); - auto add_completion = [&](const TypedResponse&) { + auto add_completion = [&](const TypedResponse>&) { signal.signal_level(1); auto commit_completion = [&](const Response&) { signal.signal_level(); }; tree.commit(commit_completion); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index c37730b4d42..125076f8792 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -6,6 +6,7 @@ #include "barretenberg/ecc/curves/bn254/fr.hpp" #include #include +#include #include #include @@ -26,9 +27,16 @@ struct GetSiblingPathResponse { fr_sibling_path path; }; -struct AddIndexedDataResponse { +template struct LowLeafWitnessData { + IndexedLeaf leaf; + index_t index; + fr_sibling_path path; +}; + +template struct AddIndexedDataResponse { AddDataResponse add_data_result; - std::shared_ptr> paths; + std::shared_ptr>> sorted_leaves; + std::shared_ptr>> low_leaf_witness_data; }; struct FindLeafIndexResponse { From a47018ed30a64a4d995ee923b20b861c595accde Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 10 Jul 2024 11:51:47 +0000 Subject: [PATCH 41/63] feat: appending returns subtree path --- .../append_only_tree/append_only_tree.hpp | 29 ++++++++++++- .../append_only_tree.test.cpp | 42 +++++++++++++++++++ .../crypto/merkle_tree/response.hpp | 5 ++- 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 4cea1ca9b96..5bf5af7eefe 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -85,6 +85,9 @@ template class AppendOnlyTree { void add_values_internal(std::shared_ptr> values, fr& new_root, index_t& new_size, + fr& subtree_root, + index_t& subtree_root_index, + fr_sibling_path& subtree_path, bool update_index); void add_values_internal(const std::vector& values, @@ -238,7 +241,13 @@ void AppendOnlyTree::add_values_internal(const std::vector auto append_op = [=, this]() -> void { ExecuteAndReport( [=, this](TypedResponse& response) { - add_values_internal(hashes, response.inner.root, response.inner.size, update_index); + add_values_internal(hashes, + response.inner.root, + response.inner.size, + response.inner.subtree_root, + response.inner.subtree_root_index, + response.inner.subtree_path, + update_index); }, on_completion); }; @@ -249,6 +258,9 @@ template void AppendOnlyTree::add_values_internal(std::shared_ptr> values, fr& new_root, index_t& new_size, + fr& subtree_root, + index_t& subtree_root_index, + fr_sibling_path& subtree_path, bool update_index) { uint32_t start_level = depth_; @@ -289,11 +301,26 @@ void AppendOnlyTree::add_values_internal(std::shared_ptr 0) { + subtree_path.reserve(level - 1); // minus the tree root + } while (level > 0) { bool is_right = static_cast(index & 0x01); fr left_hash = is_right ? get_element_or_zero(level, index - 1, *tx, true) : new_hash; fr right_hash = is_right ? new_hash : get_element_or_zero(level, index + 1, *tx, true); new_hash = HashingPolicy::hash_pair(left_hash, right_hash); + + if (is_right) { + subtree_path.emplace_back(left_hash); + } else { + subtree_path.emplace_back(right_hash); + } + index >>= 1; --level; if (level > 0) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp index 36e0af2ee34..2a9f030ea59 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -178,6 +178,25 @@ void check_leaf( signal.wait_for_level(); } +void check_sibling_path(fr expected_root, fr node, index_t index, fr_sibling_path sibling_path) +{ + fr left, right, hash = node; + for (size_t i = 0; i < sibling_path.size(); ++i) { + if (index % 2 == 0) { + left = hash; + right = sibling_path[i]; + } else { + left = sibling_path[i]; + right = hash; + } + + hash = Poseidon2HashPolicy::hash_pair(left, right); + index >>= 1; + } + + EXPECT_EQ(hash, expected_root); +} + TEST_F(PersistedAppendOnlyTreeTest, can_create) { constexpr size_t depth = 10; @@ -642,3 +661,26 @@ TEST_F(PersistedAppendOnlyTreeTest, can_get_inserted_leaves) check_leaf(tree, 0, 4, false); } + +TEST_F(PersistedAppendOnlyTreeTest, append_leaves_returns_sibling_path) +{ + constexpr size_t depth = 10; + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + ThreadPool pool(1); + TreeType tree(store, pool); + + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, true); + check_sibling_path(response.inner.root, + response.inner.subtree_root, + response.inner.subtree_root_index, + response.inner.subtree_path); + signal.signal_level(); + }; + + tree.add_values({ 40, 30, 10, 20 }, completion); + signal.wait_for_level(); +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 125076f8792..6b0b6227b66 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -19,8 +19,9 @@ struct TreeMetaResponse { }; struct AddDataResponse { - index_t size; - fr root; + index_t size, subtree_root_index; + fr root, subtree_root; + fr_sibling_path subtree_path; }; struct GetSiblingPathResponse { From 476b2f13c296ee01eef807e212cdf80040633186 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 10 Jul 2024 17:44:05 +0000 Subject: [PATCH 42/63] chore: bring in lmdb as an external dep --- barretenberg/cpp/CMakeLists.txt | 1 + barretenberg/cpp/bootstrap.sh | 4 ---- barretenberg/cpp/cmake/lmdb.cmake | 22 +++++++++++++++++++ barretenberg/cpp/src/CMakeLists.txt | 2 +- .../crypto/merkle_tree/CMakeLists.txt | 9 ++++++-- .../barretenberg/world_state/CMakeLists.txt | 4 +--- .../service/world_state_service.hpp | 9 ++++---- .../world_state_service/CMakeLists.txt | 7 +++--- 8 files changed, 40 insertions(+), 18 deletions(-) create mode 100644 barretenberg/cpp/cmake/lmdb.cmake diff --git a/barretenberg/cpp/CMakeLists.txt b/barretenberg/cpp/CMakeLists.txt index f16b898bf6f..4f5a991ebb7 100644 --- a/barretenberg/cpp/CMakeLists.txt +++ b/barretenberg/cpp/CMakeLists.txt @@ -143,6 +143,7 @@ include(cmake/gtest.cmake) include(cmake/benchmark.cmake) include(cmake/module.cmake) include(cmake/msgpack.cmake) +include(cmake/lmdb.cmake) include(cmake/backward-cpp.cmake) if (WASM) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 2d301d6c6d1..e8ab085a3cc 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -57,10 +57,6 @@ echo "#################################" function build_native { cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert cmake --build --preset $PRESET --target bb -} - -function build_native { - cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert cmake --build --preset $PRESET --target world_state } diff --git a/barretenberg/cpp/cmake/lmdb.cmake b/barretenberg/cpp/cmake/lmdb.cmake new file mode 100644 index 00000000000..4f90fe3ee40 --- /dev/null +++ b/barretenberg/cpp/cmake/lmdb.cmake @@ -0,0 +1,22 @@ +include(ExternalProject) + +set(LMDB_PREFIX "${CMAKE_BINARY_DIR}/_deps/lmdb") +set(LMDB_INCLUDE "${LMDB_PREFIX}/src/lmdb/libraries/liblmdb") +set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a") + +ExternalProject_Add( + lmdb + PREFIX ${LMDB_PREFIX} + GIT_REPOSITORY "https://github.com/LMDB/lmdb.git" + GIT_TAG ddd0a773e2f44d38e4e31ec9ed81af81f4e4ccbb + BUILD_IN_SOURCE YES + CONFIGURE_COMMAND "" # No configure step + BUILD_COMMAND make -C libraries/liblmdb liblmdb.a + INSTALL_COMMAND "" + UPDATE_COMMAND "" # No update step + BUILD_BYPRODUCTS ${LMDB_LIB} ${LMDB_INCLUDE} +) + +add_library(lmdb-lib STATIC IMPORTED) +add_dependencies(lmdb-lib lmdb) +set_target_properties(lmdb-lib PROPERTIES IMPORTED_LOCATION ${LMDB_LIB}) diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index f506da7bd5d..e0ccb64603b 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -41,7 +41,7 @@ if(WASM) add_link_options(-Wl,--export-memory,--import-memory,--stack-first,-z,stack-size=1048576,--max-memory=4294967296) endif() -include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE} ${LMDB_INCLUDE}) # I feel this should be limited to ecc, however it's currently used in headers that go across libraries, # and there currently isn't an easy way to inherit the DDISABLE_ASM parameter. diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt index 24205228325..5ccfdaa89d5 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt @@ -1,8 +1,13 @@ barretenberg_module( crypto_merkle_tree - lmdb stdlib_primitives stdlib_blake3s stdlib_pedersen_hash crypto_poseidon2 -) \ No newline at end of file + lmdb-lib +) + +add_dependencies(crypto_merkle_tree lmdb) +add_dependencies(crypto_merkle_tree_objects lmdb) +add_dependencies(crypto_merkle_tree_tests lmdb) +add_dependencies(crypto_merkle_tree_test_objects lmdb) diff --git a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt index 27372850e8c..45e9ec63a6b 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt @@ -1,3 +1 @@ -barretenberg_module(world_state lmdb crypto_merkle_tree cryto_pedersen_hash) - - +barretenberg_module(world_state crypto_merkle_tree cryto_pedersen_hash) diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp index 3f45aa76d8c..533f973aedc 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp @@ -8,6 +8,7 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" #include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/cbind.hpp" #include "barretenberg/world_state/service/message.hpp" @@ -156,13 +157,13 @@ template bool WorldStateService::insertLea } else { // Send the response async - auto completion = [=](std::vector&, fr& root, index_t size) -> void { + auto completion = [=, this](const TypedResponse>& resp) -> void { MsgHeader header(requestId); InsertLeavesResponse response; response.message = ""; response.success = true; - response.root = root; - response.size = size; + response.root = resp.inner.add_data_result.root; + response.size = resp.inner.add_data_result.size; TypedMessage insertLeavesResponse( WorldStateMsgTypes::INSERT_LEAVES_RESPONSE, header, response); outputStream.sendPackedObject(insertLeavesResponse); @@ -176,4 +177,4 @@ template bool WorldStateService::insertLea return true; } -} // namespace bb::world_state \ No newline at end of file +} // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt index 7df0ff6f290..47037d3a303 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt @@ -1,5 +1,4 @@ -if (NOT(FUZZING)) +if(NOT(FUZZING)) add_executable(world_state main.cpp) - target_link_libraries(world_state lmdb crypto_merkle_tree crypto_pedersen_hash common) - - endif() + target_link_libraries(world_state lmdb-lib crypto_merkle_tree crypto_pedersen_hash common) +endif() From 4b0b007f045c25f0ce1199814bdea823f776db52 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 10 Jul 2024 18:03:08 +0000 Subject: [PATCH 43/63] chore: disable thread_pool in wasm --- barretenberg/cpp/src/CMakeLists.txt | 5 ++--- barretenberg/cpp/src/barretenberg/common/thread_pool.cpp | 6 +++++- barretenberg/cpp/src/barretenberg/common/thread_pool.hpp | 9 +++++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index e0ccb64603b..6ef11042e81 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -21,6 +21,7 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") message(STATUS "Compiling with memory checks.") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") endif() + if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 18) # We target clang18 and need this, eventually warning should be fixed or this will be unconditional. add_compile_options(-Wno-vla-cxx-extension) @@ -90,7 +91,6 @@ add_subdirectory(barretenberg/wasi) add_subdirectory(barretenberg/world_state) add_subdirectory(barretenberg/world_state_service) - if(SMT) add_subdirectory(barretenberg/smt_verification) endif() @@ -123,7 +123,7 @@ set(BARRETENBERG_TARGET_OBJECTS $ $ $ - $ + $ $ $ $ @@ -141,7 +141,6 @@ set(BARRETENBERG_TARGET_OBJECTS $ $ $ - $ $ $ $ diff --git a/barretenberg/cpp/src/barretenberg/common/thread_pool.cpp b/barretenberg/cpp/src/barretenberg/common/thread_pool.cpp index a046885e86b..588173e6919 100644 --- a/barretenberg/cpp/src/barretenberg/common/thread_pool.cpp +++ b/barretenberg/cpp/src/barretenberg/common/thread_pool.cpp @@ -1,4 +1,6 @@ +#ifndef NO_MULTITHREADING + #include "thread_pool.hpp" #include "barretenberg/common/log.hpp" namespace bb { @@ -70,4 +72,6 @@ void ThreadPool::worker_loop(size_t /*unused*/) } // info("worker exit ", worker_num); } -} // namespace bb \ No newline at end of file +} // namespace bb + +#endif diff --git a/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp b/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp index 017c4ab1f6f..945f8a0aa3b 100644 --- a/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp +++ b/barretenberg/cpp/src/barretenberg/common/thread_pool.hpp @@ -1,5 +1,8 @@ - #pragma once + +#ifndef NO_MULTITHREADING + +#include #include #include #include @@ -32,4 +35,6 @@ class ThreadPool { void worker_loop(size_t thread_index); }; -} // namespace bb \ No newline at end of file +} // namespace bb + +#endif From 9e20b9535ca8b850c1b765397ab269846574d8cb Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 16 Jul 2024 10:17:40 +0100 Subject: [PATCH 44/63] Fix sorting --- .../crypto/merkle_tree/indexed_tree/indexed_tree.hpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 5e5e56a94a3..b9df9d5d53f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -108,7 +108,7 @@ template class IndexedTree : public App }; using InsertionGenerationCallback = std::function&)>; - void generate_insertions(const std::shared_ptr>>& values_to_be_sorted, + void generate_insertions(const std::shared_ptr>>& values_to_be_sorted, const InsertionGenerationCallback& completion); struct InsertionCompletionResponse { @@ -450,7 +450,7 @@ void IndexedTree::generate_hashes_for_appending( template void IndexedTree::generate_insertions( - const std::shared_ptr>>& values_to_be_sorted, + const std::shared_ptr>>& values_to_be_sorted, const InsertionGenerationCallback& on_completion) { ExecuteAndReport( @@ -458,14 +458,16 @@ void IndexedTree::generate_insertions( // The first thing we do is sort the values into descending order but maintain knowledge of their // orignal order struct { - bool operator()(std::pair& a, std::pair& b) const + bool operator()(std::pair& a, std::pair& b) const { - return uint256_t(a.first.get_key()) > uint256_t(b.first.get_key()); + uint256_t aValue = a.first.get_key(); + uint256_t bValue = b.first.get_key(); + return aValue == bValue ? a.second < b.second : aValue < bValue; } } comp; std::sort(values_to_be_sorted->begin(), values_to_be_sorted->end(), comp); - std::vector>& values = *values_to_be_sorted; + std::vector>& values = *values_to_be_sorted; // Now that we have the sorted values we need to identify the leaves that need updating. // This is performed sequentially and is stored in this 'leaf_insertion' struct From c9a1b74ed1fee1d5ccabd911edeef50354626912 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Tue, 16 Jul 2024 14:35:03 +0000 Subject: [PATCH 45/63] fix: tree compatibility --- barretenberg/cpp/build_and_test.sh | 13 +- barretenberg/cpp/cmake/lmdb.cmake | 17 +- .../crypto/merkle_tree/CMakeLists.txt | 15 +- .../append_only_tree/append_only_tree.hpp | 127 +++++++++++---- .../append_only_tree.test.cpp | 38 +++-- .../merkle_tree/indexed_tree/indexed_leaf.hpp | 25 ++- .../merkle_tree/indexed_tree/indexed_tree.hpp | 146 +++++++++++++----- .../indexed_tree/indexed_tree.test.cpp | 56 ++++--- .../crypto/merkle_tree/memory_tree.hpp | 13 +- .../node_store/cached_tree_store.hpp | 12 +- .../merkle_tree/node_store/tree_meta.hpp | 3 +- .../nullifier_tree/nullifier_leaf.hpp | 2 +- .../crypto/merkle_tree/response.hpp | 9 +- .../barretenberg/world_state/CMakeLists.txt | 2 +- 14 files changed, 339 insertions(+), 139 deletions(-) diff --git a/barretenberg/cpp/build_and_test.sh b/barretenberg/cpp/build_and_test.sh index 013bfe4778b..6c5659a6314 100755 --- a/barretenberg/cpp/build_and_test.sh +++ b/barretenberg/cpp/build_and_test.sh @@ -1,5 +1,10 @@ +#!/usr/bin/env bash + +set -e + +TEST=${1:-*} + cmake --build --preset default --target crypto_merkle_tree_tests -#cmake --build --preset default --target world_state -./build/bin/crypto_merkle_tree_tests --gtest_filter=PersistedIndexedTreeTest.* -./build/bin/crypto_merkle_tree_tests --gtest_filter=PersistedAppendOnlyTreeTest* -./build/bin/crypto_merkle_tree_tests --gtest_filter=LMDBStoreTest* +./build/bin/crypto_merkle_tree_tests --gtest_filter=PersistedIndexedTreeTest.$TEST +./build/bin/crypto_merkle_tree_tests --gtest_filter=PersistedAppendOnlyTreeTest.$TEST +./build/bin/crypto_merkle_tree_tests --gtest_filter=LMDBStoreTest.$TEST diff --git a/barretenberg/cpp/cmake/lmdb.cmake b/barretenberg/cpp/cmake/lmdb.cmake index 4f90fe3ee40..ca24a99c802 100644 --- a/barretenberg/cpp/cmake/lmdb.cmake +++ b/barretenberg/cpp/cmake/lmdb.cmake @@ -1,22 +1,27 @@ include(ExternalProject) set(LMDB_PREFIX "${CMAKE_BINARY_DIR}/_deps/lmdb") -set(LMDB_INCLUDE "${LMDB_PREFIX}/src/lmdb/libraries/liblmdb") +set(LMDB_INCLUDE "${LMDB_PREFIX}/src/lmdb_repo/libraries/liblmdb") set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a") +set(LMDB_OBJECT "${LMDB_INCLUDE}/*.o") ExternalProject_Add( - lmdb + lmdb_repo PREFIX ${LMDB_PREFIX} GIT_REPOSITORY "https://github.com/LMDB/lmdb.git" GIT_TAG ddd0a773e2f44d38e4e31ec9ed81af81f4e4ccbb BUILD_IN_SOURCE YES CONFIGURE_COMMAND "" # No configure step - BUILD_COMMAND make -C libraries/liblmdb liblmdb.a + BUILD_COMMAND make -C libraries/liblmdb -e XCFLAGS=-fPIC liblmdb.a INSTALL_COMMAND "" UPDATE_COMMAND "" # No update step BUILD_BYPRODUCTS ${LMDB_LIB} ${LMDB_INCLUDE} ) -add_library(lmdb-lib STATIC IMPORTED) -add_dependencies(lmdb-lib lmdb) -set_target_properties(lmdb-lib PROPERTIES IMPORTED_LOCATION ${LMDB_LIB}) +add_library(lmdb STATIC IMPORTED GLOBAL) +add_dependencies(lmdb lmdb_repo) +set_target_properties(lmdb PROPERTIES IMPORTED_LOCATION ${LMDB_LIB}) + +add_library(lmdb_objects OBJECT IMPORTED GLOBAL) +add_dependencies(lmdb_objects lmdb_repo) +set_target_properties(lmdb_objects PROPERTIES IMPORTED_LOCATION ${LMDB_OBJECT}) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt index 5ccfdaa89d5..11052ae1155 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt @@ -1,13 +1,10 @@ +# merkle tree is agnostic to hash function barretenberg_module( crypto_merkle_tree - stdlib_primitives - stdlib_blake3s - stdlib_pedersen_hash - crypto_poseidon2 - lmdb-lib + lmdb ) -add_dependencies(crypto_merkle_tree lmdb) -add_dependencies(crypto_merkle_tree_objects lmdb) -add_dependencies(crypto_merkle_tree_tests lmdb) -add_dependencies(crypto_merkle_tree_test_objects lmdb) +if (NOT FUZZING) + # but the tests use pedersen and poseidon + target_link_libraries(crypto_merkle_tree_tests PRIVATE stdlib_pedersen_hash stdlib_poseidon2) +endif() diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 5bf5af7eefe..4638371d8fa 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -55,6 +55,30 @@ template class AppendOnlyTree { */ void get_sibling_path(const index_t& index, const HashPathCallback& on_completion, bool includeUncommitted) const; + /** + * @brief Get the subtree sibling path object + * + * @param subtree_depth The depth of the subtree + * @param on_completion Callback to be called on completion + * @param includeUncommitted Whether to include uncommitted changes + */ + void get_subtree_sibling_path(uint32_t subtree_depth, + const HashPathCallback& on_completion, + bool includeUncommitted) const; + + /** + * @brief Get the subtree sibling path object to a leaf + * + * @param leaf_index The depth of the subtree + * @param subtree_depth The depth of the subtree + * @param on_completion Callback to be called on completion + * @param includeUncommitted Whether to include uncommitted changes + */ + void get_subtree_sibling_path(index_t leaf_index, + uint32_t subtree_depth, + const HashPathCallback& on_completion, + bool includeUncommitted) const; + void get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion) const; void get_leaf(const index_t& index, bool includeUncommitted, const GetLeafCallback& completion) const; @@ -82,18 +106,20 @@ template class AppendOnlyTree { ReadTransaction& tx, bool includeUncommitted) const; - void add_values_internal(std::shared_ptr> values, + void add_values_internal(const std::shared_ptr>& values, fr& new_root, index_t& new_size, - fr& subtree_root, - index_t& subtree_root_index, - fr_sibling_path& subtree_path, bool update_index); void add_values_internal(const std::vector& values, const AppendCompletionCallback& on_completion, bool update_index); + fr_sibling_path get_subtree_sibling_path_internal(index_t leaf_index, + uint32_t subtree_depth, + ReadTransaction& tx, + bool includeUncommitted) const; + Store& store_; uint32_t depth_; std::string name_; @@ -169,6 +195,66 @@ void AppendOnlyTree::get_sibling_path(const index_t& index workers_.enqueue(job); } +template +void AppendOnlyTree::get_subtree_sibling_path(const uint32_t subtree_depth, + const HashPathCallback& on_completion, + bool includeUncommitted) const +{ + auto job = [=, this]() { + ExecuteAndReport( + [=, this](TypedResponse& response) { + ReadTransactionPtr tx = store_.createReadTransaction(); + index_t index_of_next_leaf = 0; + bb::fr root; + store_.get_meta(index_of_next_leaf, root, *tx, includeUncommitted); + response.inner.path = + get_subtree_sibling_path_internal(index_of_next_leaf, subtree_depth, *tx, includeUncommitted); + }, + on_completion); + }; + workers_.enqueue(job); +} + +template +void AppendOnlyTree::get_subtree_sibling_path(const index_t leaf_index, + const uint32_t subtree_depth, + const HashPathCallback& on_completion, + bool includeUncommitted) const +{ + auto job = [=, this]() { + ExecuteAndReport( + [=, this](TypedResponse& response) { + ReadTransactionPtr tx = store_.createReadTransaction(); + response.inner.path = + get_subtree_sibling_path_internal(leaf_index, subtree_depth, *tx, includeUncommitted); + }, + on_completion); + }; + workers_.enqueue(job); +} + +template +fr_sibling_path AppendOnlyTree::get_subtree_sibling_path_internal(const index_t leaf_index, + const uint32_t subtree_depth, + ReadTransaction& tx, + bool includeUncommitted) const +{ + // skip the first levels, all the way to the subtree_root + index_t current_index = leaf_index >> subtree_depth; + fr_sibling_path path; + path.reserve(depth_ - subtree_depth); + + for (uint32_t level = depth_ - subtree_depth; level > 0; --level) { + bool is_right = static_cast(current_index & 0x01); + fr sibling = is_right ? get_element_or_zero(level, current_index - 1, tx, includeUncommitted) + : get_element_or_zero(level, current_index + 1, tx, includeUncommitted); + path.emplace_back(sibling); + current_index >>= 1; + } + + return path; +} + template void AppendOnlyTree::get_leaf(const index_t& index, bool includeUncommitted, @@ -241,13 +327,7 @@ void AppendOnlyTree::add_values_internal(const std::vector auto append_op = [=, this]() -> void { ExecuteAndReport( [=, this](TypedResponse& response) { - add_values_internal(hashes, - response.inner.root, - response.inner.size, - response.inner.subtree_root, - response.inner.subtree_root_index, - response.inner.subtree_path, - update_index); + add_values_internal(hashes, response.inner.root, response.inner.size, update_index); }, on_completion); }; @@ -255,14 +335,12 @@ void AppendOnlyTree::add_values_internal(const std::vector } template -void AppendOnlyTree::add_values_internal(std::shared_ptr> values, +void AppendOnlyTree::add_values_internal(const std::shared_ptr>& values, fr& new_root, index_t& new_size, - fr& subtree_root, - index_t& subtree_root_index, - fr_sibling_path& subtree_path, bool update_index) { + uint32_t start_level = depth_; uint32_t level = start_level; std::vector& hashes_local = *values; @@ -274,6 +352,11 @@ void AppendOnlyTree::add_values_internal(std::shared_ptrempty()) { + return; + } + if (new_size > max_size_) { throw std::runtime_error("Tree is full"); } @@ -301,26 +384,12 @@ void AppendOnlyTree::add_values_internal(std::shared_ptr 0) { - subtree_path.reserve(level - 1); // minus the tree root - } while (level > 0) { bool is_right = static_cast(index & 0x01); fr left_hash = is_right ? get_element_or_zero(level, index - 1, *tx, true) : new_hash; fr right_hash = is_right ? new_hash : get_element_or_zero(level, index + 1, *tx, true); new_hash = HashingPolicy::hash_pair(left_hash, right_hash); - if (is_right) { - subtree_path.emplace_back(left_hash); - } else { - subtree_path.emplace_back(right_hash); - } - index >>= 1; --level; if (level > 0) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp index 2a9f030ea59..d64bd123e1c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -4,6 +4,7 @@ #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" #include "barretenberg/common/thread_pool.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" @@ -662,25 +663,34 @@ TEST_F(PersistedAppendOnlyTreeTest, can_get_inserted_leaves) check_leaf(tree, 0, 4, false); } -TEST_F(PersistedAppendOnlyTreeTest, append_leaves_returns_sibling_path) +TEST_F(PersistedAppendOnlyTreeTest, returns_sibling_path) { - constexpr size_t depth = 10; + constexpr size_t depth = 4; std::string name = randomString(); LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); Store store(name, depth, db); ThreadPool pool(1); TreeType tree(store, pool); + MemoryTree memdb(depth); - Signal signal; - auto completion = [&](const TypedResponse& response) -> void { - EXPECT_EQ(response.success, true); - check_sibling_path(response.inner.root, - response.inner.subtree_root, - response.inner.subtree_root_index, - response.inner.subtree_path); - signal.signal_level(); - }; - - tree.add_values({ 40, 30, 10, 20 }, completion); - signal.wait_for_level(); + add_values(tree, { 30, 10, 20 }); + memdb.update_element(0, 30); + memdb.update_element(1, 10); + memdb.update_element(2, 20); + + tree.get_subtree_sibling_path( + 0, + [&](auto& resp) { + fr_sibling_path expected_sibling_path = memdb.get_sibling_path(3); + EXPECT_EQ(resp.inner.path, expected_sibling_path); + }, + true); + + tree.get_subtree_sibling_path( + 2, + [&](auto& resp) { + fr_sibling_path expected_sibling_path = { memdb.get_node(2, 1), memdb.get_node(1, 1) }; + EXPECT_EQ(resp.inner.path, expected_sibling_path); + }, + true); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp index 59b2b7e4563..e8b14be3a74 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp @@ -1,6 +1,7 @@ #pragma once #include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" @@ -34,6 +35,8 @@ struct NullifierLeafValue { } ~NullifierLeafValue() = default; + static bool is_updateable() { return false; } + bool operator==(NullifierLeafValue const& other) const { return value == other.value; } friend std::ostream& operator<<(std::ostream& os, const NullifierLeafValue& v) @@ -46,6 +49,11 @@ struct NullifierLeafValue { bool is_empty() const { return value == fr::zero(); } + std::vector get_hash_inputs(fr nextValue, fr nextIndex) const + { + return std::vector({ value, nextValue, nextIndex }); + } + operator uint256_t() const { return get_key(); } static NullifierLeafValue empty() { return { fr::zero() }; } @@ -85,6 +93,8 @@ struct PublicDataLeafValue { } ~PublicDataLeafValue() = default; + static bool is_updateable() { return true; } + bool operator==(PublicDataLeafValue const& other) const { return value == other.value && slot == other.slot; } friend std::ostream& operator<<(std::ostream& os, const PublicDataLeafValue& v) @@ -95,7 +105,12 @@ struct PublicDataLeafValue { fr get_key() const { return slot; } - bool is_empty() const { return value == fr::zero() && slot == fr::zero(); } + bool is_empty() const { return slot == fr::zero(); } + + std::vector get_hash_inputs(fr nextValue, fr nextIndex) const + { + return std::vector({ slot, value, nextIndex, nextValue }); + } operator uint256_t() const { return get_key(); } @@ -123,6 +138,8 @@ template struct IndexedLeaf { IndexedLeaf(IndexedLeaf&& other) noexcept = default; ~IndexedLeaf() = default; + static bool is_updateable() { return LeafType::is_updateable(); } + bool operator==(IndexedLeaf const& other) const { return value == other.value && nextValue == other.nextValue && nextIndex == other.nextIndex; @@ -154,11 +171,13 @@ template struct IndexedLeaf { return os; } - std::vector get_hash_inputs() const { return std::vector({ value.value, nextIndex, nextValue }); } + std::vector get_hash_inputs() const { return value.get_hash_inputs(nextValue, nextIndex); } + + bool is_empty() { return value.is_empty(); } static IndexedLeaf empty() { return { LeafType::empty(), 0, 0 }; } static IndexedLeaf padding(index_t i) { return { LeafType::padding(i), 0, 0 }; } }; -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index b9df9d5d53f..7aae0c3d99e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -9,6 +9,7 @@ #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" #include "indexed_leaf.hpp" +#include #include #include #include @@ -35,6 +36,7 @@ template class IndexedTree : public App using IndexedLeafValueType = typename Store::IndexedLeafValueType; using AddCompletionCallback = std::function>&)>; using LeafCallback = std::function>&)>; + using FindLowLeafCallback = std::function>&)>; IndexedTree(Store& store, ThreadPool& workers, index_t initial_size); IndexedTree(IndexedTree const& other) = delete; @@ -53,11 +55,20 @@ template class IndexedTree : public App /** * @brief Adds or updates the given set of values in the tree (updates not currently supported) * @param values The values to be added or updated - * @param no_multithreading Performs single threaded insertion, just used whilst prototyping and benchmarking - * @returns The 'previous' hash paths of all updated values + * @param completion The callback to be triggered once the values have been added */ void add_or_update_values(const std::vector& values, const AddCompletionCallback& completion); + /** + * @brief Adds or updates the given set of values in the tree (updates not currently supported) + * @param values The values to be added or updated + * @param subtree_depth The height of the subtree to be inserted. + * @param completion The callback to be triggered once the values have been added + */ + void add_or_update_values(const std::vector& values, + uint32_t subtree_depth, + const AddCompletionCallback& completion); + void get_leaf(const index_t& index, bool includeUncommitted, const LeafCallback& completion) const; void find_leaf_index(const LeafValueType& leaf, @@ -69,7 +80,7 @@ template class IndexedTree : public App bool includeUncommitted, const AppendOnlyTree::FindLeafCallback& on_completion) const; - void find_low_leaf(const LeafValueType& leaf, bool includeUncommitted, const LeafCallback& on_completion) const; + void find_low_leaf(const fr& leaf_key, bool includeUncommitted, const FindLowLeafCallback& on_completion) const; using AppendOnlyTree::get_sibling_path; @@ -116,7 +127,8 @@ template class IndexedTree : public App }; using InsertionCompletionCallback = std::function&)>; - void perform_insertions(std::shared_ptr> insertions, + void perform_insertions(size_t total_leaves, + std::shared_ptr> insertions, const InsertionCompletionCallback& completion); struct HashGenerationResponse { @@ -187,17 +199,18 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers store_.set_at_index(i, initial_leaf, true); } - bool success = true; + TypedResponse result; Signal signal(1); - AppendCompletionCallback completion = [&](const TypedResponse& result) -> void { - success = result.success; + AppendCompletionCallback completion = [&](const TypedResponse& _result) -> void { + result = _result; signal.signal_level(0); }; AppendOnlyTree::add_values_internal(appended_hashes, completion, false); signal.wait_for_level(0); - if (!success) { - throw std::runtime_error("Failed to initialise tree"); + if (!result.success) { + throw std::runtime_error("Failed to initialise tree: " + result.message); } + store_.commit(); } @@ -250,14 +263,17 @@ void IndexedTree::find_leaf_index_from( } template -void IndexedTree::find_low_leaf(const LeafValueType& leaf, +void IndexedTree::find_low_leaf(const fr& leaf_key, bool includeUncommitted, - const LeafCallback& on_completion) const + const FindLowLeafCallback& on_completion) const { - auto job = [=, &leaf, this]() { - typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); - auto result = store_.find_low_value(leaf, includeUncommitted, *tx); - get_leaf(result.second, includeUncommitted, on_completion); + auto job = [=, this]() { + ExecuteAndReport>( + [=, this](TypedResponse>& response) { + typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); + response.inner = store_.find_low_value(leaf_key, includeUncommitted, *tx); + }, + on_completion); }; workers_.enqueue(job); @@ -267,12 +283,20 @@ template void IndexedTree::add_or_update_value(const LeafValueType& value, const AddCompletionCallback& completion) { - add_or_update_values(std::vector{ value }, completion); + add_or_update_values(std::vector{ value }, 1, completion); } template void IndexedTree::add_or_update_values(const std::vector& values, const AddCompletionCallback& completion) +{ + add_or_update_values(values, 0, completion); +} + +template +void IndexedTree::add_or_update_values(const std::vector& values, + const uint32_t subtree_depth, + const AddCompletionCallback& completion) { // We first take a copy of the leaf values and their locations within the set given to us std::shared_ptr>> values_to_be_sorted = @@ -287,6 +311,7 @@ void IndexedTree::add_or_update_values(const std::vector> hashes_to_append; // info about the low leaves that have been updated std::shared_ptr>> low_leaf_witness_data; + fr_sibling_path subtree_path; std::atomic count; Status status; @@ -312,19 +337,30 @@ void IndexedTree::add_or_update_values(const std::vector& add_result) { + auto final_completion = [=](const TypedResponse& add_data_response) { TypedResponse> response; - response.success = add_result.success; - response.message = add_result.message; - if (add_result.success) { - response.inner.add_data_result = add_result.inner; + response.success = add_data_response.success; + response.message = add_data_response.message; + if (add_data_response.success) { + response.inner.subtree_path = results->subtree_path; response.inner.sorted_leaves = values_to_be_sorted; response.inner.low_leaf_witness_data = results->low_leaf_witness_data; + response.inner.add_data_result = add_data_response.inner; } // Trigger the client's provided callback completion(response); }; + auto sibling_path_completion = [=, this](const TypedResponse& response) { + if (!response.success) { + results->status.set_failure(response.message); + } else { + results->subtree_path = response.inner.path; + AppendOnlyTree::add_values_internal( + (*results->hashes_to_append), final_completion, false); + } + }; + // This signals the completion of the appended hash generation // If the low leaf updates are also completed then we will append the leaves HashGenerationCallback hash_completion = [=, this](const TypedResponse& hashes_response) { @@ -333,13 +369,14 @@ void IndexedTree::add_or_update_values(const std::vectorhashes_to_append = hashes_response.inner.hashes; } + if (results->count.fetch_sub(1) == 1) { if (!results->status.success) { on_error(results->status.message); return; } - AppendOnlyTree::add_values_internal( - (*results->hashes_to_append), final_completion, false); + AppendOnlyTree::get_subtree_sibling_path( + subtree_depth, sibling_path_completion, true); } }; @@ -352,13 +389,14 @@ void IndexedTree::add_or_update_values(const std::vectorlow_leaf_witness_data = insertion_response.inner.low_leaf_witness_data; } + if (results->count.fetch_sub(1) == 1) { if (!results->status.success) { on_error(results->status.message); return; } - AppendOnlyTree::add_values_internal( - (*results->hashes_to_append), final_completion, false); + AppendOnlyTree::get_subtree_sibling_path( + subtree_depth, sibling_path_completion, true); } }; @@ -373,7 +411,7 @@ void IndexedTree::add_or_update_values(const std::vector::add_or_update_values(const std::vector -void IndexedTree::perform_insertions(std::shared_ptr> insertions, +void IndexedTree::perform_insertions(size_t total_leaves, + std::shared_ptr> insertions, const InsertionCompletionCallback& completion) { - // We now kick off multiple workers to perform the low leaf updates - // We create set of signals to coordinate the workers as the move up the tree auto low_leaf_witness_data = std::make_shared>>( - insertions->size(), LowLeafWitnessData{ IndexedLeafValueType::empty(), 0, {} }); + total_leaves, + LowLeafWitnessData{ IndexedLeafValueType::empty(), 0, fr_sibling_path(depth_, fr::zero()) }); + + // early return, no insertions to perform + if (insertions->size() == 0) { + TypedResponse response; + response.success = true; + response.inner.low_leaf_witness_data = low_leaf_witness_data; + completion(response); + return; + } + // We now kick off multiple workers to perform the low leaf updates + // We create set of signals to coordinate the workers as the move up the tree std::shared_ptr> signals = std::make_shared>(); std::shared_ptr status = std::make_shared(); // The first signal is set to 0. This ensures the first worker up the tree is not impeded @@ -405,9 +454,10 @@ void IndexedTree::perform_insertions(std::shared_ptrat(i); + auto& current_witness_data = low_leaf_witness_data->at(i); current_witness_data.leaf = insertion.original_low_leaf; current_witness_data.index = insertion.low_leaf_index; + current_witness_data.path.clear(); update_leaf_and_hash_to_root(insertion.low_leaf_index, insertion.low_leaf, leaderSignal, @@ -439,10 +489,12 @@ void IndexedTree::generate_hashes_for_appending( { ExecuteAndReport( [=](TypedResponse& response) { - response.inner.hashes = std::make_shared>(leaves_to_hash->size()); + response.inner.hashes = std::make_shared>(leaves_to_hash->size(), 0); std::vector& leaves = *leaves_to_hash; for (uint32_t i = 0; i < leaves.size(); ++i) { - (*response.inner.hashes)[i] = HashingPolicy::hash(leaves[i].get_hash_inputs()); + if (!leaves[i].is_empty()) { + (*response.inner.hashes)[i] = (HashingPolicy::hash(leaves[i].get_hash_inputs())); + } } }, completion); @@ -462,7 +514,7 @@ void IndexedTree::generate_insertions( { uint256_t aValue = a.first.get_key(); uint256_t bValue = b.first.get_key(); - return aValue == bValue ? a.second < b.second : aValue < bValue; + return aValue == bValue ? a.second < b.second : aValue > bValue; } } comp; std::sort(values_to_be_sorted->begin(), values_to_be_sorted->end(), comp); @@ -471,7 +523,8 @@ void IndexedTree::generate_insertions( // Now that we have the sorted values we need to identify the leaves that need updating. // This is performed sequentially and is stored in this 'leaf_insertion' struct - response.inner.insertions = std::make_shared>(values.size()); + response.inner.insertions = std::make_shared>(); + response.inner.insertions->reserve(values.size()); response.inner.indexed_leaves = std::make_shared>(values.size(), IndexedLeafValueType::empty()); index_t old_size = 0; @@ -502,14 +555,20 @@ void IndexedTree::generate_insertions( // This gives us the leaf that need updating index_t current = 0; bool is_already_present = false; - std::tie(is_already_present, current) = store_.find_low_value(value_pair.first, true, *tx); + std::tie(is_already_present, current) = + store_.find_low_value(value_pair.first.get_key(), true, *tx); // .value() throws if the low leaf does not exist IndexedLeafValueType current_leaf = store_.get_leaf(current, *tx, true).value(); + response.inner.insertions->push_back({ + .low_leaf_index = current, + .low_leaf = IndexedLeafValueType::empty(), + .original_low_leaf = current_leaf, + }); LeafInsertion& insertion = (*response.inner.insertions)[i]; // Capture the index and original value of the 'low' leaf - insertion.low_leaf_index = current; - insertion.original_low_leaf = current_leaf; + // insertion.low_leaf_index = current; + // insertion.original_low_leaf = current_leaf; if (!is_already_present) { // Update the current leaf to point it to the new leaf @@ -523,7 +582,7 @@ void IndexedTree::generate_insertions( // Update the set of leaves to append (*response.inner.indexed_leaves)[index_into_appended_leaves] = new_leaf; - } else { + } else if (IndexedLeafValueType::is_updateable()) { // Update the current leaf's value, don't change it's link IndexedLeafValueType replacement_leaf = IndexedLeafValueType(value_pair.first, current_leaf.nextIndex, current_leaf.nextValue); @@ -533,6 +592,8 @@ void IndexedTree::generate_insertions( store_.set_at_index(index_of_new_leaf, empty_leaf, false); // The set of appended leaves already has an empty leaf in the slot at index // 'index_into_appended_leaves' + } else { + throw std::runtime_error("IndexedLeafValue is not updateable"); } // capture new low leaf @@ -570,8 +631,8 @@ void IndexedTree::update_leaf_and_hash_to_root(const index // Extract the value of the leaf node and it's sibling bool is_right = static_cast(index & 0x01); // extract the current leaf hash values for the previous hash path - fr sibling = get_node(level, is_right ? index - 1 : index + 1); - previous_sibling_path.emplace_back(sibling); + // fr sibling = get_node(level, is_right ? index - 1 : index + 1); + // previous_sibling_path.emplace_back(sibling); // Write the new leaf hash in place write_node(level, index, new_hash); @@ -589,7 +650,7 @@ void IndexedTree::update_leaf_and_hash_to_root(const index index_t index_of_node_above = index >> 1; bool node_above_is_right = static_cast(index_of_node_above & 0x01); fr above_sibling = get_node(level, node_above_is_right ? index_of_node_above - 1 : index_of_node_above + 1); - previous_sibling_path.emplace_back(above_sibling); + // previous_sibling_path.emplace_back(above_sibling); } // Now that we have extracted the hash path from the row above @@ -597,6 +658,7 @@ void IndexedTree::update_leaf_and_hash_to_root(const index is_right = static_cast(index & 0x01); fr new_right_value = is_right ? new_hash : get_node(level, index + 1); fr new_left_value = is_right ? get_node(level, index - 1) : new_hash; + previous_sibling_path.emplace_back(is_right ? new_left_value : new_right_value); new_hash = HashingPolicy::hash_pair(new_left_value, new_right_value); index >>= 1; --level; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index e4c8d0d5473..1773ce749a4 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -124,20 +124,19 @@ IndexedLeaf get_leaf(IndexedTree -IndexedLeaf get_low_leaf( - IndexedTree, Poseidon2HashPolicy>& tree, - const LeafValueType& leaf, - bool includeUncommitted = true) +std::pair get_low_leaf(IndexedTree, Poseidon2HashPolicy>& tree, + const LeafValueType& leaf, + bool includeUncommitted = true) { - std::optional> l; + std::pair low_leaf_info; Signal signal; - auto completion = [&](const TypedResponse>& leaf) -> void { - l = leaf.inner.indexed_leaf; + auto completion = [&](const auto& leaf) -> void { + low_leaf_info = leaf.inner; signal.signal_level(); }; - tree.find_low_leaf(leaf, includeUncommitted, completion); + tree.find_low_leaf(leaf.get_key(), includeUncommitted, completion); signal.wait_for_level(); - return l.value(); + return low_leaf_info; } template @@ -832,9 +831,9 @@ TEST_F(PersistedIndexedTreeTest, can_add_single_whilst_reading) signal.wait_for_level(); } - for (auto& path : paths) { - EXPECT_TRUE(path == initial_path || path == final_sibling_path); - } + // for (auto& path : paths) { + // EXPECT_TRUE(path == initial_path || path == final_sibling_path); + // } } TEST_F(PersistedIndexedTreeTest, test_indexed_memory_with_public_data_writes) @@ -941,7 +940,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory_with_public_data_writes) auto e001 = hash_leaf(get_leaf(tree, 1)); auto e010 = hash_leaf(get_leaf(tree, 2)); auto e011 = hash_leaf(get_leaf(tree, 3)); - auto e100 = hash_leaf(get_leaf(tree, 4)); + auto e100 = fr::zero(); // tree doesn't hash 0 leaves! auto e101 = hash_leaf(get_leaf(tree, 5)); auto e110 = fr::zero(); auto e111 = fr::zero(); @@ -1009,15 +1008,34 @@ TEST_F(PersistedIndexedTreeTest, returns_low_leaves) auto predecessor = get_low_leaf(tree, NullifierLeafValue(42)); - EXPECT_EQ(predecessor.value, NullifierLeafValue(1)); - EXPECT_EQ(predecessor.nextIndex, 0); - EXPECT_EQ(predecessor.nextValue, 0); + EXPECT_EQ(predecessor.first, false); + EXPECT_EQ(predecessor.second, 1); add_value(tree, NullifierLeafValue(42)); predecessor = get_low_leaf(tree, NullifierLeafValue(42)); // returns the current leaf since it exists already. Inserting 42 again would modify the existing leaf - EXPECT_EQ(predecessor.value, NullifierLeafValue(42)); - EXPECT_EQ(predecessor.nextIndex, 0); - EXPECT_EQ(predecessor.nextValue, 0); + EXPECT_EQ(predecessor.first, true); + EXPECT_EQ(predecessor.second, 2); +} + +TEST_F(PersistedIndexedTreeTest, duplicates) +{ + // Create a depth-8 indexed merkle tree + constexpr uint32_t depth = 8; + + ThreadPool workers(1); + std::string name = randomString(); + LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + Store store(name, depth, db); + auto tree = TreeType(store, workers, 2); + + add_value(tree, NullifierLeafValue(42)); + check_size(tree, 3); + + commit_tree(tree); + + add_value(tree, NullifierLeafValue(42)); + commit_tree(tree); + check_size(tree, 3); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp index e85e613aa9d..6533ef6857b 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp @@ -32,6 +32,8 @@ template class MemoryTree { fr root() const { return root_; } + fr get_node(uint32_t level, size_t index) const; + public: size_t depth_; size_t total_size_; @@ -61,6 +63,15 @@ MemoryTree::MemoryTree(size_t depth) root_ = current; } +template fr MemoryTree::get_node(uint32_t level, size_t index) const +{ + auto offset = 0UL; + for (size_t i = depth_; i > level; i--) { + offset += 1UL << i; + } + return hashes_[offset + index]; +} + template fr_hash_path MemoryTree::get_hash_path(size_t index) { fr_hash_path path(depth_); @@ -111,4 +122,4 @@ template fr MemoryTree::update_element(s return root_; } -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp index 566ac6d52e9..7066db2a31c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp @@ -46,9 +46,7 @@ template class CachedTreeStore CachedTreeStore& operator=(CachedTreeStore const& other) = delete; CachedTreeStore& operator=(CachedTreeStore const&& other) = delete; - std::pair find_low_value(const LeafValueType& new_value, - bool includeUncommitted, - ReadTransaction& tx) const; + std::pair find_low_value(const fr& new_leaf_key, bool includeUncommitted, ReadTransaction& tx) const; std::optional get_leaf(const index_t& index, ReadTransaction& tx, @@ -90,6 +88,8 @@ template class CachedTreeStore void rollback(); + std::string get_name() const { return name; } + ReadTransactionPtr createReadTransaction() const { return dataStore.createReadTransaction(); } private: @@ -121,13 +121,13 @@ template class CachedTreeStore }; template -std::pair CachedTreeStore::find_low_value(const LeafValueType& new_value, +std::pair CachedTreeStore::find_low_value(const fr& new_leaf_key, bool includeUncommitted, ReadTransaction& tx) const { - uint256_t new_value_as_number = uint256_t(new_value.get_key()); + uint256_t new_value_as_number = uint256_t(new_leaf_key); std::vector data; - FrKeyType key(new_value.get_key()); + FrKeyType key(new_leaf_key); tx.get_value_or_previous(key, data); Indices committed; msgpack::unpack((const char*)data.data(), data.size()).get().convert(committed); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp index 69aa147073a..94132dd5f57 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp @@ -1,5 +1,6 @@ #pragma once #include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/serialize/msgpack.hpp" #include #include @@ -21,4 +22,4 @@ struct LeavesMeta { MSGPACK_FIELDS(size) }; -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp index c3d8b3369ce..e91a0cf744d 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp @@ -21,7 +21,7 @@ struct indexed_nullifier_leaf { return os; } - std::vector get_hash_inputs() const { return std::vector{ value, nextIndex, nextValue }; } + std::vector get_hash_inputs() const { return std::vector{ value, nextValue, nextIndex }; } static indexed_nullifier_leaf zero() { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 6b0b6227b66..89d96430976 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -4,6 +4,7 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "barretenberg/serialize/msgpack.hpp" #include #include #include @@ -19,9 +20,8 @@ struct TreeMetaResponse { }; struct AddDataResponse { - index_t size, subtree_root_index; - fr root, subtree_root; - fr_sibling_path subtree_path; + index_t size; + fr root; }; struct GetSiblingPathResponse { @@ -32,10 +32,13 @@ template struct LowLeafWitnessData { IndexedLeaf leaf; index_t index; fr_sibling_path path; + + MSGPACK_FIELDS(leaf, index, path); }; template struct AddIndexedDataResponse { AddDataResponse add_data_result; + fr_sibling_path subtree_path; std::shared_ptr>> sorted_leaves; std::shared_ptr>> low_leaf_witness_data; }; diff --git a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt index 45e9ec63a6b..8c4b0296ad3 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(world_state crypto_merkle_tree cryto_pedersen_hash) +barretenberg_module(world_state crypto_merkle_tree stdlib_pedersen_hash) From 36fca2df2a84ae57757ef45dc1db94009c43aa14 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Tue, 16 Jul 2024 16:09:54 +0000 Subject: [PATCH 46/63] chore: undo changes to yarn-project --- .../aztec-node/src/aztec-node/server.ts | 8 +- .../composed/integration_l1_publisher.test.ts | 5 +- yarn-project/merkle-tree/package.json | 3 +- .../src/factory/js_tree_factory.ts | 101 -------------- yarn-project/merkle-tree/src/index.ts | 4 +- .../merkle-tree/src/interfaces/merkle_tree.ts | 8 -- .../src/interfaces/tree_factory.ts | 52 ------- yarn-project/merkle-tree/src/load_tree.ts | 33 +++++ .../merkle-tree/src/native/message.ts | 78 ----------- .../merkle-tree/src/native/native_client.ts | 105 -------------- yarn-project/merkle-tree/src/new_tree.ts | 29 ++++ .../src/snapshots/append_only_snapshot.ts | 9 +- .../src/sparse_tree/sparse_tree.test.ts | 3 +- .../test/standard_indexed_tree_native.test.ts | 130 ------------------ .../src/standard_tree/standard_tree.test.ts | 3 +- .../src/standard_tree/standard_tree_native.ts | 109 --------------- .../merkle-tree/src/test/get_native_config.ts | 28 ---- yarn-project/merkle-tree/src/tree_base.ts | 2 +- yarn-project/prover-client/package.json | 1 - .../prover-client/src/mocks/test_context.ts | 4 +- .../orchestrator_mixed_blocks_2.test.ts | 4 +- .../orchestrator_single_blocks.test.ts | 4 +- .../orchestrator_workflow.test.ts | 4 +- yarn-project/prover-client/tsconfig.json | 5 +- .../src/world-state-db/merkle_trees.ts | 98 ++++++++++--- yarn-project/yarn.lock | 9 -- 26 files changed, 162 insertions(+), 677 deletions(-) delete mode 100644 yarn-project/merkle-tree/src/factory/js_tree_factory.ts delete mode 100644 yarn-project/merkle-tree/src/interfaces/tree_factory.ts create mode 100644 yarn-project/merkle-tree/src/load_tree.ts delete mode 100644 yarn-project/merkle-tree/src/native/message.ts delete mode 100644 yarn-project/merkle-tree/src/native/native_client.ts create mode 100644 yarn-project/merkle-tree/src/new_tree.ts delete mode 100644 yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_native.test.ts delete mode 100644 yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts delete mode 100644 yarn-project/merkle-tree/src/test/get_native_config.ts diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index be0ca099f43..5ccbf6266c7 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -53,7 +53,7 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { type AztecKVStore } from '@aztec/kv-store'; import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils'; -import { JSTreeFactory, SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; +import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { AztecKVTxPool, type P2P, createP2PClient } from '@aztec/p2p'; import { getCanonicalClassRegisterer } from '@aztec/protocol-contracts/class-registerer'; import { getCanonicalGasToken } from '@aztec/protocol-contracts/gas-token'; @@ -156,8 +156,7 @@ export class AztecNodeService implements AztecNode { const p2pClient = await createP2PClient(store, config, new AztecKVTxPool(store), archiver); // now create the merkle trees and the world state synchronizer - const treeFactory = await JSTreeFactory.init(store); - const merkleTrees = await MerkleTrees.new(store, treeFactory); + const merkleTrees = await MerkleTrees.new(store); const worldStateConfig: WorldStateConfig = getWorldStateConfig(); const worldStateSynchronizer = new ServerWorldStateSynchronizer(store, merkleTrees, archiver, worldStateConfig); @@ -749,8 +748,7 @@ export class AztecNodeService implements AztecNode { // Instantiate merkle trees so uncommitted updates by this simulation are local to it. // TODO we should be able to remove this after https://github.com/AztecProtocol/aztec-packages/issues/1869 // So simulation of public functions doesn't affect the merkle trees. - const treeFactory = await JSTreeFactory.init(this.merkleTreesDb); - const merkleTrees = await MerkleTrees.new(this.merkleTreesDb, treeFactory, this.log); + const merkleTrees = await MerkleTrees.new(this.merkleTreesDb, this.log); const publicProcessorFactory = new PublicProcessorFactory( merkleTrees.asLatest(), diff --git a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts index 393c78c3f7d..4445829658c 100644 --- a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts @@ -37,7 +37,7 @@ import { type L1ContractAddresses, createEthereumChain } from '@aztec/ethereum'; import { makeTuple, range } from '@aztec/foundation/array'; import { openTmpStore } from '@aztec/kv-store/utils'; import { AvailabilityOracleAbi, InboxAbi, OutboxAbi, RollupAbi } from '@aztec/l1-artifacts'; -import { JSTreeFactory, SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; +import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { TxProver } from '@aztec/prover-client'; import { type L1Publisher, getL1Publisher } from '@aztec/sequencer-client'; import { MerkleTrees, ServerWorldStateSynchronizer, type WorldStateConfig } from '@aztec/world-state'; @@ -136,8 +136,7 @@ describe('L1Publisher integration', () => { }); const tmpStore = openTmpStore(); - const treeFactory = await JSTreeFactory.init(tmpStore); - builderDb = await MerkleTrees.new(tmpStore, treeFactory); + builderDb = await MerkleTrees.new(tmpStore); blockSource = mock(); blockSource.getBlocks.mockResolvedValue([]); const worldStateConfig: WorldStateConfig = { diff --git a/yarn-project/merkle-tree/package.json b/yarn-project/merkle-tree/package.json index 6541c9e320c..8d19e74c5a5 100644 --- a/yarn-project/merkle-tree/package.json +++ b/yarn-project/merkle-tree/package.json @@ -16,7 +16,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "yarn build && NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests" + "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests" }, "inherits": [ "../package.common.json", @@ -51,7 +51,6 @@ "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", "@aztec/types": "workspace:^", - "@msgpack/msgpack": "^3.0.0-beta2", "sha256": "^0.2.0", "tslib": "^2.4.0" }, diff --git a/yarn-project/merkle-tree/src/factory/js_tree_factory.ts b/yarn-project/merkle-tree/src/factory/js_tree_factory.ts deleted file mode 100644 index f3ae092a717..00000000000 --- a/yarn-project/merkle-tree/src/factory/js_tree_factory.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { MerkleTreeId } from '@aztec/circuit-types'; -import { Fr } from '@aztec/circuits.js'; -import { type Bufferable, type FromBuffer } from '@aztec/foundation/serialize'; -import { type AztecKVStore } from '@aztec/kv-store'; -import { type Hasher } from '@aztec/types/interfaces'; - -import { NullifierTree, PublicDataTree, type TreeFactory } from '../interfaces/tree_factory.js'; -import { Pedersen } from '../pedersen.js'; -import { StandardTree } from '../standard_tree/standard_tree.js'; -import { type TreeBase, getTreeMeta } from '../tree_base.js'; - -/** - * Creates a new tree. - * @param c - The class of the tree to be instantiated. - * @param db - A database used to store the Merkle tree data. - * @param hasher - A hasher used to compute hash paths. - * @param name - Name of the tree. - * @param depth - Depth of the tree. - * @param prefilledSize - A number of leaves that are prefilled with values. - * @returns The newly created tree. - */ -export async function newTree, D extends FromBuffer>( - c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, deserializer: D) => T, - store: AztecKVStore, - hasher: Hasher, - name: string, - deserializer: D, - depth: number, - prefilledSize = 1, -): Promise { - const tree = new c(store, hasher, name, depth, 0n, deserializer); - await tree.init(prefilledSize); - return tree; -} - -/** - * Creates a new tree and sets its root, depth and size based on the meta data which are associated with the name. - * @param c - The class of the tree to be instantiated. - * @param db - A database used to store the Merkle tree data. - * @param hasher - A hasher used to compute hash paths. - * @param name - Name of the tree. - * @returns The newly created tree. - */ -export function loadTree, D extends FromBuffer>( - c: new ( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint, - deserializer: D, - root: Buffer, - ) => T, - store: AztecKVStore, - hasher: Hasher, - name: string, - deserializer: D, -): Promise { - const { root, depth, size } = getTreeMeta(store, name); - const tree = new c(store, hasher, name, depth, size, deserializer, root); - return Promise.resolve(tree); -} - -type InitFunc = , D extends FromBuffer>( - c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, deserializer: D) => T, - store: AztecKVStore, - hasher: Hasher, - name: string, - deserializer: D, - depth: number, - prefilledSize: number, -) => Promise; - -export class JSTreeFactory implements TreeFactory { - constructor(private kvStore: AztecKVStore, private hasher: Hasher, private initFunction: InitFunc) {} - - public static init(kvStore: AztecKVStore, hasher: Hasher = new Pedersen()) { - const factoryMethod = JSTreeFactory.isDbPopulated(kvStore) ? loadTree : newTree; - return new JSTreeFactory(kvStore, hasher, factoryMethod); - } - public async createStandardTree(name: string, depth: number, prefilledSize = 1): Promise> { - return await this.initFunction(StandardTree, this.kvStore, this.hasher, name, Fr, depth, prefilledSize); - } - public async createNullifierTree(name: string, depth: number, prefilledSize = 1): Promise { - return await this.initFunction(NullifierTree, this.kvStore, this.hasher, name, {}, depth, prefilledSize); - } - public async createPublicDataTree(name: string, depth: number, prefilledSize = 1): Promise { - return await this.initFunction(PublicDataTree, this.kvStore, this.hasher, name, {}, depth, prefilledSize); - } - - private static isDbPopulated(kvStore: AztecKVStore): boolean { - try { - getTreeMeta(kvStore, MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]); - // Tree meta was found --> db is populated - return true; - } catch (e) { - // Tree meta was not found --> db is not populated - return false; - } - } -} diff --git a/yarn-project/merkle-tree/src/index.ts b/yarn-project/merkle-tree/src/index.ts index 9ac6ec71186..7ac5cb2dc6e 100644 --- a/yarn-project/merkle-tree/src/index.ts +++ b/yarn-project/merkle-tree/src/index.ts @@ -2,7 +2,6 @@ export * from './interfaces/append_only_tree.js'; export * from './interfaces/indexed_tree.js'; export * from './interfaces/merkle_tree.js'; export * from './interfaces/update_only_tree.js'; -export * from './interfaces/tree_factory.js'; export * from './pedersen.js'; export * from './sha_256.js'; export * from './sparse_tree/sparse_tree.js'; @@ -10,7 +9,8 @@ export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tr export { StandardIndexedTreeWithAppend } from './standard_indexed_tree/test/standard_indexed_tree_with_append.js'; export * from './standard_tree/standard_tree.js'; export { INITIAL_LEAF, getTreeMeta } from './tree_base.js'; -export * from './factory/js_tree_factory.js'; +export { newTree } from './new_tree.js'; +export { loadTree } from './load_tree.js'; export * from './snapshots/snapshot_builder.js'; export * from './snapshots/full_snapshot.js'; export * from './snapshots/append_only_snapshot.js'; diff --git a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts index aed3d7236c2..44fd8c4d4d7 100644 --- a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts @@ -68,11 +68,3 @@ export interface MerkleTree extends SiblingPathSo */ findLeafIndexAfter(leaf: T, startIndex: bigint, includeUncommitted: boolean): bigint | undefined; } - -export interface SnapshottedMerkleTree extends MerkleTree { - getName(): string; - - getNode(level: number, index: bigint): Buffer | undefined; - - getZeroHash(level: number): Buffer; -} diff --git a/yarn-project/merkle-tree/src/interfaces/tree_factory.ts b/yarn-project/merkle-tree/src/interfaces/tree_factory.ts deleted file mode 100644 index 0e9f9f4097a..00000000000 --- a/yarn-project/merkle-tree/src/interfaces/tree_factory.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { - type Fr, - NullifierLeaf, - NullifierLeafPreimage, - PublicDataTreeLeaf, - PublicDataTreeLeafPreimage, -} from '@aztec/circuits.js'; -import { type AztecKVStore } from '@aztec/kv-store'; -import { type Hasher } from '@aztec/types/interfaces'; - -import { StandardIndexedTree } from '../standard_indexed_tree/standard_indexed_tree.js'; -import { type StandardTree } from '../standard_tree/standard_tree.js'; - -/** - * The nullifier tree is an indexed tree. - */ -export class NullifierTree extends StandardIndexedTree { - constructor( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint = 0n, - _noop: any, - root?: Buffer, - ) { - super(store, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); - } -} - -/** - * The public data tree is an indexed tree. - */ -export class PublicDataTree extends StandardIndexedTree { - constructor( - store: AztecKVStore, - hasher: Hasher, - name: string, - depth: number, - size: bigint = 0n, - _noop: any, - root?: Buffer, - ) { - super(store, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); - } -} - -export interface TreeFactory { - createStandardTree(name: string, depth: number, prefilledSize: number): Promise>; - createNullifierTree(name: string, depth: number, prefilledSize: number): Promise; - createPublicDataTree(name: string, depth: number, prefilledSize: number): Promise; -} diff --git a/yarn-project/merkle-tree/src/load_tree.ts b/yarn-project/merkle-tree/src/load_tree.ts new file mode 100644 index 00000000000..b2a1d396b49 --- /dev/null +++ b/yarn-project/merkle-tree/src/load_tree.ts @@ -0,0 +1,33 @@ +import { type Bufferable, type FromBuffer } from '@aztec/foundation/serialize'; +import { type AztecKVStore } from '@aztec/kv-store'; +import { type Hasher } from '@aztec/types/interfaces'; + +import { type TreeBase, getTreeMeta } from './tree_base.js'; + +/** + * Creates a new tree and sets its root, depth and size based on the meta data which are associated with the name. + * @param c - The class of the tree to be instantiated. + * @param db - A database used to store the Merkle tree data. + * @param hasher - A hasher used to compute hash paths. + * @param name - Name of the tree. + * @returns The newly created tree. + */ +export function loadTree, D extends FromBuffer>( + c: new ( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint, + deserializer: D, + root: Buffer, + ) => T, + store: AztecKVStore, + hasher: Hasher, + name: string, + deserializer: D, +): Promise { + const { root, depth, size } = getTreeMeta(store, name); + const tree = new c(store, hasher, name, depth, size, deserializer, root); + return Promise.resolve(tree); +} diff --git a/yarn-project/merkle-tree/src/native/message.ts b/yarn-project/merkle-tree/src/native/message.ts deleted file mode 100644 index 9dab2155d0d..00000000000 --- a/yarn-project/merkle-tree/src/native/message.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { Fr } from "@aztec/circuits.js"; - -export enum SystemMsgTypes { TERMINATE = 0, PING = 1, PONG = 2 }; - -const FIRST_APP_MSG_TYPE = 100; - -export class MsgHeader { - constructor( - public messageId: number, - public requestId: number, - ) { - - } -} - -export const enum WorldStateMsgTypes { - START_TREE_REQUEST = 100, - START_TREE_RESPONSE, - GET_TREE_INFO_REQUEST, - GET_TREE_INFO_RESPONSE, - INSERT_LEAVES_REQUEST, - INSERT_LEAVES_RESPONSE, -}; - -// export const enum WorldStateMsgTypes { -// START_TREE_REQUEST = FIRST_APP_MSG_TYPE, -// START_TREE_RESPONSE = FIRST_APP_MSG_TYPE + 1, -// GET_TREE_INFO_REQUEST = FIRST_APP_MSG_TYPE + 2, -// GET_TREE_INFO_RESPONSE = FIRST_APP_MSG_TYPE + 3, -// INSERT_LEAVES_REQUEST = FIRST_APP_MSG_TYPE + 4, -// INSERT_LEAVES_RESPONSE = FIRST_APP_MSG_TYPE + 5, -// }; - -export type StartTreeRequest = { - name: string; - depth: number; - preFilledSize: number; -}; - -export type StartTreeResponse = { - name: string; - depth: number; - success: boolean; - message: string; -}; - -export type GetTreeInfoRequest = { - name: string; -} - -export type GetTreeInfoResponse = { - name: string; - depth: number; - success: boolean; - message: string; - root: Buffer; - size: bigint; -} - -export type InsertLeavesRequest = { - name: string; - leaves: Buffer[]; -} - -export type InsertLeavesResponse = { - success: boolean; - message: string; - root: Buffer; - size: bigint; -} - -export type MsgPackedType = { - msgType: number; - header: MsgHeader; - value: T; -} - -export type MsgPackedHeader = MsgPackedType<{}>; \ No newline at end of file diff --git a/yarn-project/merkle-tree/src/native/native_client.ts b/yarn-project/merkle-tree/src/native/native_client.ts deleted file mode 100644 index 06e7a169868..00000000000 --- a/yarn-project/merkle-tree/src/native/native_client.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { DebugLogger, createDebugLogger } from "@aztec/foundation/log"; -import { RunningPromise, promiseWithResolvers } from "@aztec/foundation/promise"; -import { decodeMultiStream, encode } from "@msgpack/msgpack"; -import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; -import { GetTreeInfoRequest, GetTreeInfoResponse, InsertLeavesRequest, InsertLeavesResponse, MsgHeader, MsgPackedHeader, MsgPackedType, StartTreeRequest, StartTreeResponse, SystemMsgTypes, WorldStateMsgTypes } from './message.js'; -import { Fr } from "@aztec/circuits.js"; - -export type NativeWorldStateConfig = { - worldStateBinaryPath: string; -} - -type Request = { - requestId: number; - responseHandler: (responseData: object) => void; -} - -export class NativeTreesClient { - private msgId = 1; - private requests = new Map(); - private terminateResolve?: (data: Buffer) => void; - private runningPromise: RunningPromise; - - constructor(private childProcess: ChildProcessWithoutNullStreams, private logger: DebugLogger) { - this.runningPromise = new RunningPromise(() => this.handleData(), 10); - this.runningPromise.start(); - childProcess.stderr.on("data", (d) => this.logger.debug(`World State Out: ${d}`)); - childProcess.on("close", () => this.onRemoteClose()); - } - public static init(config: NativeWorldStateConfig, logger = createDebugLogger('aztec:native_client')) { - logger.debug(`Starting world state at: ${config.worldStateBinaryPath}`); - const child = spawn(config.worldStateBinaryPath, [], { stdio: ['pipe', 'pipe', 'pipe']}); - return new NativeTreesClient(child, logger); - } - - public async terminate() { - const encoded = encode({header: new MsgHeader(this.msgId++, 0), msgType: SystemMsgTypes.TERMINATE, value: {}} satisfies MsgPackedHeader); - const { promise, resolve } = promiseWithResolvers(); - this.terminateResolve = resolve; - this.childProcess.stdin.write(encoded); - await promise; - this.runningPromise.stop(); - } - - public async sendPing() { - const { promise } = this.sendMessage(SystemMsgTypes.PING); - await promise; - } - - public async sendStartTree(name: string, depth: number, preFilledSize = 1) { - const request: StartTreeRequest = { - name, - depth, - preFilledSize, - }; - const { promise } = this.sendMessage(WorldStateMsgTypes.START_TREE_REQUEST, request); - const response = (await promise) as StartTreeResponse; - return response; - } - - public async getTreeInfo(name: string) { - const request: GetTreeInfoRequest = { - name, - }; - const { promise } = this.sendMessage(WorldStateMsgTypes.GET_TREE_INFO_REQUEST, request); - const response = (await promise) as GetTreeInfoResponse; - return response; - } - - public async insertLeaves(name: string, leaves: Fr[]) { - const request: InsertLeavesRequest = { - name, - leaves: leaves.map(x => x.toBuffer()), - }; - const { promise } = this.sendMessage(WorldStateMsgTypes.INSERT_LEAVES_REQUEST, request); - const response = (await promise) as InsertLeavesResponse; - return response; - } - - private onRemoteClose() { - this.terminateResolve?.(Buffer.alloc(0)); - } - - private sendMessage(msgType: number, msg?: T) { - const msgId = this.msgId++; - const header = new MsgHeader(msgId, 0); - const encoded = msg ? encode({header, msgType, value: msg} satisfies MsgPackedType) : encode({header, msgType, value: {}} satisfies MsgPackedHeader); - const { promise, resolve } = promiseWithResolvers(); - - const request: Request = { - requestId: msgId, - responseHandler: resolve, - } - this.requests.set(msgId, request); - this.childProcess.stdin.write(encoded); - return { promise, resolve }; - } - - private async handleData() { - for await (const item of decodeMultiStream(this.childProcess.stdout)) { - const itemAsHeader: MsgPackedHeader = item as MsgPackedHeader; - const request = this.requests.get(itemAsHeader.header.requestId); - request?.responseHandler(itemAsHeader.value); - } - } -} diff --git a/yarn-project/merkle-tree/src/new_tree.ts b/yarn-project/merkle-tree/src/new_tree.ts new file mode 100644 index 00000000000..d8040fc9f46 --- /dev/null +++ b/yarn-project/merkle-tree/src/new_tree.ts @@ -0,0 +1,29 @@ +import { type Bufferable, type FromBuffer } from '@aztec/foundation/serialize'; +import { type AztecKVStore } from '@aztec/kv-store'; +import { type Hasher } from '@aztec/types/interfaces'; + +import { type TreeBase } from './tree_base.js'; + +/** + * Creates a new tree. + * @param c - The class of the tree to be instantiated. + * @param db - A database used to store the Merkle tree data. + * @param hasher - A hasher used to compute hash paths. + * @param name - Name of the tree. + * @param depth - Depth of the tree. + * @param prefilledSize - A number of leaves that are prefilled with values. + * @returns The newly created tree. + */ +export async function newTree, D extends FromBuffer>( + c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, deserializer: D) => T, + store: AztecKVStore, + hasher: Hasher, + name: string, + deserializer: D, + depth: number, + prefilledSize = 1, +): Promise { + const tree = new c(store, hasher, name, depth, 0n, deserializer); + await tree.init(prefilledSize); + return tree; +} diff --git a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts index fa2355fea1f..eca6d258f36 100644 --- a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts @@ -1,9 +1,10 @@ import { SiblingPath } from '@aztec/circuit-types'; -import { serializeToBuffer, type Bufferable, type FromBuffer } from '@aztec/foundation/serialize'; +import { type Bufferable, type FromBuffer, serializeToBuffer } from '@aztec/foundation/serialize'; import { type AztecKVStore, type AztecMap } from '@aztec/kv-store'; import { type Hasher } from '@aztec/types/interfaces'; -import { SnapshottedMerkleTree } from '../interfaces/merkle_tree.js'; +import { type AppendOnlyTree } from '../interfaces/append_only_tree.js'; +import { type TreeBase } from '../tree_base.js'; import { type TreeSnapshot, type TreeSnapshotBuilder } from './snapshot_builder.js'; // stores the last block that modified this node @@ -44,7 +45,7 @@ export class AppendOnlySnapshotBuilder implements TreeSnap constructor( private db: AztecKVStore, - private tree: SnapshottedMerkleTree, + private tree: TreeBase & AppendOnlyTree, private hasher: Hasher, private deserializer: FromBuffer, ) { @@ -163,7 +164,7 @@ class AppendOnlySnapshot implements TreeSnapshot { private block: number, private leafCount: bigint, private historicalRoot: Buffer, - private tree: SnapshottedMerkleTree, + private tree: TreeBase & AppendOnlyTree, private hasher: Hasher, private deserializer: FromBuffer, ) {} diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts index 19b6c28a2e0..56cbaee0a58 100644 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts +++ b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts @@ -6,8 +6,9 @@ import { type AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type Hasher } from '@aztec/types/interfaces'; -import { INITIAL_LEAF, loadTree, newTree } from '../index.js'; +import { INITIAL_LEAF, newTree } from '../index.js'; import { type UpdateOnlyTree } from '../interfaces/update_only_tree.js'; +import { loadTree } from '../load_tree.js'; import { Pedersen } from '../pedersen.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_native.test.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_native.test.ts deleted file mode 100644 index 836c6142f1e..00000000000 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_native.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { Fr } from '@aztec/foundation/fields'; - -import { MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NullifierLeafPreimage } from '@aztec/circuits.js'; -import { createDebugLogger } from '@aztec/foundation/log'; -import { NativeTreesClient } from '../../native/native_client.js'; -import { Pedersen } from '../../pedersen.js'; -import { getWorldStateConfig } from '../../test/get_native_config.js'; -import { INITIAL_LEAF } from '../../tree_base.js'; -import { SiblingPath } from '@aztec/circuit-types'; - -const logger = createDebugLogger('aztec:standard_native_tree_test'); - -// const noopDeserializer: FromBuffer = { -// fromBuffer: (buffer: Buffer) => buffer, -// }; - -// const createDb = async (store: AztecKVStore, hasher: Hasher, name: string, depth: number) => { -// return await newTree(StandardTree, store, hasher, name, noopDeserializer, depth); -// }; - -// const createFromName = async (store: AztecKVStore, hasher: Hasher, name: string) => { -// return await loadTree(StandardTree, store, hasher, name, noopDeserializer); -// }; - -// treeTestSuite('StandardTree', createDb, createFromName); -// standardBasedTreeTestSuite('StandardTree', createDb); - -const TEST_TREE_DEPTH = 3; - -const createNullifierTreeLeafHashInputs = (value: number, nextIndex: number, nextValue: number) => { - return new NullifierLeafPreimage(new Fr(value), new Fr(nextValue), BigInt(nextIndex)).toHashInputs(); -}; - -describe('Native Indexed Tree', () => { - let nativeClient: NativeTreesClient; - let pedersen: Pedersen; - - beforeAll(async () => { - const config = await getWorldStateConfig(logger); - if (!config?.worldStateBinaryPath) { - throw new Error(`Native world state required!`); - } - nativeClient = NativeTreesClient.init(config); - pedersen = new Pedersen(); - }); - - afterAll(async () => { - await nativeClient.terminate(); - }); - - it('creates a new indexed tree', async () => { - const treeName = "Public Data Tree"; - const response = await nativeClient.sendStartTree(treeName, 32); - expect(response.success).toBeTruthy(); - - const response2 = await nativeClient.sendStartTree(treeName, 33); - expect(response2.success).toBeFalsy(); - expect(response2.message).toEqual("Tree already exists"); - expect(response2.depth).toBe(32); - }); - - it('returns the tree information', async () => { - const treeName = "Test Tree 1"; - const response = await nativeClient.sendStartTree(treeName, TEST_TREE_DEPTH); - expect(response.success).toBeTruthy(); - - const getInfoResponse = await nativeClient.getTreeInfo(treeName); - expect(getInfoResponse.success).toBeTruthy(); - expect(getInfoResponse.depth).toBe(TEST_TREE_DEPTH); - expect(getInfoResponse.size).toBe(1); - - /** - * Initial state: - * - * index 0 1 2 3 4 5 6 7 - * --------------------------------------------------------------------- - * val 0 0 0 0 0 0 0 0 - * nextIdx 0 0 0 0 0 0 0 0 - * nextVal 0 0 0 0 0 0 0 0. - */ - - const initialLeafHash = pedersen.hashInputs(createNullifierTreeLeafHashInputs(0, 0, 0)); - const level1ZeroHash = pedersen.hash(INITIAL_LEAF, INITIAL_LEAF); - const level2ZeroHash = pedersen.hash(level1ZeroHash, level1ZeroHash); - - let index0Hash = initialLeafHash; - // Each element is named by the level followed by the index on that level. E.g. e10 -> level 1, index 0, e21 -> level 2, index 1 - let e10 = pedersen.hash(index0Hash, INITIAL_LEAF); - let e20 = pedersen.hash(e10, level1ZeroHash); - - let root = pedersen.hash(e20, level2ZeroHash); - - expect(getInfoResponse.root).toEqual(root); - }); - - it('inserts new leaves', async () => { - const treeName = "Test Tree 2"; - const treeDepth = 32; - const initialSize = 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX; - const response = await nativeClient.sendStartTree(treeName, treeDepth, initialSize); - expect(response.success).toBeTruthy(); - - const getInfoResponse = await nativeClient.getTreeInfo(treeName); - expect(getInfoResponse.success).toBeTruthy(); - expect(getInfoResponse.depth).toBe(treeDepth); - expect(getInfoResponse.size).toBe(initialSize); - - const numLeavesToInsert = 12 * 1024; - const leaves = Array.from({length: numLeavesToInsert}).map(Fr.random); - const insertLeavesResponse = await nativeClient.insertLeaves(treeName, leaves); - expect(insertLeavesResponse.success).toBeTruthy(); - expect(insertLeavesResponse.size).toBe(initialSize + leaves.length); - }, 30_000); - - // it('should be able to find indexes of leaves', async () => { - // const db = openTmpStore(); - // const tree = await createDb(db, pedersen, 'test', 3); - // const values = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; - - // await tree.appendLeaves([values[0]]); - - // expect(tree.findLeafIndex(values[0], true)).toBeDefined(); - // expect(tree.findLeafIndex(values[0], false)).toBe(undefined); - // expect(tree.findLeafIndex(values[1], true)).toBe(undefined); - - // await tree.commit(); - - // expect(tree.findLeafIndex(values[0], false)).toBeDefined(); - // }); -}); diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts index 712bdb6e2f0..b01409eb14c 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts @@ -4,12 +4,13 @@ import { type AztecKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type Hasher } from '@aztec/types/interfaces'; +import { loadTree } from '../load_tree.js'; +import { newTree } from '../new_tree.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; import { PedersenWithCounter } from '../test/utils/pedersen_with_counter.js'; import { INITIAL_LEAF } from '../tree_base.js'; import { StandardTree } from './standard_tree.js'; -import { loadTree, newTree } from '../factory/js_tree_factory.js'; const noopDeserializer: FromBuffer = { fromBuffer: (buffer: Buffer) => buffer, diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts deleted file mode 100644 index 267617212a5..00000000000 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree_native.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { type TreeInsertionStats } from '@aztec/circuit-types/stats'; -import { type Bufferable, serializeToBuffer, FromBuffer } from '@aztec/foundation/serialize'; -import { Timer } from '@aztec/foundation/timer'; - -import { AppendOnlySnapshotBuilder } from '../snapshots/append_only_snapshot.js'; -import { type TreeSnapshot } from '../snapshots/snapshot_builder.js'; -import { Hasher } from '@aztec/types/interfaces'; -import { SnapshottedMerkleTree } from '../interfaces/merkle_tree.js'; -import { SiblingPath } from '@aztec/circuit-types'; -import { AztecKVStore } from '@aztec/kv-store'; -import { INITIAL_LEAF, MAX_DEPTH } from '../tree_base.js'; -import { DebugLogger, createDebugLogger } from '@aztec/foundation/log'; - -/** - * A Merkle tree implementation that uses a LevelDB database to store the tree. - */ -export class StandardTreeNative implements SnapshottedMerkleTree { - #snapshotBuilder = new AppendOnlySnapshotBuilder(this.store, this, this.hasher, this.deserializer); - private zeroHashes: Buffer[] = []; - protected readonly maxIndex: bigint; - protected log: DebugLogger; - - public constructor( - protected store: AztecKVStore, - protected hasher: Hasher, - private name: string, - private depth: number, - protected size: bigint = 0n, - protected deserializer: FromBuffer, - root?: Buffer, - ) { - if (!(depth >= 1 && depth <= MAX_DEPTH)) { - throw Error('Invalid depth'); - } - - // Compute the zero values at each layer. - let current = INITIAL_LEAF; - for (let i = depth - 1; i >= 0; --i) { - this.zeroHashes[i] = current; - current = hasher.hash(current, current); - } - this.maxIndex = 2n ** BigInt(depth) - 1n; - - this.log = createDebugLogger(`aztec:merkle-tree:${name.toLowerCase()}`); - } - getName(): string { - return this.name; - } - getNode(level: number, index: bigint): Buffer { - throw new Error('Method not implemented.'); - } - getZeroHash(level: number): Buffer { - return this.zeroHashes[level]; - } - getRoot(includeUncommitted: boolean): Buffer { - throw new Error('Method not implemented.'); - } - getNumLeaves(includeUncommitted: boolean): bigint { - throw new Error('Method not implemented.'); - } - commit(): Promise { - throw new Error('Method not implemented.'); - } - getDepth(): number { - throw new Error('Method not implemented.'); - } - rollback(): Promise { - throw new Error('Method not implemented.'); - } - getLeafValue(index: bigint, includeUncommitted: boolean): T | undefined { - throw new Error('Method not implemented.'); - } - getSiblingPath(index: bigint, includeUncommitted: boolean): Promise> { - throw new Error('Method not implemented.'); - } - - /** - * Appends the given leaves to the tree. - * @param leaves - The leaves to append. - * @returns Empty promise. - */ - public appendLeaves(leaves: T[]): Promise { - - return Promise.resolve(); - } - - public snapshot(blockNumber: number): Promise> { - return this.#snapshotBuilder.snapshot(blockNumber); - } - - public getSnapshot(blockNumber: number): Promise> { - return this.#snapshotBuilder.getSnapshot(blockNumber); - } - - public findLeafIndex(value: T, includeUncommitted: boolean): bigint | undefined { - return this.findLeafIndexAfter(value, 0n, includeUncommitted); - } - - public findLeafIndexAfter(value: T, startIndex: bigint, includeUncommitted: boolean): bigint | undefined { - const buffer = serializeToBuffer(value); - for (let i = startIndex; i < this.getNumLeaves(includeUncommitted); i++) { - const currentValue = this.getLeafValue(i, includeUncommitted); - if (currentValue && serializeToBuffer(currentValue).equals(buffer)) { - return i; - } - } - return undefined; - } -} diff --git a/yarn-project/merkle-tree/src/test/get_native_config.ts b/yarn-project/merkle-tree/src/test/get_native_config.ts deleted file mode 100644 index 158001515ce..00000000000 --- a/yarn-project/merkle-tree/src/test/get_native_config.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { type DebugLogger, fileURLToPath } from '@aztec/aztec.js'; - -import fs from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import path from 'path'; - -const { - BB_RELEASE_DIR = 'barretenberg/cpp/build/bin', - WORLD_STATE_BINARY_PATH, - TEMP_DIR = tmpdir(), - DATA_DIRECTORY = '', -} = process.env; - -export const getWorldStateConfig = async ( - logger: DebugLogger, -): Promise<{ worldStateBinaryPath: string; } | undefined> => { - try { - const worldStateBinaryPath = - WORLD_STATE_BINARY_PATH ?? - path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../', BB_RELEASE_DIR, 'world_state'); - await fs.access(worldStateBinaryPath, fs.constants.R_OK); - - return { worldStateBinaryPath }; - } catch (err) { - logger.error(`Native BB not available, error: ${err}`); - return undefined; - } -}; diff --git a/yarn-project/merkle-tree/src/tree_base.ts b/yarn-project/merkle-tree/src/tree_base.ts index a842d37bb17..cd20e8e0f2d 100644 --- a/yarn-project/merkle-tree/src/tree_base.ts +++ b/yarn-project/merkle-tree/src/tree_base.ts @@ -8,7 +8,7 @@ import { type Hasher } from '@aztec/types/interfaces'; import { HasherWithStats } from './hasher_with_stats.js'; import { type MerkleTree } from './interfaces/merkle_tree.js'; -export const MAX_DEPTH = 254; +const MAX_DEPTH = 254; const indexToKeyHash = (name: string, level: number, index: bigint) => `${name}:${level}:${index}`; const encodeMeta = (root: Buffer, depth: number, size: bigint) => { diff --git a/yarn-project/prover-client/package.json b/yarn-project/prover-client/package.json index 78ec16a9283..85080060741 100644 --- a/yarn-project/prover-client/package.json +++ b/yarn-project/prover-client/package.json @@ -55,7 +55,6 @@ "@aztec/circuits.js": "workspace:^", "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", - "@aztec/merkle-tree": "workspace:^", "@aztec/noir-protocol-circuits-types": "workspace:^", "@aztec/simulator": "workspace:^", "@aztec/world-state": "workspace:^", diff --git a/yarn-project/prover-client/src/mocks/test_context.ts b/yarn-project/prover-client/src/mocks/test_context.ts index 68ffa788527..1af8c556c48 100644 --- a/yarn-project/prover-client/src/mocks/test_context.ts +++ b/yarn-project/prover-client/src/mocks/test_context.ts @@ -19,7 +19,6 @@ import { import { type Fr } from '@aztec/foundation/fields'; import { type DebugLogger } from '@aztec/foundation/log'; import { openTmpStore } from '@aztec/kv-store/utils'; -import { JSTreeFactory } from '@aztec/merkle-tree'; import { type ContractsDataSourcePublicDB, type PublicExecution, @@ -95,8 +94,7 @@ export class TestContext { const publicContractsDB = mock(); const publicWorldStateDB = mock(); const publicKernel = new RealPublicKernelCircuitSimulator(new WASMSimulator()); - const kvStore = openTmpStore(); - const actualDb = await MerkleTrees.new(kvStore, await JSTreeFactory.init(kvStore)).then(t => t.asLatest()); + const actualDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); const processor = new PublicProcessor( actualDb, publicExecutor, diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts index 26be8d9ce37..c9f8b930df2 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_mixed_blocks_2.test.ts @@ -5,7 +5,6 @@ import { range } from '@aztec/foundation/array'; import { times } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; import { openTmpStore } from '@aztec/kv-store/utils'; -import { JSTreeFactory } from '@aztec/merkle-tree'; import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { makeBloatedProcessedTx, makeEmptyProcessedTestTx, updateExpectedTreesFromTxs } from '../mocks/fixtures.js'; @@ -19,8 +18,7 @@ describe('prover/orchestrator/mixed-blocks', () => { beforeEach(async () => { context = await TestContext.new(logger); - const kvStore = openTmpStore(); - expectsDb = await MerkleTrees.new(kvStore, await JSTreeFactory.init(kvStore)).then(t => t.asLatest()); + expectsDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); }); afterEach(async () => { diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts index 1d4b2e53a00..808aa3dae8d 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts @@ -5,7 +5,6 @@ import { range } from '@aztec/foundation/array'; import { createDebugLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; import { openTmpStore } from '@aztec/kv-store/utils'; -import { JSTreeFactory } from '@aztec/merkle-tree'; import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { makeBloatedProcessedTx, makeEmptyProcessedTestTx, updateExpectedTreesFromTxs } from '../mocks/fixtures.js'; @@ -19,8 +18,7 @@ describe('prover/orchestrator/blocks', () => { beforeEach(async () => { context = await TestContext.new(logger); - const kvStore = openTmpStore(); - expectsDb = await MerkleTrees.new(kvStore, await JSTreeFactory.init(kvStore)).then(t => t.asLatest()); + expectsDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); }); afterEach(async () => { diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts index 764b37007dc..07158f9aeec 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts @@ -11,7 +11,6 @@ import { makeGlobalVariables, makeRootParityInput } from '@aztec/circuits.js/tes import { promiseWithResolvers } from '@aztec/foundation/promise'; import { sleep } from '@aztec/foundation/sleep'; import { openTmpStore } from '@aztec/kv-store/utils'; -import { JSTreeFactory } from '@aztec/merkle-tree'; import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -24,8 +23,7 @@ describe('prover/orchestrator', () => { let mockProver: MockProxy; let actualDb: MerkleTreeOperations; beforeEach(async () => { - const kvStore = openTmpStore(); - actualDb = await MerkleTrees.new(kvStore, await JSTreeFactory.init(kvStore)).then(t => t.asLatest()); + actualDb = await MerkleTrees.new(openTmpStore()).then(t => t.asLatest()); mockProver = mock(); orchestrator = new ProvingOrchestrator(actualDb, mockProver); }); diff --git a/yarn-project/prover-client/tsconfig.json b/yarn-project/prover-client/tsconfig.json index 5f748569b8c..5f4666ebf03 100644 --- a/yarn-project/prover-client/tsconfig.json +++ b/yarn-project/prover-client/tsconfig.json @@ -21,9 +21,6 @@ { "path": "../kv-store" }, - { - "path": "../merkle-tree" - }, { "path": "../noir-protocol-circuits-types" }, @@ -34,5 +31,5 @@ "path": "../world-state" } ], - "include": ["src"] + "include": ["src", "../bb-prover/src/test/test_circuit_prover.ts"] } diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index d781daddf6e..be41f087fdc 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -1,4 +1,4 @@ -import { MerkleTreeId, PublicDataWrite, TxEffect, type L2Block, type SiblingPath } from '@aztec/circuit-types'; +import { type L2Block, MerkleTreeId, PublicDataWrite, type SiblingPath, TxEffect } from '@aztec/circuit-types'; import { ARCHIVE_HEIGHT, AppendOnlyTreeSnapshot, @@ -14,30 +14,33 @@ import { NULLIFIER_SUBTREE_HEIGHT, NULLIFIER_TREE_HEIGHT, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, + NullifierLeaf, + NullifierLeafPreimage, PUBLIC_DATA_SUBTREE_HEIGHT, PUBLIC_DATA_TREE_HEIGHT, PartialStateReference, PublicDataTreeLeaf, - StateReference + PublicDataTreeLeafPreimage, + StateReference, } from '@aztec/circuits.js'; import { padArrayEnd } from '@aztec/foundation/collection'; import { SerialQueue } from '@aztec/foundation/fifo'; -import { createDebugLogger, type DebugLogger } from '@aztec/foundation/log'; +import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { type AztecKVStore } from '@aztec/kv-store'; import { + type AppendOnlyTree, + type BatchInsertionResult, + type IndexedTree, Pedersen, - type StandardIndexedTree, + StandardIndexedTree, StandardTree, + type UpdateOnlyTree, getTreeMeta, loadTree, newTree, - type AppendOnlyTree, - type BatchInsertionResult, - type IndexedTree, - type UpdateOnlyTree, - type TreeFactory, } from '@aztec/merkle-tree'; +import { type Hasher } from '@aztec/types/interfaces'; import { INITIAL_NULLIFIER_TREE_SIZE, @@ -55,6 +58,39 @@ import { } from './merkle_tree_operations.js'; import { MerkleTreeOperationsFacade } from './merkle_tree_operations_facade.js'; +/** + * The nullifier tree is an indexed tree. + */ +class NullifierTree extends StandardIndexedTree { + constructor( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint = 0n, + _noop: any, + root?: Buffer, + ) { + super(store, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); + } +} + +/** + * The public data tree is an indexed tree. + */ +class PublicDataTree extends StandardIndexedTree { + constructor( + store: AztecKVStore, + hasher: Hasher, + name: string, + depth: number, + size: bigint = 0n, + _noop: any, + root?: Buffer, + ) { + super(store, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); + } +} /** * A convenience class for managing multiple merkle trees. @@ -71,42 +107,62 @@ export class MerkleTrees implements MerkleTreeDb { * @param store - The db instance to use for data persistance. * @returns - A fully initialized MerkleTrees instance. */ - public static async new(store: AztecKVStore, treeFactory: TreeFactory, log = createDebugLogger('aztec:merkle_trees')) { + public static async new(store: AztecKVStore, log = createDebugLogger('aztec:merkle_trees')) { const merkleTrees = new MerkleTrees(store, log); - await merkleTrees.#init(treeFactory); + await merkleTrees.#init(); return merkleTrees; } /** * Initializes the collection of Merkle Trees. */ - async #init(treeFactory: TreeFactory) { + async #init() { const fromDb = this.#isDbPopulated(); const initializeTree = fromDb ? loadTree : newTree; const hasher = new Pedersen(); - const nullifierTree = await treeFactory.createNullifierTree(`${MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]}`, NULLIFIER_TREE_HEIGHT, INITIAL_NULLIFIER_TREE_SIZE); - - const noteHashTree: AppendOnlyTree = await treeFactory.createStandardTree( + const nullifierTree = await initializeTree( + NullifierTree, + this.store, + hasher, + `${MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]}`, + {}, + NULLIFIER_TREE_HEIGHT, + INITIAL_NULLIFIER_TREE_SIZE, + ); + const noteHashTree: AppendOnlyTree = await initializeTree( + StandardTree, + this.store, + hasher, `${MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE]}`, + Fr, NOTE_HASH_TREE_HEIGHT, - 1 ); - const publicDataTree = await treeFactory.createPublicDataTree( + const publicDataTree = await initializeTree( + PublicDataTree, + this.store, + hasher, `${MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE]}`, + {}, PUBLIC_DATA_TREE_HEIGHT, INITIAL_PUBLIC_DATA_TREE_SIZE, ); - const l1Tol2MessageTree: AppendOnlyTree = await treeFactory.createStandardTree( + const l1Tol2MessageTree: AppendOnlyTree = await initializeTree( + StandardTree, + this.store, + hasher, `${MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]}`, + Fr, L1_TO_L2_MSG_TREE_HEIGHT, - 1 ); - const archive: AppendOnlyTree = await treeFactory.createStandardTree( + const archive: AppendOnlyTree = await initializeTree( + StandardTree, + this.store, + hasher, `${MerkleTreeId[MerkleTreeId.ARCHIVE]}`, + Fr, ARCHIVE_HEIGHT, - 1, ); this.trees = [nullifierTree, noteHashTree, publicDataTree, l1Tol2MessageTree, archive]; diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 98fb2afd255..d1fe44fd9df 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -624,7 +624,6 @@ __metadata: "@aztec/kv-store": "workspace:^" "@aztec/types": "workspace:^" "@jest/globals": ^29.5.0 - "@msgpack/msgpack": ^3.0.0-beta2 "@types/jest": ^29.5.0 "@types/node": ^18.15.3 "@types/sha256": ^0.2.0 @@ -775,7 +774,6 @@ __metadata: "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" - "@aztec/merkle-tree": "workspace:^" "@aztec/noir-protocol-circuits-types": "workspace:^" "@aztec/simulator": "workspace:^" "@aztec/world-state": "workspace:^" @@ -2747,13 +2745,6 @@ __metadata: languageName: node linkType: hard -"@msgpack/msgpack@npm:^3.0.0-beta2": - version: 3.0.0-beta2 - resolution: "@msgpack/msgpack@npm:3.0.0-beta2" - checksum: d86e5d48146051952d6bea35a6cf733a401cf65ad5614d79689aa48c7076021737ca2c782978dd1b6c0c9c45888b246e379e45ae906179e3a0e8ef4ee6f221c1 - languageName: node - linkType: hard - "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.2": version: 3.0.2 resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.2" From 044301d82888d9fc03c4669a20b2a7d12ed124bf Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 17 Jul 2024 06:32:33 +0000 Subject: [PATCH 47/63] chore: change preset to clang16 --- barretenberg/cpp/build_and_test.sh | 10 ---------- barretenberg/cpp/scripts/merkle_tree_tests.sh | 13 +++++++++++++ 2 files changed, 13 insertions(+), 10 deletions(-) delete mode 100755 barretenberg/cpp/build_and_test.sh create mode 100755 barretenberg/cpp/scripts/merkle_tree_tests.sh diff --git a/barretenberg/cpp/build_and_test.sh b/barretenberg/cpp/build_and_test.sh deleted file mode 100755 index 6c5659a6314..00000000000 --- a/barretenberg/cpp/build_and_test.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -e - -TEST=${1:-*} - -cmake --build --preset default --target crypto_merkle_tree_tests -./build/bin/crypto_merkle_tree_tests --gtest_filter=PersistedIndexedTreeTest.$TEST -./build/bin/crypto_merkle_tree_tests --gtest_filter=PersistedAppendOnlyTreeTest.$TEST -./build/bin/crypto_merkle_tree_tests --gtest_filter=LMDBStoreTest.$TEST diff --git a/barretenberg/cpp/scripts/merkle_tree_tests.sh b/barretenberg/cpp/scripts/merkle_tree_tests.sh new file mode 100755 index 00000000000..6cd44b2374c --- /dev/null +++ b/barretenberg/cpp/scripts/merkle_tree_tests.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +# run commands relative to parent directory +cd $(dirname $0)/.. + +DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBStoreTest.* +TEST=${1:-$DEFAULT_TESTS} +PRESET=${PRESET:-clang16} + +cmake --build --preset $PRESET --target crypto_merkle_tree_tests +./build/bin/crypto_merkle_tree_tests --gtest_filter=$TEST From 7e3ede76d6a3ac0fcb5e40a0edebd9a37d930781 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 17 Jul 2024 08:58:16 +0000 Subject: [PATCH 48/63] ci: add merkle tree tests to earthly --- .github/workflows/ci.yml | 2 +- barretenberg/cpp/Earthfile | 4 +- barretenberg/cpp/bootstrap.sh | 1 - barretenberg/cpp/src/CMakeLists.txt | 2 - .../append_only_tree_bench/CMakeLists.txt | 2 +- .../indexed_tree_bench/CMakeLists.txt | 2 +- .../indexed_tree_bench/indexed_tree.bench.cpp | 4 +- .../merkle_tree_bench/CMakeLists.txt | 2 +- .../benchmark/ultra_bench/CMakeLists.txt | 4 +- .../nullifier_memory_tree.test.cpp | 157 ++++++++------- .../nullifier_tree/nullifier_tree.cpp | 6 +- .../nullifier_tree/nullifier_tree.test.cpp | 17 +- .../barretenberg/world_state/CMakeLists.txt | 1 - .../world_state/service/message.hpp | 74 ------- .../service/world_state_service.hpp | 180 ------------------ .../world_state_service/CMakeLists.txt | 4 - .../barretenberg/world_state_service/main.cpp | 45 ----- .../world_state_service/std_io.hpp | 61 ------ 18 files changed, 106 insertions(+), 462 deletions(-) delete mode 100644 barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt delete mode 100644 barretenberg/cpp/src/barretenberg/world_state/service/message.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt delete mode 100644 barretenberg/cpp/src/barretenberg/world_state_service/main.cpp delete mode 100644 barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02ee4253c10..21d59c1f040 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -274,7 +274,7 @@ jobs: # limit our parallelism to half our cores run: earthly-ci --no-output +preset-gcc - # barretenberg (prover) native and AVM (public VM) tests + # barretenberg (prover) native, AVM (public VM) and Merkle tree (world state) tests # only ran on x86 for resource reasons (memory intensive) bb-native-tests: needs: [build-images, changes] diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index f340432b505..b3495f45457 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -31,7 +31,7 @@ preset-release: preset-release-assert: FROM +source - RUN cmake --preset clang16 -Bbuild && cmake --build build --target bb && rm -rf build/{deps,lib,src} + RUN cmake --preset clang16 -Bbuild && cmake --build build --target bb crypto_merkle_tree_tests && rm -rf build/{deps,lib,src} SAVE ARTIFACT build/bin preset-debug: @@ -266,4 +266,4 @@ vm-full-test: build: BUILD +preset-wasm BUILD +preset-wasm-threads - BUILD +preset-release \ No newline at end of file + BUILD +preset-release diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index fc8aa65dd39..bd82cc67e1c 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -57,7 +57,6 @@ echo "#################################" function build_native { cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert cmake --build --preset $PRESET --target bb - cmake --build --preset $PRESET --target world_state } function build_wasm { diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 0292e6b8c0f..2bdbb2b1a71 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -89,8 +89,6 @@ add_subdirectory(barretenberg/translator_vm_recursion) add_subdirectory(barretenberg/ultra_honk) add_subdirectory(barretenberg/vm) add_subdirectory(barretenberg/wasi) -add_subdirectory(barretenberg/world_state) -add_subdirectory(barretenberg/world_state_service) if(SMT) add_subdirectory(barretenberg/smt_verification) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt index 7270e7e9530..c2f2a6c01c5 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(append_only_tree_bench crypto_poseidon2 crypto_merkle_tree) +barretenberg_module(append_only_tree_bench crypto_poseidon2 crypto_pedersen_hash crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt index 998bbdf55de..59caee96f97 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(indexed_tree_bench crypto_poseidon2 crypto_merkle_tree) +barretenberg_module(indexed_tree_bench crypto_poseidon2 crypto_pedersen_hash crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp index 65ba9384a75..6d6cd393a91 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp @@ -23,7 +23,7 @@ const size_t MAX_BATCH_SIZE = 128; template void add_values(TreeType& tree, const std::vector& values) { Signal signal(1); - auto completion = [&](const TypedResponse&) -> void { signal.signal_level(0); }; + auto completion = [&](const auto&) -> void { signal.signal_level(0); }; tree.add_or_update_values(values, completion); signal.wait_for_level(0); @@ -105,4 +105,4 @@ BENCHMARK(multi_thread_indexed_tree_bench) ->Range(2, MAX_BATCH_SIZE) ->Iterations(1000); -BENCHMARK_MAIN(); \ No newline at end of file +BENCHMARK_MAIN(); diff --git a/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt index f497e924d56..c5501d1016a 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(merkle_tree_bench crypto_poseidon2 crypto_merkle_tree) +barretenberg_module(merkle_tree_bench crypto_poseidon2 crypto_pedersen_hash crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt index 2fe5a61098b..92d6126f83c 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt @@ -3,6 +3,8 @@ barretenberg_module( ultra_honk stdlib_sha256 stdlib_keccak + stdlib_pedersen_hash + stdlib_poseidon2 crypto_merkle_tree - plonk + plonk ) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp index bcccaa0472f..e3cfcdb4f06 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp @@ -53,9 +53,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) * nextIdx 0 0 0 0 0 0 0 0 * nextVal 0 0 0 0 0 0 0 0 */ - indexed_nullifier_leaf zero_leaf = { 0, 0, 0 }; - EXPECT_EQ(tree.get_leaves().size(), 1); - EXPECT_EQ(tree.get_leaves()[0].unwrap(), zero_leaf); + indexed_nullifier_leaf first_leaf = { 0, 1, 1 }; + EXPECT_EQ(tree.get_leaves().size(), 2); + EXPECT_EQ(tree.get_leaves()[0].unwrap(), first_leaf); /** * Add new value 30: @@ -67,9 +67,10 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) * nextVal 30 0 0 0 0 0 0 0 */ tree.update_element(30); - EXPECT_EQ(tree.get_leaves().size(), 2); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves().size(), 3); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 2, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); /** * Add new value 10: @@ -81,10 +82,11 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) * nextVal 10 0 30 0 0 0 0 0 */ tree.update_element(10); - EXPECT_EQ(tree.get_leaves().size(), 3); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves().size(), 4); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 3, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 10, 2, 30 }).hash()); /** * Add new value 20: @@ -96,19 +98,21 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) * nextVal 10 0 20 30 0 0 0 0 */ tree.update_element(20); - EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves().size(), 5); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 3, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 10, 4, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 20, 2, 30 }).hash()); // Adding the same value must not affect anything tree.update_element(20); - EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves().size(), 5); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 3, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 10, 4, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 20, 2, 30 }).hash()); /** * Add new value 50: @@ -120,12 +124,13 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) * nextVal 10 50 20 30 0 0 0 0 */ tree.update_element(50); - EXPECT_EQ(tree.get_leaves().size(), 5); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 4, 50 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 50, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves().size(), 6); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 3, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 5, 50 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 10, 4, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 20, 2, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[5].hash(), WrappedLeaf({ 50, 0, 0 }).hash()); // Manually compute the node values auto e000 = tree.get_leaves()[0].hash(); @@ -133,9 +138,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) auto e010 = tree.get_leaves()[2].hash(); auto e011 = tree.get_leaves()[3].hash(); auto e100 = tree.get_leaves()[4].hash(); - auto e101 = WrappedLeaf(zero_leaf).hash(); - auto e110 = WrappedLeaf(zero_leaf).hash(); - auto e111 = WrappedLeaf(zero_leaf).hash(); + auto e101 = tree.get_leaves()[5].hash(); + auto e110 = fr::zero(); + auto e111 = fr::zero(); auto e00 = HashPolicy::hash_pair(e000, e001); auto e01 = HashPolicy::hash_pair(e010, e011); @@ -182,9 +187,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) * nextIdx 0 0 0 0 0 0 0 0 * nextVal 0 0 0 0 0 0 0 0 */ - WrappedLeaf zero_leaf = WrappedLeaf({ 0, 0, 0 }); - EXPECT_EQ(tree.get_leaves().size(), 1); - EXPECT_EQ(tree.get_leaves()[0], zero_leaf); + WrappedLeaf first_leaf = WrappedLeaf({ 0, 1, 1 }); + EXPECT_EQ(tree.get_leaves().size(), 2); + EXPECT_EQ(tree.get_leaves()[0], first_leaf); /** * Add new value 30: @@ -196,9 +201,10 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) * nextVal 30 0 0 0 0 0 0 0 */ tree.update_element(30); - EXPECT_EQ(tree.get_leaves().size(), 2); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves().size(), 3); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 2, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); /** * Add new value 10: @@ -210,10 +216,11 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) * nextVal 10 0 30 0 0 0 0 0 */ tree.update_element(10); - EXPECT_EQ(tree.get_leaves().size(), 3); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves().size(), 4); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 3, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 10, 2, 30 }).hash()); /** * Add new value 20: @@ -225,19 +232,21 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) * nextVal 10 0 20 30 0 0 0 0 */ tree.update_element(20); - EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves().size(), 5); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 3, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 10, 4, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 20, 2, 30 }).hash()); // Adding the same value must not affect anything tree.update_element(20); - EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves().size(), 5); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 3, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 10, 4, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 20, 2, 30 }).hash()); /** * Add new value 0: @@ -249,12 +258,13 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) * nextVal 10 0 20 30 0 0 0 0 */ tree.update_element(0); - EXPECT_EQ(tree.get_leaves().size(), 5); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves().size(), 6); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 3, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 10, 4, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 20, 2, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[5].hash(), WrappedLeaf::zero().hash()); /* * Add new value 0: @@ -266,13 +276,14 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) * nextVal 10 0 20 30 0 0 0 0 */ tree.update_element(0); - EXPECT_EQ(tree.get_leaves().size(), 6); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves().size(), 7); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 3, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 10, 4, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 20, 2, 30 }).hash()); EXPECT_EQ(tree.get_leaves()[5].hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves()[6].hash(), WrappedLeaf::zero().hash()); /** * Add new value 50: @@ -284,15 +295,16 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) * nextVal 10 50 20 30 0 0 0 0 */ tree.update_element(50); - EXPECT_EQ(tree.get_leaves().size(), 7); - EXPECT_EQ(tree.get_leaf(0).hash(), WrappedLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaf(1).hash(), WrappedLeaf({ 30, 6, 50 }).hash()); - EXPECT_EQ(tree.get_leaf(2).hash(), WrappedLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaf(3).hash(), WrappedLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaf(4).hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves().size(), 8); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 1 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 1, 3, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 30, 7, 50 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 10, 4, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 20, 2, 30 }).hash()); EXPECT_EQ(tree.get_leaf(5).hash(), WrappedLeaf::zero().hash()); - EXPECT_EQ(tree.get_leaf(6).hash(), WrappedLeaf({ 50, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaf(7).hash(), zero_leaf.hash()); + EXPECT_EQ(tree.get_leaf(6).hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(7).hash(), WrappedLeaf({ 50, 0, 0 }).hash()); + // EXPECT_EQ(tree.get_leaf(7).hash(), first_leaf.hash()); // Manually compute the node values auto e000 = tree.get_leaf(0).hash(); @@ -347,9 +359,8 @@ TEST(crypto_nullifier_tree, test_nullifier_tree) constexpr size_t depth = 8; NullifierMemoryTree tree(depth); - indexed_nullifier_leaf zero_leaf = { 0, 0, 0 }; - EXPECT_EQ(tree.get_leaves().size(), 1); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf(zero_leaf).hash()); + EXPECT_EQ(tree.get_leaves().size(), 2); // prefill is 2 + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf(indexed_nullifier_leaf{ 0, 1, 1 }).hash()); // Add 20 random values to the tree for (size_t i = 0; i < 20; i++) { @@ -382,4 +393,4 @@ TEST(crypto_nullifier_tree, test_nullifier_tree) // Merkle proof at `index` proves non-membership of `new_member` auto hash_path = tree.get_hash_path(index); EXPECT_TRUE(check_hash_path(tree.root(), hash_path, leaves[index].unwrap(), index)); -} \ No newline at end of file +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp index 0b7f3adffe9..a6e43cfcc58 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp @@ -20,9 +20,7 @@ NullifierTree::NullifierTree(Store& store, size_t depth, s zero_hashes_.resize(depth); // Create the zero hashes for the tree - auto current = - WrappedNullifierLeaf(indexed_nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }) - .hash(); + auto current = fr::zero(); for (size_t i = 0; i < depth; ++i) { zero_hashes_[i] = current; current = HashingPolicy::hash_pair(current, current); @@ -87,4 +85,4 @@ fr NullifierTree::update_element(fr const& value) template class NullifierTree; -} // namespace bb::crypto::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.test.cpp index 41ddc1670b2..63622db5643 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.test.cpp @@ -39,26 +39,27 @@ inline void print_tree(const size_t depth, std::vector hashes, std::string c TEST(stdlib_nullifier_tree, test_kv_memory_vs_memory_consistency) { constexpr size_t depth = 2; - NullifierMemoryTree memdb(depth); + constexpr size_t prefill = 2; + NullifierMemoryTree memdb(depth, prefill); MemoryStore store; - NullifierTree db(store, depth); + NullifierTree db(store, depth, prefill); EXPECT_EQ(db.root(), memdb.root()); - std::vector indicies(1 << depth); + std::vector indicies((1 << depth) - prefill); std::iota(indicies.begin(), indicies.end(), 0); std::random_device rd; std::mt19937 g(rd()); std::shuffle(indicies.begin(), indicies.end(), g); - for (size_t i = 0; i < indicies.size() - 1; ++i) { + for (size_t i = 0; i < indicies.size(); ++i) { size_t idx = indicies[i]; memdb.update_element(VALUES[idx]); db.update_element(VALUES[idx]); } - for (size_t i = 0; i < indicies.size() - 1; ++i) { + for (size_t i = 0; i < indicies.size(); ++i) { size_t idx = indicies[i]; EXPECT_EQ(db.get_hash_path(idx), memdb.get_hash_path(idx)); } @@ -93,10 +94,10 @@ TEST(stdlib_nullifier_tree, test_size) TEST(stdlib_nullifier_tree, test_get_hash_path) { - NullifierMemoryTree memdb(10); + NullifierMemoryTree memdb(10, 2); MemoryStore store; - auto db = NullifierTree(store, 10); + auto db = NullifierTree(store, 10, 2); EXPECT_EQ(memdb.get_hash_path(512), db.get_hash_path(512)); @@ -140,4 +141,4 @@ TEST(stdlib_nullifier_tree, test_get_hash_path_layers) EXPECT_EQ(before[1], after[1]); EXPECT_NE(before[2], after[2]); } -} \ No newline at end of file +} diff --git a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt deleted file mode 100644 index 8c4b0296ad3..00000000000 --- a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -barretenberg_module(world_state crypto_merkle_tree stdlib_pedersen_hash) diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/message.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/message.hpp deleted file mode 100644 index 4ae6bad5f5d..00000000000 --- a/barretenberg/cpp/src/barretenberg/world_state/service/message.hpp +++ /dev/null @@ -1,74 +0,0 @@ -#pragma once -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/messaging/header.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/serialize/msgpack_impl/struct_map_impl.hpp" -#include -#include - -namespace bb::world_state { - -using namespace bb::messaging; - -enum WorldStateMsgTypes { - START_TREE_REQUEST = FIRST_APP_MSG_TYPE, - START_TREE_RESPONSE, - GET_TREE_INFO_REQUEST, - GET_TREE_INFO_RESPONSE, - INSERT_LEAVES_REQUEST, - INSERT_LEAVES_RESPONSE, -}; - -struct StartTreeRequest { - std::string name; - uint32_t depth; - uint32_t preFilledSize; - - MSGPACK_FIELDS(name, depth, preFilledSize); -}; - -struct StartTreeResponse { - std::string name; - uint32_t depth; - bool success; - std::string message; - - MSGPACK_FIELDS(name, depth, success, message); -}; - -struct GetTreeInfoRequest { - std::string name; - - MSGPACK_FIELDS(name); -}; - -struct GetTreeInfoResponse { - std::string name; - uint32_t depth; - bb::fr root; - uint64_t size; - bool success; - std::string message; - - MSGPACK_FIELDS(name, depth, root, size, success, message); -}; - -struct InsertLeavesRequest { - std::string name; - std::vector leaves; - - MSGPACK_FIELDS(name, leaves); -}; - -struct InsertLeavesResponse { - bb::fr root; - uint64_t size; - bool success; - std::string message; - - MSGPACK_FIELDS(root, size, success, message); -}; - -} // namespace bb::world_state - -MSGPACK_ADD_ENUM(bb::world_state::WorldStateMsgTypes) diff --git a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp b/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp deleted file mode 100644 index 533f973aedc..00000000000 --- a/barretenberg/cpp/src/barretenberg/world_state/service/world_state_service.hpp +++ /dev/null @@ -1,180 +0,0 @@ -#pragma once -#include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree//lmdb_store/lmdb_store.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/messaging/header.hpp" -#include "barretenberg/serialize/cbind.hpp" -#include "barretenberg/world_state/service/message.hpp" -#include "msgpack/v3/sbuffer_decl.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace bb::world_state { -using namespace bb::messaging; -using namespace bb::crypto::merkle_tree; -using HashPolicy = PedersenHashPolicy; - -struct TreeWithStore { - typedef CachedTreeStore Store; - std::unique_ptr lmdbStore; - std::unique_ptr> tree; - std::unique_ptr store; - - TreeWithStore(std::unique_ptr l, - std::unique_ptr> t, - std::unique_ptr s) - : lmdbStore(std::move(l)) - , tree(std::move(t)) - , store(std::move(s)) - {} - - TreeWithStore(TreeWithStore&& other) noexcept - : lmdbStore(std::move(other.lmdbStore)) - , tree(std::move(other.tree)) - , store(std::move(other.store)) - {} - - TreeWithStore& operator=(TreeWithStore&& other) = delete; - ~TreeWithStore() = default; - TreeWithStore(const TreeWithStore& other) = delete; - TreeWithStore& operator=(const TreeWithStore& other) = delete; -}; - -template class WorldStateService { - private: - OutputStream& outputStream; - std::unordered_map> trees; - ThreadPool workers; - LMDBEnvironment lmdbEnvironment; - - public: - WorldStateService(OutputStream& out, uint32_t numThreads) - : outputStream(out) - , workers(numThreads) - , lmdbEnvironment(randomTempDirectory(), 1024, 50, numThreads) - {} - bool startTree(msgpack::object& obj); - bool getTreeInfo(msgpack::object& obj); - bool insertLeaves(msgpack::object& obj); - - private: -}; - -template bool WorldStateService::startTree(msgpack::object& obj) -{ - TypedMessage startTreeRequest; - obj.convert(startTreeRequest); - - uint32_t actualDepth = startTreeRequest.value.depth; - - auto it = trees.find(startTreeRequest.value.name); - if (it == trees.end()) { - auto lmdbStore = - std::make_unique(lmdbEnvironment, startTreeRequest.value.name, false, false, IntegerKeyCmp); - auto store = std::make_unique( - startTreeRequest.value.name, startTreeRequest.value.depth, *lmdbStore); - - auto tree = std::make_unique>( - *store, workers, startTreeRequest.value.preFilledSize); - - auto treeWithStore = std::make_unique(std::move(lmdbStore), std::move(tree), std::move(store)); - trees[startTreeRequest.value.name] = std::move(treeWithStore); - - } else { - actualDepth = static_cast(it->second->tree->depth()); - } - - StartTreeResponse response; - response.depth = actualDepth; - response.name = startTreeRequest.value.name; - response.success = it == trees.end(); - response.message = response.success ? "" : "Tree already exists"; - - MsgHeader header(startTreeRequest.header.messageId); - TypedMessage startTreeResponse(WorldStateMsgTypes::START_TREE_RESPONSE, header, response); - outputStream.sendPackedObject(startTreeResponse); - return true; -} - -template bool WorldStateService::getTreeInfo(msgpack::object& obj) -{ - TypedMessage treeInfoRequest; - obj.convert(treeInfoRequest); - - GetTreeInfoResponse treeInfoResponse; - treeInfoResponse.name = treeInfoRequest.value.name; - - auto it = trees.find(treeInfoRequest.value.name); - if (it == trees.end()) { - treeInfoResponse.success = false; - treeInfoResponse.message = "Tree not found"; - } else { - treeInfoResponse.message = ""; - treeInfoResponse.success = true; - treeInfoResponse.depth = static_cast(it->second->tree->depth()); - // treeInfoResponse.root = it->second->tree->root(); - // treeInfoResponse.size = static_cast(it->second->tree->size()); - } - - MsgHeader header(treeInfoRequest.header.messageId); - TypedMessage getTreeInfoResponse( - WorldStateMsgTypes::GET_TREE_INFO_RESPONSE, header, treeInfoResponse); - outputStream.sendPackedObject(getTreeInfoResponse); - return true; -} - -template bool WorldStateService::insertLeaves(msgpack::object& obj) -{ - TypedMessage insertLeavesRequest; - obj.convert(insertLeavesRequest); - - uint32_t requestId = insertLeavesRequest.header.messageId; - - auto it = trees.find(insertLeavesRequest.value.name); - if (it == trees.end()) { - InsertLeavesResponse response; - response.success = false; - response.message = "Tree not found"; - - // Send the response immediately - MsgHeader header(requestId); - TypedMessage insertLeavesResponse( - WorldStateMsgTypes::INSERT_LEAVES_RESPONSE, header, response); - outputStream.sendPackedObject(insertLeavesResponse); - - } else { - - // Send the response async - auto completion = [=, this](const TypedResponse>& resp) -> void { - MsgHeader header(requestId); - InsertLeavesResponse response; - response.message = ""; - response.success = true; - response.root = resp.inner.add_data_result.root; - response.size = resp.inner.add_data_result.size; - TypedMessage insertLeavesResponse( - WorldStateMsgTypes::INSERT_LEAVES_RESPONSE, header, response); - outputStream.sendPackedObject(insertLeavesResponse); - }; - std::vector leaves(insertLeavesRequest.value.leaves.size()); - for (auto& v : insertLeavesRequest.value.leaves) { - leaves.emplace_back(v); - } - it->second->tree->add_or_update_values(leaves, completion); - } - - return true; -} -} // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt deleted file mode 100644 index 47037d3a303..00000000000 --- a/barretenberg/cpp/src/barretenberg/world_state_service/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -if(NOT(FUZZING)) - add_executable(world_state main.cpp) - target_link_libraries(world_state lmdb-lib crypto_merkle_tree crypto_pedersen_hash common) -endif() diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp b/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp deleted file mode 100644 index 2eeaffa83cd..00000000000 --- a/barretenberg/cpp/src/barretenberg/world_state_service/main.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "barretenberg/messaging/header.hpp" -#include "barretenberg/messaging/stream_parser.hpp" -#include "barretenberg/world_state/service/message.hpp" -#include "barretenberg/world_state/service/world_state_service.hpp" -#include "std_io.hpp" -#include -#include -#include - -using namespace bb::world_state; - -template void waitForStream(std::basic_istream& inputStream, StreamDispatcher& dispatcher) -{ - msgpack::unpacker unp; - bool moreDataExpected = true; - while (moreDataExpected) { - unp.reserve_buffer(1); - inputStream.read(unp.buffer(), 1); - unp.buffer_consumed(1); - - msgpack::object_handle result; - while (moreDataExpected && unp.next(result)) { - msgpack::object obj(result.get()); - moreDataExpected = dispatcher.onNewData(obj); - } - } -} - -int main(int, char**) -{ - SynchronisedStdOutput outputStream(std::cout); - WorldStateService service(outputStream, 16); - StreamDispatcher dispatcher(outputStream); - - std::function f1 = [&service](msgpack::object& obj) { return service.startTree(obj); }; - dispatcher.registerTarget(WorldStateMsgTypes::START_TREE_REQUEST, f1); - - std::function f2 = [&service](msgpack::object& obj) { return service.getTreeInfo(obj); }; - dispatcher.registerTarget(WorldStateMsgTypes::GET_TREE_INFO_REQUEST, f2); - - std::function f3 = [&service](msgpack::object& obj) { return service.insertLeaves(obj); }; - dispatcher.registerTarget(WorldStateMsgTypes::INSERT_LEAVES_REQUEST, f3); - - waitForStream(std::cin, dispatcher); -} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp b/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp deleted file mode 100644 index 239e0c6f824..00000000000 --- a/barretenberg/cpp/src/barretenberg/world_state_service/std_io.hpp +++ /dev/null @@ -1,61 +0,0 @@ -#pragma once - -#include "barretenberg/messaging/header.hpp" -#include "barretenberg/messaging/stream_parser.hpp" -#include "barretenberg/serialize/cbind.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/world_state/service/message.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -// struct MsgPackedHeader { -// uint32_t msgType; -// msgpack::object header; - -// MSGPACK_FIELDS(msgType, header); -// }; - -// struct MsgPackedType { -// uint32_t msgType; -// msgpack::object header; -// msgpack::object value; - -// MSGPACK_FIELDS(msgType, header, value); -// }; - -class SynchronisedStdOutput { - private: - std::basic_ostream& stream; - std::mutex mutex; - uint32_t msgId = 1; - - public: - SynchronisedStdOutput(std::basic_ostream& str) - : stream(str) - {} - void send(bb::messaging::HeaderOnlyMessage& header) - { - std::unique_lock lock(mutex); - header.header.messageId = msgId++; - msgpack::sbuffer buffer; - msgpack::pack(buffer, header); - stream.write(buffer.data(), static_cast(buffer.size())); - stream.flush(); - } - template void sendPackedObject(bb::messaging::TypedMessage& header) - { - std::unique_lock lock(mutex); - header.header.messageId = msgId++; - msgpack::sbuffer buffer; - msgpack::pack(buffer, header); - stream.write(buffer.data(), static_cast(buffer.size())); - stream.flush(); - } -}; From abc9621d48e63317c38dff253d50e9e4ad85a9ef Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 18 Jul 2024 17:13:20 +0100 Subject: [PATCH 49/63] Comments --- .../append_only_tree.bench.cpp | 63 +++--------------- .../append_only_tree/append_only_tree.hpp | 65 ++++++++++++++----- .../merkle_tree/indexed_tree/indexed_tree.hpp | 23 +++++-- 3 files changed, 75 insertions(+), 76 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp index 811eeecaea0..869702792b2 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp @@ -17,13 +17,14 @@ using namespace benchmark; using namespace bb::crypto::merkle_tree; +namespace { using StoreType = CachedTreeStore; using Pedersen = AppendOnlyTree; using Poseidon2 = AppendOnlyTree; const size_t TREE_DEPTH = 32; -// const size_t MAX_BATCH_SIZE = 128; +const size_t MAX_BATCH_SIZE = 128; template void perform_batch_insert(TreeType& tree, const std::vector& values) { @@ -70,63 +71,17 @@ template void append_only_tree_bench(State& state) noexcept std::filesystem::remove_all(directory); } - -template void hash_subtree(std::vector& values, uint32_t offset, uint32_t size) noexcept -{ - uint32_t current = offset; - while (current < offset + size) { - HashPolicy::hash_pair(values[current], values[current + 1]); - current += 2; - } -} - -template void hash_values(State& state) noexcept -{ - - const size_t batch_size = size_t(1024); - size_t num_threads = static_cast(state.range(0)); - uint32_t per_thread = batch_size / num_threads; - for (auto _ : state) { - state.PauseTiming(); - std::vector values(batch_size); - for (size_t i = 0; i < batch_size; ++i) { - values[i] = fr(random_engine.get_random_uint256()); - } - ThreadPool pool(num_threads); - state.ResumeTiming(); - if (num_threads > 1) { - for (uint32_t i = 0; i < num_threads; i++) { - uint32_t current = 0; - pool.enqueue([&]() { hash_subtree(values, current, per_thread); }); - current += per_thread; - } - pool.wait(); - } else { - - hash_subtree(values, 0, batch_size); - } - } -} -// BENCHMARK(append_only_tree_bench) -// ->Unit(benchmark::kMillisecond) -// ->RangeMultiplier(2) -// ->Range(2, MAX_BATCH_SIZE) -// ->Iterations(100); -// BENCHMARK(append_only_tree_bench) -// ->Unit(benchmark::kMillisecond) -// ->RangeMultiplier(2) -// ->Range(2, MAX_BATCH_SIZE) -// ->Iterations(1000); - -BENCHMARK(hash_values) +BENCHMARK(append_only_tree_bench) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) - ->Range(1, 8) + ->Range(2, MAX_BATCH_SIZE) ->Iterations(100); -BENCHMARK(hash_values) +BENCHMARK(append_only_tree_bench) ->Unit(benchmark::kMillisecond) ->RangeMultiplier(2) - ->Range(1, 8) - ->Iterations(100); + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); + +} // namespace BENCHMARK_MAIN(); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 5bf5af7eefe..c9baa019ea3 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -19,12 +19,17 @@ using namespace bb; /** * @brief Implements a simple append-only merkle tree - * Accepts template argument of the type of store backing the tree and the hashing policy + * All methods are asynchronous unless specified as otherwise + * Accepts template arguments of the type of store backing the tree and the hashing policy + * Accepts the store as an argument on construction as well as a thread pool instance + * Asynchronous methods are exeucted on the provided thread pool * */ template class AppendOnlyTree { public: using StoreType = Store; + + // Asynchronous methods accept these callback function types as arguments using AppendCompletionCallback = std::function&)>; using MetaDataCallback = std::function&)>; using HashPathCallback = std::function&)>; @@ -33,6 +38,7 @@ template class AppendOnlyTree { using CommitCallback = std::function; using RollbackCallback = std::function; + // Only construct from provided store and thread pool, no copies or moves AppendOnlyTree(Store& store, ThreadPool& workers); AppendOnlyTree(AppendOnlyTree const& other) = delete; AppendOnlyTree(AppendOnlyTree&& other) = delete; @@ -55,20 +61,42 @@ template class AppendOnlyTree { */ void get_sibling_path(const index_t& index, const HashPathCallback& on_completion, bool includeUncommitted) const; + /** + * @brief Returns the tree meta data such as tree root and size + */ void get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion) const; + /** + * @brief Returns the leaf value at the provided index + */ void get_leaf(const index_t& index, bool includeUncommitted, const GetLeafCallback& completion) const; + /** + * @brief Returns the index of the provided leaf in the tree + */ void find_leaf_index(const fr& leaf, bool includeUncommitted, const FindLeafCallback& on_completion) const; + /** + * @brief Returns the index of the provided leaf in the tree only if it exists after the index value provided + */ void find_leaf_index_from(const fr& leaf, index_t start_index, bool includeUncommitted, const FindLeafCallback& on_completion) const; + /** + * @brief Commit the tree to the backing store + */ void commit(const CommitCallback& on_completion); + + /** + * @brief Rollback the uncommitted changes + */ void rollback(const RollbackCallback& on_completion); + /** + * @brief Synchronous method to retrieve the depth of the tree + */ uint32_t depth() const { return depth_; } protected: @@ -110,6 +138,7 @@ AppendOnlyTree::AppendOnlyTree(Store& store, ThreadPool& w index_t stored_size = 0; bb::fr stored_root = fr::zero(); { + // start by reading the meta data from the backing store ReadTransactionPtr tx = store_.createReadTransaction(); store_.get_full_meta(stored_size, stored_root, name_, depth_, *tx, false); } @@ -124,6 +153,7 @@ AppendOnlyTree::AppendOnlyTree(Store& store, ThreadPool& w zero_hashes_[0] = current; if (stored_size == 0) { + // if the tree is empty then we want to write the initial root store_.put_meta(0, current); store_.commit(); } @@ -194,7 +224,7 @@ void AppendOnlyTree::find_leaf_index(const fr& leaf, bool includeUncommitted, const FindLeafCallback& on_completion) const { - return find_leaf_index_from(leaf, 0, includeUncommitted, on_completion); + find_leaf_index_from(leaf, 0, includeUncommitted, on_completion); } template @@ -254,6 +284,20 @@ void AppendOnlyTree::add_values_internal(const std::vector workers_.enqueue(append_op); } +template +void AppendOnlyTree::commit(const CommitCallback& on_completion) +{ + auto job = [=, this]() { ExecuteAndReport([=, this]() { store_.commit(); }, on_completion); }; + workers_.enqueue(job); +} + +template +void AppendOnlyTree::rollback(const RollbackCallback& on_completion) +{ + auto job = [=, this]() { ExecuteAndReport([=, this]() { store_.rollback(); }, on_completion); }; + workers_.enqueue(job); +} + template void AppendOnlyTree::add_values_internal(std::shared_ptr> values, fr& new_root, @@ -282,6 +326,8 @@ void AppendOnlyTree::add_values_internal(std::shared_ptr::add_values_internal(std::shared_ptr fr AppendOnlyTree::get_element_or_zero(uint32_t level, const index_t& index, @@ -367,18 +414,4 @@ std::pair AppendOnlyTree::read_node(uint32_t lev return std::make_pair(true, value); } -template -void AppendOnlyTree::commit(const CommitCallback& on_completion) -{ - auto job = [=, this]() { ExecuteAndReport([=, this]() { store_.commit(); }, on_completion); }; - workers_.enqueue(job); -} - -template -void AppendOnlyTree::rollback(const RollbackCallback& on_completion) -{ - auto job = [=, this]() { ExecuteAndReport([=, this]() { store_.rollback(); }, on_completion); }; - workers_.enqueue(job); -} - } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index b9df9d5d53f..f26aa77df54 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -26,11 +26,13 @@ namespace bb::crypto::merkle_tree { * @brief Implements a parallelised batch insertion indexed tree * Accepts template argument of the type of store backing the tree, the type of store containing the leaves and the * hashing policy - * + * All public methods are asynchronous unless marked otherwise */ template class IndexedTree : public AppendOnlyTree { public: using StoreType = Store; + + // The public methods accept these function types as asynchronous callbacks using LeafValueType = typename Store::LeafType; using IndexedLeafValueType = typename Store::IndexedLeafValueType; using AddCompletionCallback = std::function>&)>; @@ -45,30 +47,37 @@ template class IndexedTree : public App /** * @brief Adds or updates a single values in the tree (updates not currently supported) - * @param value The value to be added or updated - * @returns The 'previous' hash paths of all updated values */ void add_or_update_value(const LeafValueType& value, const AddCompletionCallback& completion); /** * @brief Adds or updates the given set of values in the tree (updates not currently supported) - * @param values The values to be added or updated - * @param no_multithreading Performs single threaded insertion, just used whilst prototyping and benchmarking - * @returns The 'previous' hash paths of all updated values */ void add_or_update_values(const std::vector& values, const AddCompletionCallback& completion); + /** + * @brief Returns the leaf at the provided index + */ void get_leaf(const index_t& index, bool includeUncommitted, const LeafCallback& completion) const; + /** + * @brief Find the index of the provided leaf value if it exists + */ void find_leaf_index(const LeafValueType& leaf, bool includeUncommitted, const AppendOnlyTree::FindLeafCallback& on_completion) const; + /** + * @brief Find the index of the provided leaf value if it exists, only considers indexed beyond the value provided + */ void find_leaf_index_from(const LeafValueType& leaf, index_t start_index, bool includeUncommitted, const AppendOnlyTree::FindLeafCallback& on_completion) const; + /** + * @brief Find the value of the leaf that is immediately lower in value than that provided + */ void find_low_leaf(const LeafValueType& leaf, bool includeUncommitted, const LeafCallback& on_completion) const; using AppendOnlyTree::get_sibling_path; @@ -187,6 +196,7 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers store_.set_at_index(i, initial_leaf, true); } + // Append the leaves and wait for the operation to complete bool success = true; Signal signal(1); AppendCompletionCallback completion = [&](const TypedResponse& result) -> void { @@ -198,6 +208,7 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers if (!success) { throw std::runtime_error("Failed to initialise tree"); } + // Commit the state store_.commit(); } From 0d7aade141479c84b9f81d5da70a1f180a925818 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 19 Jul 2024 12:06:53 +0100 Subject: [PATCH 50/63] Review cleanup --- .../append_only_tree/append_only_tree.hpp | 15 ++++ .../merkle_tree/indexed_tree/indexed_tree.hpp | 17 ++-- .../merkle_tree/lmdb_store/functions.cpp | 25 ++++++ .../merkle_tree/lmdb_store/functions.hpp | 4 + .../merkle_tree/lmdb_store/lmdb_database.hpp | 36 +-------- .../lmdb_store/lmdb_db_transaction.hpp | 3 + .../lmdb_store/lmdb_environment.hpp | 13 ++++ .../lmdb_store/lmdb_read_transaction.cpp | 5 +- .../lmdb_store/lmdb_read_transaction.hpp | 9 ++- .../merkle_tree/lmdb_store/lmdb_store.hpp | 5 ++ .../lmdb_store/lmdb_store.test.cpp | 16 ++-- .../lmdb_store/lmdb_transaction.cpp | 2 + .../lmdb_store/lmdb_transaction.hpp | 12 ++- .../lmdb_store/lmdb_write_transaction.cpp | 4 +- .../lmdb_store/lmdb_write_transaction.hpp | 11 ++- .../node_store/cached_tree_store.hpp | 77 ++++++++++++++++--- 16 files changed, 184 insertions(+), 70 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 0914f516a26..7efbf9f7148 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -48,16 +48,23 @@ template class AppendOnlyTree { /** * @brief Adds a single value to the end of the tree + * @param value The value to be added + * @param on_completion Callback to be called on completion */ virtual void add_value(const fr& value, const AppendCompletionCallback& on_completion); /** * @brief Adds the given set of values to the end of the tree + * @param values The values to be added + * @param on_completion Callback to be called on completion */ virtual void add_values(const std::vector& values, const AppendCompletionCallback& on_completion); /** * @brief Returns the sibling path from the leaf at the given index to the root + * @param index The index at which to read the sibling path + * @param on_completion Callback to be called on completion + * @param includeUncommitted Whether to include uncommitted changes */ void get_sibling_path(const index_t& index, const HashPathCallback& on_completion, bool includeUncommitted) const; @@ -85,10 +92,18 @@ template class AppendOnlyTree { const HashPathCallback& on_completion, bool includeUncommitted) const; + /** + * @brief Returns the tree meta data + * @param includeUncommitted Whether to include uncommitted changes + * @param on_completion Callback to be called on completion + */ void get_meta_data(bool includeUncommitted, const MetaDataCallback& on_completion) const; /** * @brief Returns the leaf value at the provided index + * @param index The index of the leaf to be retrieved + * @param includeUncommitted Whether to include uncommitted changes + * @param on_completion Callback to be called on completion */ void get_leaf(const index_t& index, bool includeUncommitted, const GetLeafCallback& completion) const; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 25d8ed0bf15..36c346ef15f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -86,6 +86,9 @@ template class IndexedTree : public App bool includeUncommitted, const AppendOnlyTree::FindLeafCallback& on_completion) const; + /** + * @brief Find the leaf with the value immediately lower then the value provided + */ void find_low_leaf(const fr& leaf_key, bool includeUncommitted, const FindLowLeafCallback& on_completion) const; using AppendOnlyTree::get_sibling_path; @@ -565,15 +568,13 @@ void IndexedTree::generate_insertions( // .value() throws if the low leaf does not exist IndexedLeafValueType current_leaf = store_.get_leaf(current, *tx, true).value(); - response.inner.insertions->push_back({ + LeafInsertion insertion = { .low_leaf_index = current, .low_leaf = IndexedLeafValueType::empty(), .original_low_leaf = current_leaf, - }); - LeafInsertion& insertion = (*response.inner.insertions)[i]; + }; + // Capture the index and original value of the 'low' leaf - // insertion.low_leaf_index = current; - // insertion.original_low_leaf = current_leaf; if (!is_already_present) { // Update the current leaf to point it to the new leaf @@ -604,6 +605,8 @@ void IndexedTree::generate_insertions( // capture new low leaf insertion.low_leaf = IndexedLeafValueType(current_leaf.value, current_leaf.nextIndex, current_leaf.nextValue); + + response.inner.insertions->push_back(insertion); } } }, @@ -635,9 +638,6 @@ void IndexedTree::update_leaf_and_hash_to_root(const index // Extract the value of the leaf node and it's sibling bool is_right = static_cast(index & 0x01); - // extract the current leaf hash values for the previous hash path - // fr sibling = get_node(level, is_right ? index - 1 : index + 1); - // previous_sibling_path.emplace_back(sibling); // Write the new leaf hash in place write_node(level, index, new_hash); @@ -655,7 +655,6 @@ void IndexedTree::update_leaf_and_hash_to_root(const index index_t index_of_node_above = index >> 1; bool node_above_is_right = static_cast(index_of_node_above & 0x01); fr above_sibling = get_node(level, node_above_is_right ? index_of_node_above - 1 : index_of_node_above + 1); - // previous_sibling_path.emplace_back(above_sibling); } // Now that we have extracted the hash path from the row above diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp index 5f586ea67d8..58f02d8b7d5 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp @@ -12,6 +12,14 @@ void ThrowError(const std::string& errorString, int error) throw std::runtime_error(ss.str()); } +// Nodes are stored as a heap +NodeKeyType GetKeyForNode(uint32_t level, index_t index) +{ + NodeKeyType key = static_cast(1) << level; + key += static_cast(index); + return key - 1; +} + int SizeCmp(const MDB_val* a, const MDB_val* b) { if (a->mv_size < b->mv_size) { @@ -23,26 +31,43 @@ int SizeCmp(const MDB_val* a, const MDB_val* b) return 0; } +/** + * Default lexicographical implementation of key comparisons used in our LMDB implementation + */ int MemCmp(const MDB_val* a, const MDB_val* b) { return std::memcmp(static_cast(a->mv_data), static_cast(b->mv_data), a->mv_size); } +/** + * Integer key comparison function. + * We use this to divide the key space into discrete integer sizes + * We check the key length in bytes to establish if it is exactly + * 1. 1 byte + * 2. 8 bytes + * 3. 16 bytes + * 4. 32 bytes + * If it is one of the above sizes then we assume it is an integer value and we compare it as such + */ int IntegerKeyCmp(const MDB_val* a, const MDB_val* b) { + // Id the keys sizes are different, sort by key size if (a->mv_size != b->mv_size) { return SizeCmp(a, b); } uint64_t factor = a->mv_size / sizeof(uint64_t); uint64_t remainder = a->mv_size % sizeof(uint64_t); + // If the size is > 32 bytes, use default comparison if (a->mv_size > sizeof(uint256_t)) { return MemCmp(a, b); } + // If the size is not a divisible by 8 then use default comparison, unless it is 1 byte if (a->mv_size > 1 && remainder != 0) { return MemCmp(a, b); } + // Size is either 1, 8, 16 or 32 bytes, compare based on integer keys using f = std::function; static std::vector functions{ ValueCmp, ValueCmp, ValueCmp, MemCmp, ValueCmp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp index eb8f6fb4a8c..f8d4089efc4 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp @@ -1,7 +1,9 @@ #pragma once +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" +#include #include namespace bb::crypto::merkle_tree { @@ -16,6 +18,8 @@ int SizeCmp(const MDB_val* a, const MDB_val* b); int MemCmp(const MDB_val*, const MDB_val*); +NodeKeyType GetKeyForNode(uint32_t level, index_t index); + template int ValueCmp(const MDB_val* a, const MDB_val* b) { const T* lhs = static_cast(a->mv_data); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp index 732476183ec..48c63c4838a 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp @@ -4,6 +4,10 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" namespace bb::crypto::merkle_tree { +/** + * RAII wrapper atound the opening and closing of an LMDB database + * Contains a reference to its LMDB environment + */ class LMDBDatabase { public: LMDBDatabase(const LMDBEnvironment& env, @@ -26,36 +30,4 @@ class LMDBDatabase { MDB_dbi _dbi; const LMDBEnvironment& _environment; }; - -// LMDBDatabase::LMDBDatabase(const LMDBEnvironment& env, -// const LMDBDatabaseCreationTransaction& transaction, -// const std::string& name, -// bool integerKeys, -// bool reverseKeys, -// MDB_cmp_func* cmp) -// : _environment(env) -// { -// unsigned int flags = MDB_CREATE; -// if (integerKeys) { -// flags |= MDB_INTEGERKEY; -// } -// if (reverseKeys) { -// flags |= MDB_REVERSEKEY; -// } -// call_lmdb_func("mdb_dbi_open", mdb_dbi_open, transaction.underlying(), name.c_str(), flags, &_dbi); -// if (cmp != nullptr) { -// call_lmdb_func("mdb_set_compare", mdb_set_compare, transaction.underlying(), _dbi, cmp); -// } -// transaction.commit(); -// } - -// LMDBDatabase::~LMDBDatabase() -// { -// call_lmdb_func(mdb_dbi_close, _environment.underlying(), _dbi); -// } - -// const MDB_dbi& LMDBDatabase::underlying() const -// { -// return _dbi; -// } } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp index 48645059592..af5cf528ac2 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp @@ -3,6 +3,9 @@ namespace bb::crypto::merkle_tree { +/* + * RAII wrapper to construct a transaction for the purpose of creating/opening a database + */ class LMDBDatabaseCreationTransaction : public LMDBTransaction { public: using Ptr = std::unique_ptr; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp index f4c1f826fe9..29262360304 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp @@ -5,8 +5,21 @@ #include #include namespace bb::crypto::merkle_tree { +/* + * RAII wrapper around an LMDB environment. + * Opens/creates the environemnt and manages read access to the enviroment. + * The environment has an upper limit on the number of concurrent read transactions + * and this is managed through the use of mutex/condition variables + */ class LMDBEnvironment { public: + /** + * @brief Opens/creates the LMDB environment + * @param directory The directory in which the environment is to be created + * @param mapSizeKb The maximum size of the database, can be increased from a previously used value + * @param maxNumDbs The maximum number of databases that can be created withn this environment + * @param maxNumReaders The maximum number of concurrent read transactions permitted. + */ LMDBEnvironment(const std::string& directory, uint64_t mapSizeKb, uint32_t maxNumDBs, uint32_t maxNumReaders); LMDBEnvironment(const LMDBEnvironment& other) = delete; LMDBEnvironment(LMDBEnvironment&& other) = delete; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp index 63f3bd6bcd8..a5b9320b3bd 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp @@ -1,4 +1,5 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" namespace bb::crypto::merkle_tree { LMDBReadTransaction::LMDBReadTransaction(LMDBEnvironment& env, const LMDBDatabase& database) @@ -34,7 +35,7 @@ bool LMDBReadTransaction::get_value(std::vector& key, std::vector& data) const { - NodeKeyType key = ((static_cast(1) << level) + static_cast(index)) - 1; - return get_value_by_integer(key, data); + NodeKeyType key = GetKeyForNode(level, index); + return get_value(key, data); } } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp index ec68f48fe06..8f96a8cb126 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp @@ -8,6 +8,11 @@ namespace bb::crypto::merkle_tree { +/** + * RAII wrapper around a read transaction. + * Contains various methods for retrieving values by their keys. + * Aborts the transaction upon object destruction. + */ class LMDBReadTransaction : public LMDBTransaction { public: using Ptr = std::unique_ptr; @@ -24,7 +29,7 @@ class LMDBReadTransaction : public LMDBTransaction { bool get_node(uint32_t level, index_t index, std::vector& data) const; - template bool get_value_by_integer(T& key, std::vector& data) const; + template bool get_value(T& key, std::vector& data) const; bool get_value(std::vector& key, std::vector& data) const; @@ -34,7 +39,7 @@ class LMDBReadTransaction : public LMDBTransaction { const LMDBDatabase& _database; }; -template bool LMDBReadTransaction::get_value_by_integer(T& key, std::vector& data) const +template bool LMDBReadTransaction::get_value(T& key, std::vector& data) const { MDB_val dbKey; dbKey.mv_size = sizeof(T); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp index e5f3490cf92..b2073cbc371 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp @@ -5,6 +5,11 @@ namespace bb::crypto::merkle_tree { +/** + * Creates an named LMDB 'Store' abstraction on top of an environment. + * Provides methods for creating read and write transactions against the store. + */ + class LMDBStore { public: diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp index 5c8a492f48b..d24fddefea9 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp @@ -238,7 +238,7 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) for (size_t i = 0; i < num_keys; i++) { std::vector value; write(value, values[i]); - transaction->put_value_by_integer(keys[i], value); + transaction->put_value(keys[i], value); } // Now put keys in the db that are smaller and larger in length and smaller and larger in value @@ -250,10 +250,10 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) uint32_t value_to_write = 6; std::vector value; write(value, value_to_write); - transaction->put_value_by_integer(lower64, value); - transaction->put_value_by_integer(higher64, value); - transaction->put_value_by_integer(lower256, value); - transaction->put_value_by_integer(higher256, value); + transaction->put_value(lower64, value); + transaction->put_value(higher64, value); + transaction->put_value(lower256, value); + transaction->put_value(higher256, value); transaction->commit(); } @@ -317,7 +317,7 @@ TEST_F(LMDBStoreTest, can_not_retrieve_previous_key_from_empty_db) LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); uint128_t key = 20; std::vector data; - bool success = transaction->get_value_by_integer(key, data); + bool success = transaction->get_value(key, data); EXPECT_EQ(success, false); } @@ -467,7 +467,7 @@ TEST_F(LMDBStoreTest, can_handle_different_key_spaces) std::vector data; write(data, values[values_index][j]); auto key = static_cast(keys[j]); - transaction.put_value_by_integer(key, data); + transaction.put_value(key, data); } }; @@ -475,7 +475,7 @@ TEST_F(LMDBStoreTest, can_handle_different_key_spaces) for (uint32_t j = 0; j < keys.size(); j++) { std::vector data; auto key = static_cast(keys[j]); - transaction.get_value_by_integer(key, data); + transaction.get_value(key, data); auto retrieved_value = from_buffer(data); ASSERT_EQ(retrieved_value, values[values_index][j]); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp index dc29f8baaa8..b925a216d7b 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp @@ -11,6 +11,8 @@ LMDBTransaction::LMDBTransaction(LMDBEnvironment& env, bool readOnly) "mdb_txn_begin", mdb_txn_begin, _environment.underlying(), p, readOnly ? MDB_RDONLY : 0U, &_transaction); } +LMDBTransaction::~LMDBTransaction() = default; + MDB_txn* LMDBTransaction::underlying() const { return _transaction; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp index b8b0ff2bb19..0a160fedd5a 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp @@ -3,6 +3,11 @@ namespace bb::crypto::merkle_tree { +/* + * Abstract base class to represent and LMDB transaction. + * Needs to be sub-classed to be either a read or write transaction. + */ + enum TransactionState { OPEN, COMMITTED, @@ -17,10 +22,15 @@ class LMDBTransaction { LMDBTransaction& operator=(const LMDBTransaction& other) = delete; LMDBTransaction& operator=(LMDBTransaction&& other) = delete; - virtual ~LMDBTransaction() = default; + virtual ~LMDBTransaction() = 0; MDB_txn* underlying() const; + /* + * Rolls back the transaction. + * Must be called by read transactions to signal the end of the transaction. + * Must be called by write transactions if the changes are not to be committed. + */ virtual void abort(); protected: diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp index aae18f49eaa..c64bdfb6fe5 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp @@ -33,8 +33,8 @@ void LMDBWriteTransaction::try_abort() void LMDBWriteTransaction::put_node(uint32_t level, index_t index, std::vector& data) { - NodeKeyType key = ((static_cast(1) << level) + static_cast(index)) - 1; - put_value_by_integer(key, data); + NodeKeyType key = GetKeyForNode(level, index); + put_value(key, data); } void LMDBWriteTransaction::put_value(std::vector& key, std::vector& data) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp index c1fe99ab4b8..9cb7c10734e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp @@ -5,6 +5,13 @@ namespace bb::crypto::merkle_tree { +/** + * RAII wrapper for an LMDB write transaction. + * Provides methods for writing values by their key. + * Must be either committed to persist the changes or aborted to roll them back. + * Will automatically abort the transaction during destruction if changes have not been committed. + */ + class LMDBWriteTransaction : public LMDBTransaction { public: using Ptr = std::unique_ptr; @@ -18,7 +25,7 @@ class LMDBWriteTransaction : public LMDBTransaction { void put_node(uint32_t level, index_t index, std::vector& data); - template void put_value_by_integer(T& key, std::vector& data); + template void put_value(T& key, std::vector& data); void put_value(std::vector& key, std::vector& data); @@ -30,7 +37,7 @@ class LMDBWriteTransaction : public LMDBTransaction { const LMDBDatabase& _database; }; -template void LMDBWriteTransaction::put_value_by_integer(T& key, std::vector& data) +template void LMDBWriteTransaction::put_value(T& key, std::vector& data) { MDB_val dbKey; dbKey.mv_size = sizeof(T); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp index 7066db2a31c..7af89dbecbd 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp @@ -17,7 +17,13 @@ namespace bb::crypto::merkle_tree { /** - * @brief Serves as a key-value node store for merkle trees, uses an unordered_map as a cache + * @brief Serves as a key-value node store for merkle trees. Caches all changes in memory before persisting them during + * a 'commit' operation. + * Manages the persisted store by seperating the key spaces as follows: + * 1 byte key of 0: Tree meta data + * 8 byte integers: The index of each leaf to the value of that leaf + * 16 byte integers: Nodes in the tree, key value = ((2 ^ level) + index - 1) + * 32 bytes integers: The value of the leaf (32 bytes) to the set of indices where the leaf exists in the tree. */ template class CachedTreeStore { public: @@ -46,28 +52,55 @@ template class CachedTreeStore CachedTreeStore& operator=(CachedTreeStore const& other) = delete; CachedTreeStore& operator=(CachedTreeStore const&& other) = delete; + /** + * @brief Returns the index of the leaf with a value immediately lower than the value provided + */ std::pair find_low_value(const fr& new_leaf_key, bool includeUncommitted, ReadTransaction& tx) const; + /** + * @brief Returns the leaf at the provided index, if one exists + */ std::optional get_leaf(const index_t& index, ReadTransaction& tx, bool includeUncommitted) const; + /** + * @brief Adds the leaf at the given index, updates the leaf index if requested + */ void set_at_index(const index_t& index, const IndexedLeafValueType& leaf, bool add_to_index); + /** + * @brief Updates the leaf index + */ void update_index(const index_t& index, const fr& leaf); + /** + * @brief Writes the provided data at the given node coordinates. Only writes to uncommitted data. + */ void put_node(uint32_t level, index_t index, const std::vector& data); + /** + * @brief Returns the data at the given node coordinates if available. Reads from uncommitted state if requested. + */ bool get_node(uint32_t level, index_t index, std::vector& data, ReadTransaction& transaction, bool includeUncommitted) const; + /** + * @brief Writes the provided meta data to uncommitted state + */ void put_meta(const index_t& size, const bb::fr& root); + /** + * @brief Reads the tree meta data, including uncommitted data if requested + */ void get_meta(index_t& size, bb::fr& root, ReadTransaction& tx, bool includeUncommitted) const; + /** + * @brief Reads the extended tree meta data, including uncommitted data if requested + */ void get_full_meta(index_t& size, bb::fr& root, std::string& name, @@ -75,31 +108,45 @@ template class CachedTreeStore ReadTransaction& tx, bool includeUncommitted) const; + /** + * @brief Finds the index of the given leaf value in the tree if available. Includes uncommitted data if requested. + */ std::optional find_leaf_index(const LeafValueType& leaf, ReadTransaction& tx, bool includeUncommitted) const; + /** + * @brief Finds the index of the given leaf value in the tree if available. Includes uncommitted data if requested. + */ std::optional find_leaf_index_from(const LeafValueType& leaf, index_t start_index, ReadTransaction& tx, bool includeUncommitted) const; + /** + * @brief Commits the uncommitted data to the underlying store + */ void commit(); + /** + * @brief Rolls back the uncommitted state + */ void rollback(); + /** + * @brief Returns the name of the tree + */ std::string get_name() const { return name; } + /** + * @brief Returns a read transaction against the underlying store. + */ ReadTransactionPtr createReadTransaction() const { return dataStore.createReadTransaction(); } private: struct Indices { std::vector indices; - // Indices(index_t index) - // : indices{ index } - // {} - MSGPACK_FIELDS(indices); }; @@ -178,7 +225,7 @@ std::optional::IndexedLe } LeafIndexKeyType key = index; std::vector data; - bool success = tx.get_value_by_integer(key, data); + bool success = tx.get_value(key, data); if (success) { IndexedLeafValueType return_value; msgpack::unpack((const char*)data.data(), data.size()).get().convert(return_value); @@ -234,7 +281,7 @@ std::optional CachedTreeStore::find_leaf std::optional result = std::nullopt; FrKeyType key = leaf; std::vector value; - bool success = tx.get_value_by_integer(key, value); + bool success = tx.get_value(key, value); if (success) { msgpack::unpack((const char*)value.data(), value.size()).get().convert(committed); if (!committed.indices.empty()) { @@ -347,7 +394,7 @@ template void CachedTreeStore< for (auto& idx : indices_) { std::vector value; FrKeyType key = idx.first; - bool success = tx->get_value_by_integer(key, value); + bool success = tx->get_value(key, value); if (success) { Indices indices; msgpack::unpack((const char*)value.data(), value.size()).get().convert(indices); @@ -371,14 +418,14 @@ template void CachedTreeStore< msgpack::pack(buffer, idx.second); std::vector encoded(buffer.data(), buffer.data() + buffer.size()); FrKeyType key = idx.first; - tx->put_value_by_integer(key, encoded); + tx->put_value(key, encoded); } for (const auto& leaf : leaves_) { msgpack::sbuffer buffer; msgpack::pack(buffer, leaf.second); std::vector value(buffer.data(), buffer.data() + buffer.size()); LeafIndexKeyType key = leaf.first; - tx->put_value_by_integer(key, value); + tx->put_value(key, value); } persistMeta(meta, *tx); tx->commit(); @@ -393,12 +440,15 @@ template void CachedTreeStore< template void CachedTreeStore::rollback() { + // Extract the committed meta data and destroy the cache + { + ReadTransactionPtr tx = createReadTransaction(); + readPersistedMeta(meta, *tx); + } nodes = std::vector>>( depth + 1, std::unordered_map>()); indices_ = std::map(); leaves_ = std::unordered_map(); - ReadTransactionPtr tx = createReadTransaction(); - readPersistedMeta(meta, *tx); } template @@ -424,6 +474,8 @@ void CachedTreeStore::persistMeta(TreeMeta& m, Wr template void CachedTreeStore::initialise() { + // Read the persisted meta data, if the name or depth of the tree is not consistent with what was provided during + // construction then we throw std::vector data; { ReadTransactionPtr tx = createReadTransaction(); @@ -436,6 +488,7 @@ void CachedTreeStore::initialise() } } + // No meta data available. Write the initial state down meta.name = name; meta.size = 0; meta.depth = depth; From db1858b18cfa935beb643425f556868bc5e0ef5a Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 19 Jul 2024 14:31:08 +0100 Subject: [PATCH 51/63] Build fix --- .../crypto/merkle_tree/indexed_tree/indexed_leaf.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp index e8b14be3a74..796abefc1a1 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp @@ -134,9 +134,9 @@ template struct IndexedLeaf { , nextValue(nextVal) {} - IndexedLeaf(const IndexedLeaf& other) = default; - IndexedLeaf(IndexedLeaf&& other) noexcept = default; - ~IndexedLeaf() = default; + IndexedLeaf(const IndexedLeaf& other) = default; + IndexedLeaf(IndexedLeaf&& other) noexcept = default; + ~IndexedLeaf() = default; static bool is_updateable() { return LeafType::is_updateable(); } From d1d12524d6fe101df19ac939e452ae49d15ba5ff Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 19 Jul 2024 17:54:59 +0100 Subject: [PATCH 52/63] Explicitly serialize keys --- .../merkle_tree/lmdb_store/functions.cpp | 54 ++++++++++++++++++- .../merkle_tree/lmdb_store/functions.hpp | 21 ++++++-- .../lmdb_store/lmdb_read_transaction.hpp | 31 +++++------ .../lmdb_store/lmdb_store.test.cpp | 2 - .../lmdb_store/lmdb_write_transaction.hpp | 12 ++--- 5 files changed, 86 insertions(+), 34 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp index 58f02d8b7d5..78d24337d26 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace bb::crypto::merkle_tree { void ThrowError(const std::string& errorString, int error) @@ -12,6 +13,57 @@ void ThrowError(const std::string& errorString, int error) throw std::runtime_error(ss.str()); } +std::vector SerialiseKey(uint8_t key) +{ + return { key }; +} +std::vector SerialiseKey(uint64_t key) +{ + std::vector buf(sizeof(key)); + std::memcpy(buf.data(), &key, sizeof(key)); + return buf; +} +std::vector SerialiseKey(uint128_t key) +{ + std::vector buf(16); +#ifdef __i386__ + std::memcpy(buf.data(), key.data, 16); +#else + std::memcpy(buf.data(), &key, 16); +#endif + return buf; +} + +std::vector SerialiseKey(uint256_t key) +{ + std::vector buf(32); + std::memcpy(buf.data(), key.data, 32); + return buf; +} + +void DeserialiseKey(void* data, uint8_t& key) +{ + uint8_t* p = (uint8_t*)data; + key = *p; +} +void DeserialiseKey(void* data, uint64_t& key) +{ + std::memcpy(&key, data, sizeof(key)); +} +void DeserialiseKey(void* data, uint128_t& key) +{ +#ifdef __i386__ + std::memcpy(key.data, data, 16); +#else + std::memcpy(&key, data, 16); +#endif +} + +void DeserialiseKey(void* data, uint256_t& key) +{ + std::memcpy(key.data, data, 32); +} + // Nodes are stored as a heap NodeKeyType GetKeyForNode(uint32_t level, index_t index) { @@ -59,7 +111,7 @@ int IntegerKeyCmp(const MDB_val* a, const MDB_val* b) uint64_t remainder = a->mv_size % sizeof(uint64_t); // If the size is > 32 bytes, use default comparison - if (a->mv_size > sizeof(uint256_t)) { + if (a->mv_size > 32) { return MemCmp(a, b); } // If the size is not a divisible by 8 then use default comparison, unless it is 1 byte diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp index f8d4089efc4..25d533881eb 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp @@ -5,6 +5,7 @@ #include "barretenberg/numeric/uint256/uint256.hpp" #include #include +#include namespace bb::crypto::merkle_tree { using NodeKeyType = uint128_t; @@ -20,14 +21,26 @@ int MemCmp(const MDB_val*, const MDB_val*); NodeKeyType GetKeyForNode(uint32_t level, index_t index); +std::vector SerialiseKey(uint8_t key); +std::vector SerialiseKey(uint64_t key); +std::vector SerialiseKey(uint128_t key); +std::vector SerialiseKey(uint256_t key); + +void DeserialiseKey(void* data, uint8_t& key); +void DeserialiseKey(void* data, uint64_t& key); +void DeserialiseKey(void* data, uint128_t& key); +void DeserialiseKey(void* data, uint256_t& key); + template int ValueCmp(const MDB_val* a, const MDB_val* b) { - const T* lhs = static_cast(a->mv_data); - const T* rhs = static_cast(b->mv_data); - if (*lhs < *rhs) { + T lhs; + T rhs; + DeserialiseKey(a->mv_data, lhs); + DeserialiseKey(b->mv_data, rhs); + if (lhs < rhs) { return -1; } - if (*lhs > *rhs) { + if (lhs > rhs) { return 1; } return 0; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp index 8f96a8cb126..e20986eba15 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp @@ -1,4 +1,5 @@ #pragma once +#include "barretenberg/common/serialize.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" @@ -41,28 +42,20 @@ class LMDBReadTransaction : public LMDBTransaction { template bool LMDBReadTransaction::get_value(T& key, std::vector& data) const { - MDB_val dbKey; - dbKey.mv_size = sizeof(T); - dbKey.mv_data = (void*)&key; - - MDB_val dbVal; - if (!call_lmdb_func(mdb_get, underlying(), _database.underlying(), &dbKey, &dbVal)) { - return false; - } - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); - return true; + std::vector keyBuffer = SerialiseKey(key); + return get_value(keyBuffer, data); } template bool LMDBReadTransaction::get_value_or_previous(T& key, std::vector& data) const { - T keyCopy = key; + std::vector keyBuffer = SerialiseKey(key); + uint32_t keySize = static_cast(keyBuffer.size()); MDB_cursor* cursor = nullptr; call_lmdb_func("mdb_cursor_open", mdb_cursor_open, underlying(), _database.underlying(), &cursor); MDB_val dbKey; - dbKey.mv_size = sizeof(T); - dbKey.mv_data = (void*)&keyCopy; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); MDB_val dbVal; @@ -72,7 +65,7 @@ template bool LMDBReadTransaction::get_value_or_previous(T& key, st int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); if (code == 0) { // we found the key, now determine if it is the exact key - if (dbKey.mv_size == sizeof(T) && std::memcmp(dbKey.mv_data, &key, dbKey.mv_size) == 0) { + if (dbKey.mv_size == keySize && std::memcmp(dbKey.mv_data, keyBuffer.data(), dbKey.mv_size) == 0) { // we have the exact key data.resize(dbVal.mv_size); std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); @@ -84,12 +77,12 @@ template bool LMDBReadTransaction::get_value_or_previous(T& key, st if (code == 0) { // We have found a previous key. It could be of the same size but smaller value, or smaller size which // is equal to not found - if (dbKey.mv_size != sizeof(T)) { + if (dbKey.mv_size != keySize) { // There is no previous key, do nothing } else { data.resize(dbVal.mv_size); std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); - std::memcpy(&key, dbKey.mv_data, dbKey.mv_size); + DeserialiseKey(dbKey.mv_data, key); success = true; } } else if (code == MDB_NOTFOUND) { @@ -103,12 +96,12 @@ template bool LMDBReadTransaction::get_value_or_previous(T& key, st code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); if (code == 0) { // We found the last key, but we need to ensure it is the same size - if (dbKey.mv_size != sizeof(T)) { + if (dbKey.mv_size != keySize) { // The key is not the same size, same as not found, do nothing } else { data.resize(dbVal.mv_size); std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); - std::memcpy(&key, dbKey.mv_data, dbKey.mv_size); + DeserialiseKey(dbKey.mv_data, key); success = true; } } else if (code == MDB_NOTFOUND) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp index d24fddefea9..e79792dbabc 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp @@ -269,7 +269,6 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) EXPECT_EQ(success, true); bb::fr value = from_buffer(data, 0); EXPECT_EQ(value, values[i]); - // uint256_t new_key = from_buffer(key, 0); EXPECT_EQ(key_copy, keys[i]); } @@ -282,7 +281,6 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) EXPECT_EQ(success, true); bb::fr value = from_buffer(data, 0); EXPECT_EQ(value, values[1]); - // uint256_t new_key = from_buffer(key, 0); EXPECT_EQ(key, keys[1]); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp index 9cb7c10734e..e5d9876e269 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp @@ -1,4 +1,6 @@ #pragma once +#include "barretenberg/common/serialize.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" @@ -39,13 +41,7 @@ class LMDBWriteTransaction : public LMDBTransaction { template void LMDBWriteTransaction::put_value(T& key, std::vector& data) { - MDB_val dbKey; - dbKey.mv_size = sizeof(T); - dbKey.mv_data = &key; - - MDB_val dbVal; - dbVal.mv_size = data.size(); - dbVal.mv_data = (void*)data.data(); - call_lmdb_func("mdb_put", mdb_put, underlying(), _database.underlying(), &dbKey, &dbVal, 0U); + std::vector keyBuffer = SerialiseKey(key); + put_value(keyBuffer, data); } } // namespace bb::crypto::merkle_tree \ No newline at end of file From f27a1a537f48943a7086408605d5f8f1b42d35bd Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Sun, 21 Jul 2024 17:06:19 +0100 Subject: [PATCH 53/63] Check serialisation of key sizes --- .../lmdb_store/lmdb_store.test.cpp | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp index e79792dbabc..ddc9a1a34fa 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp @@ -11,6 +11,7 @@ #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" #include "barretenberg/crypto/merkle_tree/fixtures.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" #include "barretenberg/numeric/random/engine.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" @@ -498,4 +499,38 @@ TEST_F(LMDBStoreTest, can_handle_different_key_spaces) check_values.template operator()(*transaction, 2); check_values.template operator()(*transaction, 3); } +} + +template void TestSerialisation(const T& key, uint32_t expectedSize) +{ + std::vector buf = SerialiseKey(key); + // Should serialise to expected size + EXPECT_EQ(expectedSize, buf.size()); + T newValue; + DeserialiseKey(buf.data(), newValue); + // Should be different objects + EXPECT_NE(&newValue, &key); + // Should have the same value + EXPECT_EQ(newValue, key); +} + +TEST_F(LMDBStoreTest, produces_correct_key_sizes) +{ + auto& random_engine = bb::numeric::get_randomness(); + { + uint8_t value = random_engine.get_random_uint8(); + TestSerialisation(value, 1); + } + { + uint64_t value = random_engine.get_random_uint64(); + TestSerialisation(value, 8); + } + { + uint128_t value = random_engine.get_random_uint128(); + TestSerialisation(value, 16); + } + { + uint256_t value = random_engine.get_random_uint256(); + TestSerialisation(value, 32); + } } \ No newline at end of file From 3691b261640fe5f4b1adee45c5578d81730af6ca Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 20 Aug 2024 15:47:40 +0000 Subject: [PATCH 54/63] Fix attempt --- barretenberg/cpp/src/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 43ccc5e2d56..73dcf324c9f 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -120,7 +120,6 @@ set(BARRETENBERG_TARGET_OBJECTS $ $ $ - $ $ $ $ From d11822e2039e82249ab63fc58bdbfe0f45e50fea Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 20 Aug 2024 16:03:03 +0000 Subject: [PATCH 55/63] Attempt fix --- barretenberg/cpp/src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 73dcf324c9f..45479d17746 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -140,6 +140,7 @@ set(BARRETENBERG_TARGET_OBJECTS $ $ $ + $ $ $ $ From 8980c620908781ac4e9721f8e8e1e16aa2a9908d Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 20 Aug 2024 16:52:42 +0000 Subject: [PATCH 56/63] Fix attempt --- barretenberg/cpp/src/CMakeLists.txt | 6 +++++- barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt | 5 ++++- .../cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 45479d17746..a2c8ba9eb15 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -140,7 +140,6 @@ set(BARRETENBERG_TARGET_OBJECTS $ $ $ - $ $ $ $ @@ -160,6 +159,11 @@ if(NOT DISABLE_AZTEC_VM) list(APPEND BARRETENBERG_TARGET_OBJECTS $) endif() +if(NOT WASM) + # enable merkle trees + list(APPEND BARRETENBERG_TARGET_OBJECTS $) +endif() + add_library( barretenberg STATIC diff --git a/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt index 1319f7bc668..f165b403102 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt @@ -11,4 +11,7 @@ add_subdirectory(sha256) add_subdirectory(ecdsa) add_subdirectory(aes128) add_subdirectory(poseidon2) -add_subdirectory(merkle_tree) \ No newline at end of file + +if (NOT WASM) + add_subdirectory(merkle_tree) +endif() diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt index 11052ae1155..4749a1a2021 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt @@ -7,4 +7,10 @@ barretenberg_module( if (NOT FUZZING) # but the tests use pedersen and poseidon target_link_libraries(crypto_merkle_tree_tests PRIVATE stdlib_pedersen_hash stdlib_poseidon2) + add_dependencies(crypto_merkle_tree_tests lmdb_repo) + add_dependencies(crypto_merkle_tree_test_objects lmdb_repo) endif() + +add_dependencies(crypto_merkle_tree lmdb_repo) +add_dependencies(crypto_merkle_tree_objects lmdb_repo) + From 9dd1d4944c6f847f114576ddf39d87b868adeb67 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 20 Aug 2024 17:05:04 +0000 Subject: [PATCH 57/63] Attempt fix --- .../src/barretenberg/benchmark/client_ivc_bench/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/CMakeLists.txt index 2757bea5d94..2ccd992aec3 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(client_ivc_bench client_ivc stdlib_honk_verifier stdlib_sha256 crypto_merkle_tree stdlib_primitives) +barretenberg_module(client_ivc_bench client_ivc stdlib_honk_verifier stdlib_sha256 stdlib_primitives) From 6518b6a889352118ce941dc3a525ea2fc1feae61 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 20 Aug 2024 17:12:02 +0000 Subject: [PATCH 58/63] Fix build --- .../src/barretenberg/benchmark/simulator_bench/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/CMakeLists.txt index 5cad28d95dc..caff4f1c2bc 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/CMakeLists.txt @@ -1 +1 @@ - barretenberg_module(simulator_bench stdlib_honk_verifier stdlib_sha256 crypto_merkle_tree) \ No newline at end of file + barretenberg_module(simulator_bench stdlib_honk_verifier stdlib_sha256 stdlib_pedersen_hash) \ No newline at end of file From 9249446f72756d938ce921fa10206fa0375aabb7 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 20 Aug 2024 17:31:42 +0000 Subject: [PATCH 59/63] Another build fix --- .../cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt index 92d6126f83c..34d8e03f7b2 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt @@ -5,6 +5,5 @@ barretenberg_module( stdlib_keccak stdlib_pedersen_hash stdlib_poseidon2 - crypto_merkle_tree plonk ) From 90f8a5d433694d164d21b707653d134d6fed6e31 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 21 Aug 2024 19:05:45 +0100 Subject: [PATCH 60/63] Reduced incidences of memcpy and memcmp --- .../merkle_tree/lmdb_store/functions.cpp | 59 ++++++++++++------- .../merkle_tree/lmdb_store/functions.hpp | 2 + .../lmdb_store/lmdb_read_transaction.cpp | 4 +- .../lmdb_store/lmdb_read_transaction.hpp | 13 ++-- 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp index 78d24337d26..c67531e7753 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp @@ -1,5 +1,7 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" +#include "lmdb.h" +#include #include #include #include @@ -17,12 +19,24 @@ std::vector SerialiseKey(uint8_t key) { return { key }; } + +void DeserialiseKey(void* data, uint8_t& key) +{ + uint8_t* p = static_cast(data); + key = *p; +} + std::vector SerialiseKey(uint64_t key) { - std::vector buf(sizeof(key)); - std::memcpy(buf.data(), &key, sizeof(key)); - return buf; + const uint8_t* p = reinterpret_cast(&key); + return std::vector(p, p + sizeof(key)); } + +void DeserialiseKey(void* data, uint64_t& key) +{ + std::memcpy(&key, data, sizeof(key)); +} + std::vector SerialiseKey(uint128_t key) { std::vector buf(16); @@ -34,22 +48,6 @@ std::vector SerialiseKey(uint128_t key) return buf; } -std::vector SerialiseKey(uint256_t key) -{ - std::vector buf(32); - std::memcpy(buf.data(), key.data, 32); - return buf; -} - -void DeserialiseKey(void* data, uint8_t& key) -{ - uint8_t* p = (uint8_t*)data; - key = *p; -} -void DeserialiseKey(void* data, uint64_t& key) -{ - std::memcpy(&key, data, sizeof(key)); -} void DeserialiseKey(void* data, uint128_t& key) { #ifdef __i386__ @@ -59,6 +57,13 @@ void DeserialiseKey(void* data, uint128_t& key) #endif } +std::vector SerialiseKey(uint256_t key) +{ + std::vector buf(32); + std::memcpy(buf.data(), key.data, 32); + return buf; +} + void DeserialiseKey(void* data, uint256_t& key) { std::memcpy(key.data, data, 32); @@ -83,12 +88,20 @@ int SizeCmp(const MDB_val* a, const MDB_val* b) return 0; } +std::vector mdb_val_to_vector(const MDB_val& dbVal) +{ + const uint8_t* p = static_cast(dbVal.mv_data); + return std::vector(p, p + dbVal.mv_size); +} + /** * Default lexicographical implementation of key comparisons used in our LMDB implementation */ int MemCmp(const MDB_val* a, const MDB_val* b) { - return std::memcmp(static_cast(a->mv_data), static_cast(b->mv_data), a->mv_size); + std::vector a_vector = mdb_val_to_vector(*a); + std::vector b_vector = mdb_val_to_vector(*b); + return std::lexicographical_compare(a_vector.begin(), a_vector.end(), b_vector.begin(), b_vector.end()); } /** @@ -126,4 +139,10 @@ int IntegerKeyCmp(const MDB_val* a, const MDB_val* b) }; return functions[factor](a, b); } + +void copy_to_vector(const MDB_val& dbVal, std::vector& target) +{ + std::vector temp = mdb_val_to_vector(dbVal); + target.swap(temp); +} } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp index 25d533881eb..3239d0aac7c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp @@ -47,6 +47,8 @@ template int ValueCmp(const MDB_val* a, const MDB_val* b) } int IntegerKeyCmp(const MDB_val* a, const MDB_val* b); +std::vector mdb_val_to_vector(const MDB_val& dbVal); +void copy_to_vector(const MDB_val& dbVal, std::vector& target); template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp index a5b9320b3bd..11310297461 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp @@ -1,5 +1,6 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include namespace bb::crypto::merkle_tree { LMDBReadTransaction::LMDBReadTransaction(LMDBEnvironment& env, const LMDBDatabase& database) @@ -28,8 +29,7 @@ bool LMDBReadTransaction::get_value(std::vector& key, std::vector #include #include @@ -65,10 +66,10 @@ template bool LMDBReadTransaction::get_value_or_previous(T& key, st int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); if (code == 0) { // we found the key, now determine if it is the exact key - if (dbKey.mv_size == keySize && std::memcmp(dbKey.mv_data, keyBuffer.data(), dbKey.mv_size) == 0) { + std::vector temp = mdb_val_to_vector(dbKey); + if (keyBuffer == temp) { // we have the exact key - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + copy_to_vector(dbVal, data); success = true; } else { // We have a key of the same size but larger value OR a larger size @@ -80,8 +81,7 @@ template bool LMDBReadTransaction::get_value_or_previous(T& key, st if (dbKey.mv_size != keySize) { // There is no previous key, do nothing } else { - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + copy_to_vector(dbVal, data); DeserialiseKey(dbKey.mv_data, key); success = true; } @@ -99,8 +99,7 @@ template bool LMDBReadTransaction::get_value_or_previous(T& key, st if (dbKey.mv_size != keySize) { // The key is not the same size, same as not found, do nothing } else { - data.resize(dbVal.mv_size); - std::memcpy(&data[0], dbVal.mv_data, dbVal.mv_size); + copy_to_vector(dbVal, data); DeserialiseKey(dbKey.mv_data, key); success = true; } From 29a6726c937802bdc3d6c30247b85bc93de288c8 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 21 Aug 2024 19:19:27 +0100 Subject: [PATCH 61/63] Change to snake case --- .../append_only_tree/append_only_tree.hpp | 34 +++++----- .../append_only_tree.test.cpp | 30 ++++----- .../merkle_tree/indexed_tree/indexed_tree.hpp | 22 +++---- .../indexed_tree/indexed_tree.test.cpp | 36 +++++------ .../merkle_tree/lmdb_store/functions.cpp | 34 +++++----- .../merkle_tree/lmdb_store/functions.hpp | 34 +++++----- .../lmdb_store/lmdb_environment.cpp | 4 +- .../lmdb_store/lmdb_environment.hpp | 4 +- .../lmdb_store/lmdb_read_transaction.cpp | 4 +- .../lmdb_store/lmdb_read_transaction.hpp | 14 ++-- .../merkle_tree/lmdb_store/lmdb_store.cpp | 6 +- .../merkle_tree/lmdb_store/lmdb_store.hpp | 4 +- .../lmdb_store/lmdb_store.test.cpp | 64 +++++++++---------- .../lmdb_store/lmdb_write_transaction.cpp | 3 +- .../lmdb_store/lmdb_write_transaction.hpp | 2 +- .../merkle_tree/node_store/array_store.hpp | 6 +- .../node_store/cached_tree_store.hpp | 34 +++++----- .../crypto/merkle_tree/response.hpp | 7 +- 18 files changed, 172 insertions(+), 170 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp index 7efbf9f7148..6041a9b4930 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -177,7 +177,7 @@ AppendOnlyTree::AppendOnlyTree(Store& store, ThreadPool& w bb::fr stored_root = fr::zero(); { // start by reading the meta data from the backing store - ReadTransactionPtr tx = store_.createReadTransaction(); + ReadTransactionPtr tx = store_.create_read_transaction(); store_.get_full_meta(stored_size, stored_root, name_, depth_, *tx, false); } zero_hashes_.resize(depth_ + 1); @@ -203,9 +203,9 @@ void AppendOnlyTree::get_meta_data(bool includeUncommitted const MetaDataCallback& on_completion) const { auto job = [=, this]() { - ExecuteAndReport( + execute_and_report( [=, this](TypedResponse& response) { - ReadTransactionPtr tx = store_.createReadTransaction(); + ReadTransactionPtr tx = store_.create_read_transaction(); store_.get_meta(response.inner.size, response.inner.root, *tx, includeUncommitted); response.inner.depth = depth_; }, @@ -220,10 +220,10 @@ void AppendOnlyTree::get_sibling_path(const index_t& index bool includeUncommitted) const { auto job = [=, this]() { - ExecuteAndReport( + execute_and_report( [=, this](TypedResponse& response) { index_t current_index = index; - ReadTransactionPtr tx = store_.createReadTransaction(); + ReadTransactionPtr tx = store_.create_read_transaction(); for (uint32_t level = depth_; level > 0; --level) { bool is_right = static_cast(current_index & 0x01); fr sibling = is_right ? get_element_or_zero(level, current_index - 1, *tx, includeUncommitted) @@ -243,9 +243,9 @@ void AppendOnlyTree::get_subtree_sibling_path(const uint32 bool includeUncommitted) const { auto job = [=, this]() { - ExecuteAndReport( + execute_and_report( [=, this](TypedResponse& response) { - ReadTransactionPtr tx = store_.createReadTransaction(); + ReadTransactionPtr tx = store_.create_read_transaction(); index_t index_of_next_leaf = 0; bb::fr root; store_.get_meta(index_of_next_leaf, root, *tx, includeUncommitted); @@ -264,9 +264,9 @@ void AppendOnlyTree::get_subtree_sibling_path(const index_ bool includeUncommitted) const { auto job = [=, this]() { - ExecuteAndReport( + execute_and_report( [=, this](TypedResponse& response) { - ReadTransactionPtr tx = store_.createReadTransaction(); + ReadTransactionPtr tx = store_.create_read_transaction(); response.inner.path = get_subtree_sibling_path_internal(leaf_index, subtree_depth, *tx, includeUncommitted); }, @@ -303,9 +303,9 @@ void AppendOnlyTree::get_leaf(const index_t& index, const GetLeafCallback& on_completion) const { auto job = [=, this]() { - ExecuteAndReport( + execute_and_report( [=, this](TypedResponse& response) { - ReadTransactionPtr tx = store_.createReadTransaction(); + ReadTransactionPtr tx = store_.create_read_transaction(); auto leaf = read_node(depth_, index, *tx, includeUncommitted); response.success = leaf.first; if (leaf.first) { @@ -332,9 +332,9 @@ void AppendOnlyTree::find_leaf_index_from(const fr& leaf, const FindLeafCallback& on_completion) const { auto job = [=, this]() -> void { - ExecuteAndReport( + execute_and_report( [=, this](TypedResponse& response) { - typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); + typename Store::ReadTransactionPtr tx = store_.create_read_transaction(); std::optional leaf_index = store_.find_leaf_index_from(leaf, start_index, *tx, includeUncommitted); response.success = leaf_index.has_value(); @@ -367,7 +367,7 @@ void AppendOnlyTree::add_values_internal(const std::vector { std::shared_ptr> hashes = std::make_shared>(values); auto append_op = [=, this]() -> void { - ExecuteAndReport( + execute_and_report( [=, this](TypedResponse& response) { add_values_internal(hashes, response.inner.root, response.inner.size, update_index); }, @@ -379,14 +379,14 @@ void AppendOnlyTree::add_values_internal(const std::vector template void AppendOnlyTree::commit(const CommitCallback& on_completion) { - auto job = [=, this]() { ExecuteAndReport([=, this]() { store_.commit(); }, on_completion); }; + auto job = [=, this]() { execute_and_report([=, this]() { store_.commit(); }, on_completion); }; workers_.enqueue(job); } template void AppendOnlyTree::rollback(const RollbackCallback& on_completion) { - auto job = [=, this]() { ExecuteAndReport([=, this]() { store_.rollback(); }, on_completion); }; + auto job = [=, this]() { execute_and_report([=, this]() { store_.rollback(); }, on_completion); }; workers_.enqueue(job); } @@ -404,7 +404,7 @@ void AppendOnlyTree::add_values_internal(std::shared_ptr(directory, 1024, 1, 2); - LMDBStore db(*environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); @@ -285,7 +285,7 @@ TEST_F(PersistedAppendOnlyTreeTest, errors_are_caught_and_handled) std::string directory = randomTempDirectory(); std::filesystem::create_directories(directory); auto environment = std::make_unique(directory, 300, 1, 2); - LMDBStore db(*environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); @@ -364,7 +364,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_commit_and_restore) std::string name = randomString(); MemoryTree memdb(depth); { - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); @@ -406,7 +406,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_commit_and_restore) // Re-create the store and tree, it should be the same as how we left it { - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); @@ -428,7 +428,7 @@ TEST_F(PersistedAppendOnlyTreeTest, test_size) { constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); TreeType tree(store, pool); @@ -456,7 +456,7 @@ TEST_F(PersistedAppendOnlyTreeTest, test_find_leaf_index) { constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); TreeType tree(store, pool); @@ -542,7 +542,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values) { constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); TreeType tree(store, pool); @@ -562,7 +562,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values_in_a_batch) { constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); TreeType tree(store, pool); @@ -582,7 +582,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_be_filled) { constexpr size_t depth = 3; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); TreeType tree(store, pool); @@ -614,7 +614,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_single_whilst_reading) { std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(8); TreeType tree(store, pool); @@ -648,7 +648,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_get_inserted_leaves) { constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); TreeType tree(store, pool); @@ -667,7 +667,7 @@ TEST_F(PersistedAppendOnlyTreeTest, returns_sibling_path) { constexpr size_t depth = 4; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); TreeType tree(store, pool); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp index 36c346ef15f..6f561553968 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -185,7 +185,7 @@ IndexedTree::IndexedTree(Store& store, ThreadPool& workers index_t stored_size = 0; bb::fr stored_root = fr::zero(); { - ReadTransactionPtr tx = store_.createReadTransaction(); + ReadTransactionPtr tx = store_.create_read_transaction(); std::string name; uint32_t depth = 0; store_.get_full_meta(stored_size, stored_root, name, depth, *tx, false); @@ -228,9 +228,9 @@ void IndexedTree::get_leaf(const index_t& index, const LeafCallback& completion) const { auto job = [=, this]() { - ExecuteAndReport>( + execute_and_report>( [=, this](TypedResponse>& response) { - ReadTransactionPtr tx = store_.createReadTransaction(); + ReadTransactionPtr tx = store_.create_read_transaction(); response.inner.indexed_leaf = store_.get_leaf(index, *tx, includeUncommitted); }, completion); @@ -255,9 +255,9 @@ void IndexedTree::find_leaf_index_from( const AppendOnlyTree::FindLeafCallback& on_completion) const { auto job = [=, this]() -> void { - ExecuteAndReport( + execute_and_report( [=, this](TypedResponse& response) { - typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); + typename Store::ReadTransactionPtr tx = store_.create_read_transaction(); std::optional leaf_index = store_.find_leaf_index_from(leaf, start_index, *tx, includeUncommitted); response.success = leaf_index.has_value(); @@ -276,9 +276,9 @@ void IndexedTree::find_low_leaf(const fr& leaf_key, const FindLowLeafCallback& on_completion) const { auto job = [=, this]() { - ExecuteAndReport>( + execute_and_report>( [=, this](TypedResponse>& response) { - typename Store::ReadTransactionPtr tx = store_.createReadTransaction(); + typename Store::ReadTransactionPtr tx = store_.create_read_transaction(); response.inner = store_.find_low_value(leaf_key, includeUncommitted, *tx); }, on_completion); @@ -461,7 +461,7 @@ void IndexedTree::perform_insertions(size_t total_leaves, Signal& leaderSignal = (*signals)[i]; Signal& followerSignal = (*signals)[i + 1]; try { - ReadTransactionPtr tx = store_.createReadTransaction(); + ReadTransactionPtr tx = store_.create_read_transaction(); auto& current_witness_data = low_leaf_witness_data->at(i); current_witness_data.leaf = insertion.original_low_leaf; current_witness_data.index = insertion.low_leaf_index; @@ -495,7 +495,7 @@ template void IndexedTree::generate_hashes_for_appending( std::shared_ptr> leaves_to_hash, const HashGenerationCallback& completion) { - ExecuteAndReport( + execute_and_report( [=](TypedResponse& response) { response.inner.hashes = std::make_shared>(leaves_to_hash->size(), 0); std::vector& leaves = *leaves_to_hash; @@ -513,7 +513,7 @@ void IndexedTree::generate_insertions( const std::shared_ptr>>& values_to_be_sorted, const InsertionGenerationCallback& on_completion) { - ExecuteAndReport( + execute_and_report( [=, this](TypedResponse& response) { // The first thing we do is sort the values into descending order but maintain knowledge of their // orignal order @@ -539,7 +539,7 @@ void IndexedTree::generate_insertions( index_t num_leaves_to_be_inserted = values.size(); std::set unique_values; { - ReadTransactionPtr tx = store_.createReadTransaction(); + ReadTransactionPtr tx = store_.create_read_transaction(); bb::fr old_root = fr::zero(); store_.get_meta(old_size, old_root, *tx, true); // Ensure that the tree is not going to be overfilled diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index 1773ce749a4..5ec63e05c2b 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -211,7 +211,7 @@ TEST_F(PersistedIndexedTreeTest, can_create) { constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); EXPECT_NO_THROW(Store store(name, depth, db)); Store store(name, depth, db); ThreadPool workers(1); @@ -226,7 +226,7 @@ TEST_F(PersistedIndexedTreeTest, can_only_recreate_with_same_name_and_depth) { constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); EXPECT_ANY_THROW(Store store_wrong_name("Wrong name", depth, db)); @@ -239,7 +239,7 @@ TEST_F(PersistedIndexedTreeTest, test_size) ThreadPool workers(1); constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -258,7 +258,7 @@ TEST_F(PersistedIndexedTreeTest, indexed_tree_must_have_at_least_2_initial_size) ThreadPool workers(1); constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); EXPECT_THROW(TreeType(store, workers, current_size), std::runtime_error); } @@ -269,7 +269,7 @@ TEST_F(PersistedIndexedTreeTest, reports_an_error_if_tree_is_overfilled) ThreadPool workers(1); constexpr size_t depth = 4; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -297,7 +297,7 @@ TEST_F(PersistedIndexedTreeTest, test_get_sibling_path) ThreadPool workers(1); constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -335,7 +335,7 @@ TEST_F(PersistedIndexedTreeTest, test_find_leaf_index) ThreadPool workers(1); constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, initial_size); @@ -396,7 +396,7 @@ TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) std::string name = randomString(); { - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -432,7 +432,7 @@ TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) // Now restore and it should continue from where we left off { - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -459,12 +459,12 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) NullifierMemoryTree memdb(depth, batch_size); std::string name1 = randomString(); - LMDBStore db1(*_environment, name1, false, false, IntegerKeyCmp); + LMDBStore db1(*_environment, name1, false, false, integer_key_cmp); Store store1(name1, depth, db1); auto tree1 = TreeType(store1, workers, batch_size); std::string name2 = randomString(); - LMDBStore db2(*_environment, name2, false, false, IntegerKeyCmp); + LMDBStore db2(*_environment, name2, false, false, integer_key_cmp); Store store2(name2, depth, db2); auto tree2 = TreeType(store2, workers, batch_size); @@ -530,7 +530,7 @@ TEST_F(PersistedIndexedTreeTest, reports_an_error_if_batch_contains_duplicate) ThreadPool workers(1); constexpr size_t depth = 10; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -591,7 +591,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) // Create a depth-3 indexed merkle tree constexpr size_t depth = 3; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -753,7 +753,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_tree) // Create a depth-8 indexed merkle tree constexpr uint32_t depth = 8; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -806,7 +806,7 @@ TEST_F(PersistedIndexedTreeTest, can_add_single_whilst_reading) { std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(8); TreeType tree(store, pool, 2); @@ -843,7 +843,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory_with_public_data_writes) // Create a depth-3 indexed merkle tree constexpr size_t depth = 3; std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); CachedTreeStore store(name, depth, db); auto tree = IndexedTree, Poseidon2HashPolicy>(store, workers, current_size); @@ -1002,7 +1002,7 @@ TEST_F(PersistedIndexedTreeTest, returns_low_leaves) ThreadPool workers(1); std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, 2); @@ -1026,7 +1026,7 @@ TEST_F(PersistedIndexedTreeTest, duplicates) ThreadPool workers(1); std::string name = randomString(); - LMDBStore db(*_environment, name, false, false, IntegerKeyCmp); + LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, 2); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp index c67531e7753..ff652ce3461 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp @@ -8,36 +8,36 @@ #include namespace bb::crypto::merkle_tree { -void ThrowError(const std::string& errorString, int error) +void throw_error(const std::string& errorString, int error) { std::stringstream ss; ss << errorString << ": " << error << " - " << mdb_strerror(error) << std::endl; throw std::runtime_error(ss.str()); } -std::vector SerialiseKey(uint8_t key) +std::vector serialise_key(uint8_t key) { return { key }; } -void DeserialiseKey(void* data, uint8_t& key) +void deserialise_key(void* data, uint8_t& key) { uint8_t* p = static_cast(data); key = *p; } -std::vector SerialiseKey(uint64_t key) +std::vector serialise_key(uint64_t key) { const uint8_t* p = reinterpret_cast(&key); return std::vector(p, p + sizeof(key)); } -void DeserialiseKey(void* data, uint64_t& key) +void deserialise_key(void* data, uint64_t& key) { std::memcpy(&key, data, sizeof(key)); } -std::vector SerialiseKey(uint128_t key) +std::vector serialise_key(uint128_t key) { std::vector buf(16); #ifdef __i386__ @@ -48,7 +48,7 @@ std::vector SerialiseKey(uint128_t key) return buf; } -void DeserialiseKey(void* data, uint128_t& key) +void deserialise_key(void* data, uint128_t& key) { #ifdef __i386__ std::memcpy(key.data, data, 16); @@ -57,27 +57,27 @@ void DeserialiseKey(void* data, uint128_t& key) #endif } -std::vector SerialiseKey(uint256_t key) +std::vector serialise_key(uint256_t key) { std::vector buf(32); std::memcpy(buf.data(), key.data, 32); return buf; } -void DeserialiseKey(void* data, uint256_t& key) +void deserialise_key(void* data, uint256_t& key) { std::memcpy(key.data, data, 32); } // Nodes are stored as a heap -NodeKeyType GetKeyForNode(uint32_t level, index_t index) +NodeKeyType get_key_for_node(uint32_t level, index_t index) { NodeKeyType key = static_cast(1) << level; key += static_cast(index); return key - 1; } -int SizeCmp(const MDB_val* a, const MDB_val* b) +int size_cmp(const MDB_val* a, const MDB_val* b) { if (a->mv_size < b->mv_size) { return -1; @@ -97,7 +97,7 @@ std::vector mdb_val_to_vector(const MDB_val& dbVal) /** * Default lexicographical implementation of key comparisons used in our LMDB implementation */ -int MemCmp(const MDB_val* a, const MDB_val* b) +int lexico_cmp(const MDB_val* a, const MDB_val* b) { std::vector a_vector = mdb_val_to_vector(*a); std::vector b_vector = mdb_val_to_vector(*b); @@ -114,28 +114,28 @@ int MemCmp(const MDB_val* a, const MDB_val* b) * 4. 32 bytes * If it is one of the above sizes then we assume it is an integer value and we compare it as such */ -int IntegerKeyCmp(const MDB_val* a, const MDB_val* b) +int integer_key_cmp(const MDB_val* a, const MDB_val* b) { // Id the keys sizes are different, sort by key size if (a->mv_size != b->mv_size) { - return SizeCmp(a, b); + return size_cmp(a, b); } uint64_t factor = a->mv_size / sizeof(uint64_t); uint64_t remainder = a->mv_size % sizeof(uint64_t); // If the size is > 32 bytes, use default comparison if (a->mv_size > 32) { - return MemCmp(a, b); + return lexico_cmp(a, b); } // If the size is not a divisible by 8 then use default comparison, unless it is 1 byte if (a->mv_size > 1 && remainder != 0) { - return MemCmp(a, b); + return lexico_cmp(a, b); } // Size is either 1, 8, 16 or 32 bytes, compare based on integer keys using f = std::function; static std::vector functions{ - ValueCmp, ValueCmp, ValueCmp, MemCmp, ValueCmp + value_cmp, value_cmp, value_cmp, lexico_cmp, value_cmp }; return functions[factor](a, b); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp index 3239d0aac7c..0c9174f09ec 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp @@ -13,30 +13,30 @@ using LeafIndexKeyType = uint64_t; using FrKeyType = uint256_t; using MetaKeyType = uint8_t; -void ThrowError(const std::string& errorString, int error); +void throw_error(const std::string& errorString, int error); -int SizeCmp(const MDB_val* a, const MDB_val* b); +int size_cmp(const MDB_val* a, const MDB_val* b); -int MemCmp(const MDB_val*, const MDB_val*); +int lexico_cmp(const MDB_val*, const MDB_val*); -NodeKeyType GetKeyForNode(uint32_t level, index_t index); +NodeKeyType get_key_for_node(uint32_t level, index_t index); -std::vector SerialiseKey(uint8_t key); -std::vector SerialiseKey(uint64_t key); -std::vector SerialiseKey(uint128_t key); -std::vector SerialiseKey(uint256_t key); +std::vector serialise_key(uint8_t key); +std::vector serialise_key(uint64_t key); +std::vector serialise_key(uint128_t key); +std::vector serialise_key(uint256_t key); -void DeserialiseKey(void* data, uint8_t& key); -void DeserialiseKey(void* data, uint64_t& key); -void DeserialiseKey(void* data, uint128_t& key); -void DeserialiseKey(void* data, uint256_t& key); +void deserialise_key(void* data, uint8_t& key); +void deserialise_key(void* data, uint64_t& key); +void deserialise_key(void* data, uint128_t& key); +void deserialise_key(void* data, uint256_t& key); -template int ValueCmp(const MDB_val* a, const MDB_val* b) +template int value_cmp(const MDB_val* a, const MDB_val* b) { T lhs; T rhs; - DeserialiseKey(a->mv_data, lhs); - DeserialiseKey(b->mv_data, rhs); + deserialise_key(a->mv_data, lhs); + deserialise_key(b->mv_data, rhs); if (lhs < rhs) { return -1; } @@ -46,7 +46,7 @@ template int ValueCmp(const MDB_val* a, const MDB_val* b) return 0; } -int IntegerKeyCmp(const MDB_val* a, const MDB_val* b); +int integer_key_cmp(const MDB_val* a, const MDB_val* b); std::vector mdb_val_to_vector(const MDB_val& dbVal); void copy_to_vector(const MDB_val& dbVal, std::vector& target); @@ -60,7 +60,7 @@ template void call_lmdb_func(const std::string& errorString, { int error = f(args...); if (error != 0) { - ThrowError(errorString, error); + throw_error(errorString, error); } } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp index 437ca66a926..63219e2f2a6 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp @@ -31,7 +31,7 @@ LMDBEnvironment::LMDBEnvironment(const std::string& directory, } } -void LMDBEnvironment::waitForReader() +void LMDBEnvironment::wait_for_reader() { std::unique_lock lock(_readersLock); if (_numReaders >= _maxReaders) { @@ -40,7 +40,7 @@ void LMDBEnvironment::waitForReader() ++_numReaders; } -void LMDBEnvironment::releaseReader() +void LMDBEnvironment::release_reader() { std::unique_lock lock(_readersLock); --_numReaders; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp index 29262360304..32e4ba13270 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp @@ -30,9 +30,9 @@ class LMDBEnvironment { MDB_env* underlying() const; - void waitForReader(); + void wait_for_reader(); - void releaseReader(); + void release_reader(); private: MDB_env* _mdbEnv; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp index 11310297461..1f582ac6f0f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp @@ -16,7 +16,7 @@ LMDBReadTransaction::~LMDBReadTransaction() void LMDBReadTransaction::abort() { LMDBTransaction::abort(); - _environment.releaseReader(); + _environment.release_reader(); } bool LMDBReadTransaction::get_value(std::vector& key, std::vector& data) const @@ -35,7 +35,7 @@ bool LMDBReadTransaction::get_value(std::vector& key, std::vector& data) const { - NodeKeyType key = GetKeyForNode(level, index); + NodeKeyType key = get_key_for_node(level, index); return get_value(key, data); } } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp index 308f28ef458..39f4ee21e89 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp @@ -43,13 +43,13 @@ class LMDBReadTransaction : public LMDBTransaction { template bool LMDBReadTransaction::get_value(T& key, std::vector& data) const { - std::vector keyBuffer = SerialiseKey(key); + std::vector keyBuffer = serialise_key(key); return get_value(keyBuffer, data); } template bool LMDBReadTransaction::get_value_or_previous(T& key, std::vector& data) const { - std::vector keyBuffer = SerialiseKey(key); + std::vector keyBuffer = serialise_key(key); uint32_t keySize = static_cast(keyBuffer.size()); MDB_cursor* cursor = nullptr; call_lmdb_func("mdb_cursor_open", mdb_cursor_open, underlying(), _database.underlying(), &cursor); @@ -82,13 +82,13 @@ template bool LMDBReadTransaction::get_value_or_previous(T& key, st // There is no previous key, do nothing } else { copy_to_vector(dbVal, data); - DeserialiseKey(dbKey.mv_data, key); + deserialise_key(dbKey.mv_data, key); success = true; } } else if (code == MDB_NOTFOUND) { // There is no previous key, do nothing } else { - ThrowError("get_value_or_previous::mdb_cursor_get", code); + throw_error("get_value_or_previous::mdb_cursor_get", code); } } } else if (code == MDB_NOTFOUND) { @@ -100,16 +100,16 @@ template bool LMDBReadTransaction::get_value_or_previous(T& key, st // The key is not the same size, same as not found, do nothing } else { copy_to_vector(dbVal, data); - DeserialiseKey(dbKey.mv_data, key); + deserialise_key(dbKey.mv_data, key); success = true; } } else if (code == MDB_NOTFOUND) { // DB is empty? } else { - ThrowError("get_value_or_previous::mdb_cursor_get", code); + throw_error("get_value_or_previous::mdb_cursor_get", code); } } else { - ThrowError("get_value_or_previous::mdb_cursor_get", code); + throw_error("get_value_or_previous::mdb_cursor_get", code); } call_lmdb_func(mdb_cursor_close, cursor); return success; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp index 91bd5fcb727..b209599e4bd 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.cpp @@ -16,13 +16,13 @@ LMDBStore::LMDBStore( , _database(_environment, LMDBDatabaseCreationTransaction(_environment), _name, integerKeys, reverseKeys, cmp) {} -LMDBWriteTransaction::Ptr LMDBStore::createWriteTransaction() const +LMDBWriteTransaction::Ptr LMDBStore::create_write_transaction() const { return std::make_unique(_environment, _database); } -LMDBReadTransaction::Ptr LMDBStore::createReadTransaction() +LMDBReadTransaction::Ptr LMDBStore::create_read_transaction() { - _environment.waitForReader(); + _environment.wait_for_reader(); return std::make_unique(_environment, _database); } } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp index b2073cbc371..52dc5b67f04 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp @@ -26,8 +26,8 @@ class LMDBStore { LMDBStore& operator=(LMDBStore&& other) = delete; ~LMDBStore() = default; - LMDBWriteTransaction::Ptr createWriteTransaction() const; - LMDBReadTransaction::Ptr createReadTransaction(); + LMDBWriteTransaction::Ptr create_write_transaction() const; + LMDBReadTransaction::Ptr create_read_transaction(); private: LMDBEnvironment& _environment; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp index ddc9a1a34fa..ba635aa4723 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp @@ -54,13 +54,13 @@ TEST_F(LMDBStoreTest, can_write_to_and_read_from_store) { std::vector buf; write(buf, VALUES[0]); - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); transaction->put_node(0, 0, buf); transaction->commit(); } { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); std::vector buf2; bool success = transaction->get_node(0, 0, buf2); EXPECT_EQ(success, true); @@ -76,13 +76,13 @@ TEST_F(LMDBStoreTest, can_write_to_and_read_from_store) write(key, VALUES[0]); std::vector value; write(value, VALUES[1]); - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); transaction->put_value(key, value); transaction->commit(); } { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); std::vector key; write(key, VALUES[0]); std::vector value; @@ -102,13 +102,13 @@ TEST_F(LMDBStoreTest, reading_an_empty_key_reports_correctly) { std::vector buf; write(buf, VALUES[0]); - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); transaction->put_node(0, 0, buf); transaction->commit(); } { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); std::vector buf2; bool success = transaction->get_node(0, 1, buf2); EXPECT_EQ(success, false); @@ -123,12 +123,12 @@ TEST_F(LMDBStoreTest, reading_an_empty_key_reports_correctly) write(key, VALUES[0]); std::vector value; write(value, VALUES[1]); - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); transaction->put_value(key, value); transaction->commit(); } { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); std::vector key2; write(key2, VALUES[5]); std::vector value; @@ -145,7 +145,7 @@ TEST_F(LMDBStoreTest, can_write_and_read_multiple) LMDBStore store(*_environment, "DB1"); { - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { std::vector buf; write(buf, VALUES[i]); @@ -155,7 +155,7 @@ TEST_F(LMDBStoreTest, can_write_and_read_multiple) } { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { std::vector buf2; bool success = transaction->get_node(10, i, buf2); @@ -171,7 +171,7 @@ TEST_F(LMDBStoreTest, can_write_and_read_multiple) uint32_t num_reads = 128; { - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); for (size_t i = 0; i < num_reads; i++) { std::vector key; write(key, VALUES[i]); @@ -184,7 +184,7 @@ TEST_F(LMDBStoreTest, can_write_and_read_multiple) { for (size_t i = 0; i < num_reads; i++) { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); std::vector key; write(key, VALUES[i]); std::vector buf2; @@ -204,7 +204,7 @@ TEST_F(LMDBStoreTest, throws_if_write_transaction_is_reused) { std::vector buf; write(buf, VALUES[0]); - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); transaction->put_node(0, 0, buf); transaction->commit(); EXPECT_THROW(transaction->put_node(0, 1, buf), std::runtime_error); @@ -218,7 +218,7 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) // This test is performed using integer keys of uint128_t // We also add keys of different sizes but with larger and smaller values // This ensures we don't erroneously return keys of different sizes - LMDBStore store(*_environment, "note hash tree", false, false, IntegerKeyCmp); + LMDBStore store(*_environment, "note hash tree", false, false, integer_key_cmp); std::vector values{ 1, 2, 3, 4, 5 }; @@ -235,7 +235,7 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) } { - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); for (size_t i = 0; i < num_keys; i++) { std::vector value; write(value, values[i]); @@ -263,7 +263,7 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) // First look for the value at each key, should return the exact keys for (uint32_t i = 0; i < num_keys; i++) { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); std::vector data; uint128_t key_copy = keys[i]; bool success = transaction->get_value_or_previous(key_copy, data); @@ -275,7 +275,7 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) // Now look for the value at key <= key[1] + 5 (does not exist), should be at keys[1] { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); std::vector data; uint128_t key = keys[1] + 5; bool success = transaction->get_value_or_previous(key, data); @@ -289,7 +289,7 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) // keys[4] { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); std::vector data; uint128_t key = keys[4] + 5; bool success = transaction->get_value_or_previous(key, data); @@ -301,7 +301,7 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) // Now look for the value at key <= keys[0] - 5 (does not exist, less than first key), should not exist { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); std::vector data; uint128_t key = keys[0] - 5; bool success = transaction->get_value_or_previous(key, data); @@ -313,7 +313,7 @@ TEST_F(LMDBStoreTest, can_retrieve_the_value_at_the_previous_key) TEST_F(LMDBStoreTest, can_not_retrieve_previous_key_from_empty_db) { LMDBStore store(*_environment, "note hash tree", false, false); - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); uint128_t key = 20; std::vector data; bool success = transaction->get_value(key, data); @@ -327,7 +327,7 @@ TEST_F(LMDBStoreTest, can_write_and_read_at_random_keys) std::vector keys; { - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { std::vector buf; @@ -340,7 +340,7 @@ TEST_F(LMDBStoreTest, can_write_and_read_at_random_keys) } { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { std::vector buf2; bool success = transaction->get_node(0, keys[i], buf2); @@ -357,7 +357,7 @@ TEST_F(LMDBStoreTest, can_recreate_the_store_and_use_again) { LMDBStore store(*_environment, "note hash tree"); - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { std::vector buf; @@ -372,7 +372,7 @@ TEST_F(LMDBStoreTest, can_recreate_the_store_and_use_again) { LMDBStore store(*_environment, "note hash tree"); - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); for (size_t i = 0; i < SAMPLE_DATA_SIZE; i++) { std::vector buf2; bool success = transaction->get_node(0, keys[i], buf2); @@ -387,7 +387,7 @@ void read_loop(LMDBStore& store, size_t key, std::atomic& flag, bb::fr s { bool seen = false; while (true) { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); std::vector buf; bool success = transaction->get_node(0, key, buf); EXPECT_EQ(success, true); @@ -413,7 +413,7 @@ TEST_F(LMDBStoreTest, can_read_from_multiple_threads) { // we write VALUES[0] to a slot - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); std::vector buf; write(buf, VALUES[0]); transaction->put_node(0, key, buf); @@ -433,7 +433,7 @@ TEST_F(LMDBStoreTest, can_read_from_multiple_threads) } { // we write VALUES[0] + 1 to the slot - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); std::vector buf; write(buf, VALUES[0] + 1); transaction->put_node(0, key, buf); @@ -448,7 +448,7 @@ TEST_F(LMDBStoreTest, can_read_from_multiple_threads) TEST_F(LMDBStoreTest, can_handle_different_key_spaces) { - LMDBStore store(*_environment, "DB1", false, false, IntegerKeyCmp); + LMDBStore store(*_environment, "DB1", false, false, integer_key_cmp); // create a set of keys std::vector keys{ 10, 20, 30, 40, 50 }; @@ -481,7 +481,7 @@ TEST_F(LMDBStoreTest, can_handle_different_key_spaces) }; { - LMDBWriteTransaction::Ptr transaction = store.createWriteTransaction(); + LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); write_values.template operator()(*transaction, 0); write_values.template operator()(*transaction, 1); @@ -491,7 +491,7 @@ TEST_F(LMDBStoreTest, can_handle_different_key_spaces) } { - LMDBReadTransaction::Ptr transaction = store.createReadTransaction(); + LMDBReadTransaction::Ptr transaction = store.create_read_transaction(); // we should be able to read values from different key spaces check_values.template operator()(*transaction, 0); @@ -503,11 +503,11 @@ TEST_F(LMDBStoreTest, can_handle_different_key_spaces) template void TestSerialisation(const T& key, uint32_t expectedSize) { - std::vector buf = SerialiseKey(key); + std::vector buf = serialise_key(key); // Should serialise to expected size EXPECT_EQ(expectedSize, buf.size()); T newValue; - DeserialiseKey(buf.data(), newValue); + deserialise_key(buf.data(), newValue); // Should be different objects EXPECT_NE(&newValue, &key); // Should have the same value diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp index c64bdfb6fe5..60b1b10ba34 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp @@ -1,6 +1,7 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" namespace bb::crypto::merkle_tree { @@ -33,7 +34,7 @@ void LMDBWriteTransaction::try_abort() void LMDBWriteTransaction::put_node(uint32_t level, index_t index, std::vector& data) { - NodeKeyType key = GetKeyForNode(level, index); + NodeKeyType key = get_key_for_node(level, index); put_value(key, data); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp index e5d9876e269..0dd729edf79 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp @@ -41,7 +41,7 @@ class LMDBWriteTransaction : public LMDBTransaction { template void LMDBWriteTransaction::put_value(T& key, std::vector& data) { - std::vector keyBuffer = SerialiseKey(key); + std::vector keyBuffer = serialise_key(key); put_value(keyBuffer, data); } } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp index 701a1e603b7..2a6a860f808 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp @@ -26,8 +26,8 @@ class MockPersistedStore { public: using ReadTransaction = MockTransaction; using WriteTransaction = MockTransaction; - static MockTransaction::Ptr createWriteTransaction() { return std::make_unique(); } - static MockTransaction::Ptr createReadTransaction() { return std::make_unique(); } + static MockTransaction::Ptr create_write_transaction() { return std::make_unique(); } + static MockTransaction::Ptr create_read_transaction() { return std::make_unique(); } }; /** @@ -85,7 +85,7 @@ template class ArrayStore { void commit(){}; void rollback(){}; - ReadTransactionPtr createReadTransaction() { return std::make_unique(); } + ReadTransactionPtr create_read_transactiono() { return std::make_unique(); } private: std::vector>>> map; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp index 7af89dbecbd..e1fef1a747e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp @@ -141,7 +141,7 @@ template class CachedTreeStore /** * @brief Returns a read transaction against the underlying store. */ - ReadTransactionPtr createReadTransaction() const { return dataStore.createReadTransaction(); } + ReadTransactionPtr create_read_transaction() const { return dataStore.create_read_transaction(); } private: struct Indices { @@ -160,11 +160,11 @@ template class CachedTreeStore void initialise(); - bool readPersistedMeta(TreeMeta& m, ReadTransaction& tx) const; + bool read_persisted_meta(TreeMeta& m, ReadTransaction& tx) const; - void persistMeta(TreeMeta& m, WriteTransaction& tx); + void persist_meta(TreeMeta& m, WriteTransaction& tx); - WriteTransactionPtr createWriteTransaction() const { return dataStore.createWriteTransaction(); } + WriteTransactionPtr create_write_transaction() const { return dataStore.create_write_transaction(); } }; template @@ -362,7 +362,7 @@ void CachedTreeStore::get_meta(index_t& size, return; } TreeMeta m; - readPersistedMeta(m, tx); + read_persisted_meta(m, tx); size = m.size; root = m.root; } @@ -379,7 +379,7 @@ void CachedTreeStore::get_full_meta( return; } TreeMeta m; - readPersistedMeta(m, tx); + read_persisted_meta(m, tx); size = m.size; root = m.root; depth = m.depth; @@ -390,7 +390,7 @@ template void CachedTreeStore< { { { - ReadTransactionPtr tx = createReadTransaction(); + ReadTransactionPtr tx = create_read_transaction(); for (auto& idx : indices_) { std::vector value; FrKeyType key = idx.first; @@ -403,7 +403,7 @@ template void CachedTreeStore< } } } - WriteTransactionPtr tx = createWriteTransaction(); + WriteTransactionPtr tx = create_write_transaction(); try { for (uint32_t i = 1; i < nodes.size(); i++) { auto& level = nodes[i]; @@ -427,7 +427,7 @@ template void CachedTreeStore< LeafIndexKeyType key = leaf.first; tx->put_value(key, value); } - persistMeta(meta, *tx); + persist_meta(meta, *tx); tx->commit(); } catch (std::exception& e) { tx->try_abort(); @@ -442,8 +442,8 @@ void CachedTreeStore::rollback() { // Extract the committed meta data and destroy the cache { - ReadTransactionPtr tx = createReadTransaction(); - readPersistedMeta(meta, *tx); + ReadTransactionPtr tx = create_read_transaction(); + read_persisted_meta(meta, *tx); } nodes = std::vector>>( depth + 1, std::unordered_map>()); @@ -452,7 +452,7 @@ void CachedTreeStore::rollback() } template -bool CachedTreeStore::readPersistedMeta(TreeMeta& m, ReadTransaction& tx) const +bool CachedTreeStore::read_persisted_meta(TreeMeta& m, ReadTransaction& tx) const { std::vector data; bool success = tx.get_node(0, 0, data); @@ -463,7 +463,7 @@ bool CachedTreeStore::readPersistedMeta(TreeMeta& } template -void CachedTreeStore::persistMeta(TreeMeta& m, WriteTransaction& tx) +void CachedTreeStore::persist_meta(TreeMeta& m, WriteTransaction& tx) { msgpack::sbuffer buffer; msgpack::pack(buffer, m); @@ -478,8 +478,8 @@ void CachedTreeStore::initialise() // construction then we throw std::vector data; { - ReadTransactionPtr tx = createReadTransaction(); - bool success = readPersistedMeta(meta, *tx); + ReadTransactionPtr tx = create_read_transaction(); + bool success = read_persisted_meta(meta, *tx); if (success) { if (name == meta.name && depth == meta.depth) { return; @@ -492,9 +492,9 @@ void CachedTreeStore::initialise() meta.name = name; meta.size = 0; meta.depth = depth; - WriteTransactionPtr tx = createWriteTransaction(); + WriteTransactionPtr tx = create_write_transaction(); try { - persistMeta(meta, *tx); + persist_meta(meta, *tx); tx->commit(); } catch (std::exception& e) { tx->try_abort(); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 89d96430976..38e7f8c6e4d 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -67,8 +67,8 @@ struct Response { }; template -void ExecuteAndReport(const std::function&)>& f, - const std::function&)>& on_completion) +void execute_and_report(const std::function&)>& f, + const std::function&)>& on_completion) { TypedResponse response; try { @@ -83,7 +83,8 @@ void ExecuteAndReport(const std::function&)>& f } } -inline void ExecuteAndReport(const std::function& f, const std::function& on_completion) +inline void execute_and_report(const std::function& f, + const std::function& on_completion) { Response response{ true, "" }; try { From 6c9a3147ad9b8b0ddf10d3429a300a95e8918d50 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 21 Aug 2024 19:29:20 +0100 Subject: [PATCH 62/63] File rename --- .../merkle_tree/lmdb_store/{functions.cpp => callbacks.cpp} | 2 +- .../merkle_tree/lmdb_store/{functions.hpp => callbacks.hpp} | 0 .../crypto/merkle_tree/lmdb_store/lmdb_database.cpp | 2 +- .../crypto/merkle_tree/lmdb_store/lmdb_database.hpp | 2 +- .../crypto/merkle_tree/lmdb_store/lmdb_db_transaction.cpp | 2 +- .../crypto/merkle_tree/lmdb_store/lmdb_environment.cpp | 2 +- .../crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp | 2 +- .../crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp | 2 +- .../crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp | 2 +- .../crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp | 2 +- .../crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp | 2 +- .../crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp | 2 +- 12 files changed, 11 insertions(+), 11 deletions(-) rename barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/{functions.cpp => callbacks.cpp} (98%) rename barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/{functions.hpp => callbacks.hpp} (100%) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.cpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.cpp index ff652ce3461..485b1bff766 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.cpp @@ -1,4 +1,4 @@ -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" #include "lmdb.h" #include diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.cpp index b565b92a828..2ac14c70fd0 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.cpp @@ -1,5 +1,5 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" namespace bb::crypto::merkle_tree { LMDBDatabase::LMDBDatabase(const LMDBEnvironment& env, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp index 48c63c4838a..c656cf023bb 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.cpp index 5b4204193a3..532b3818684 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.cpp @@ -1,5 +1,5 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_db_transaction.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" namespace bb::crypto::merkle_tree { LMDBDatabaseCreationTransaction::LMDBDatabaseCreationTransaction(LMDBEnvironment& env) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp index 63219e2f2a6..5350919c8f5 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp @@ -1,5 +1,5 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp index 1f582ac6f0f..03c3894020d 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.cpp @@ -1,5 +1,5 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include namespace bb::crypto::merkle_tree { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp index 39f4ee21e89..be0080f6cb0 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_read_transaction.hpp @@ -1,6 +1,6 @@ #pragma once #include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp index ba635aa4723..915d98b6368 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp @@ -11,7 +11,7 @@ #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" #include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/numeric/random/engine.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp index b925a216d7b..5393ffde3ff 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.cpp @@ -1,5 +1,5 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" namespace bb::crypto::merkle_tree { LMDBTransaction::LMDBTransaction(LMDBEnvironment& env, bool readOnly) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp index 60b1b10ba34..41a15c0be29 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.cpp @@ -1,7 +1,7 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" namespace bb::crypto::merkle_tree { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp index 0dd729edf79..a6723c78574 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_write_transaction.hpp @@ -1,6 +1,6 @@ #pragma once #include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/functions.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" From aeee8af87adc579bea9b8d8e732cda8bf2cf2cfc Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 21 Aug 2024 21:15:22 +0100 Subject: [PATCH 63/63] Fixes --- .../append_only_tree.bench.cpp | 6 ++-- .../indexed_tree_bench/indexed_tree.bench.cpp | 13 +++---- .../append_only_tree.test.cpp | 34 +++++++++--------- .../crypto/merkle_tree/fixtures.hpp | 6 ++-- .../indexed_tree/indexed_tree.test.cpp | 36 +++++++++---------- .../lmdb_store/lmdb_store.test.cpp | 2 +- 6 files changed, 49 insertions(+), 48 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp index 869702792b2..833ce035d1e 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp @@ -48,13 +48,13 @@ template void append_only_tree_bench(State& state) noexcept const size_t batch_size = size_t(state.range(0)); const size_t depth = TREE_DEPTH; - std::string directory = randomTempDirectory(); - std::string name = randomString(); + std::string directory = random_temp_directory(); + std::string name = random_string(); std::filesystem::create_directories(directory); uint32_t num_threads = 16; LMDBEnvironment environment = LMDBEnvironment(directory, 1024 * 1024, 2, num_threads); - LMDBStore db(environment, name, false, false, IntegerKeyCmp); + LMDBStore db(environment, name, false, false, integer_key_cmp); StoreType store(name, depth, db); ThreadPool workers(num_threads); TreeType tree = TreeType(store, workers); diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp index 6d6cd393a91..ae653a1dad3 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp @@ -2,6 +2,7 @@ #include "barretenberg/crypto/merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.hpp" #include "barretenberg/crypto/merkle_tree/node_store/cached_tree_store.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" @@ -34,13 +35,13 @@ template void multi_thread_indexed_tree_bench(State& state) const size_t batch_size = size_t(state.range(0)); const size_t depth = TREE_DEPTH; - std::string directory = randomTempDirectory(); - std::string name = randomString(); + std::string directory = random_temp_directory(); + std::string name = random_string(); std::filesystem::create_directories(directory); uint32_t num_threads = 16; LMDBEnvironment environment = LMDBEnvironment(directory, 1024 * 1024, 2, num_threads); - LMDBStore db(environment, name, false, false, IntegerKeyCmp); + LMDBStore db(environment, name, false, false, integer_key_cmp); StoreType store(name, depth, db); ThreadPool workers(num_threads); TreeType tree = TreeType(store, workers, batch_size); @@ -61,13 +62,13 @@ template void single_thread_indexed_tree_bench(State& state) const size_t batch_size = size_t(state.range(0)); const size_t depth = TREE_DEPTH; - std::string directory = randomTempDirectory(); - std::string name = randomString(); + std::string directory = random_temp_directory(); + std::string name = random_string(); std::filesystem::create_directories(directory); uint32_t num_threads = 1; LMDBEnvironment environment = LMDBEnvironment(directory, 1024 * 1024, 2, num_threads); - LMDBStore db(environment, name, false, false, IntegerKeyCmp); + LMDBStore db(environment, name, false, false, integer_key_cmp); StoreType store(name, depth, db); ThreadPool workers(num_threads); TreeType tree = TreeType(store, workers, batch_size); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp index 49ffe37949c..656242cab18 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -30,7 +30,7 @@ class PersistedAppendOnlyTreeTest : public testing::Test { void SetUp() override { // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers - _directory = randomTempDirectory(); + _directory = random_temp_directory(); std::filesystem::create_directories(_directory); _environment = std::make_unique(_directory, 1024, 2, 2); } @@ -201,7 +201,7 @@ void check_sibling_path(fr expected_root, fr node, index_t index, fr_sibling_pat TEST_F(PersistedAppendOnlyTreeTest, can_create) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); EXPECT_NO_THROW(Store store(name, depth, db)); Store store(name, depth, db); @@ -217,7 +217,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_create) TEST_F(PersistedAppendOnlyTreeTest, can_only_recreate_with_same_name_and_depth) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); @@ -228,7 +228,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_only_recreate_with_same_name_and_depth) TEST_F(PersistedAppendOnlyTreeTest, can_add_value_and_get_sibling_path) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); @@ -250,8 +250,8 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_value_and_get_sibling_path) TEST_F(PersistedAppendOnlyTreeTest, reports_an_error_if_tree_is_overfilled) { constexpr size_t depth = 4; - std::string name = randomString(); - std::string directory = randomTempDirectory(); + std::string name = random_string(); + std::string directory = random_temp_directory(); std::filesystem::create_directories(directory); auto environment = std::make_unique(directory, 1024, 1, 2); LMDBStore db(*environment, name, false, false, integer_key_cmp); @@ -281,8 +281,8 @@ TEST_F(PersistedAppendOnlyTreeTest, errors_are_caught_and_handled) { // We use a deep tree with a small amount of storage (20 * 1024) bytes constexpr size_t depth = 16; - std::string name = randomString(); - std::string directory = randomTempDirectory(); + std::string name = random_string(); + std::string directory = random_temp_directory(); std::filesystem::create_directories(directory); auto environment = std::make_unique(directory, 300, 1, 2); LMDBStore db(*environment, name, false, false, integer_key_cmp); @@ -361,7 +361,7 @@ TEST_F(PersistedAppendOnlyTreeTest, errors_are_caught_and_handled) TEST_F(PersistedAppendOnlyTreeTest, can_commit_and_restore) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); MemoryTree memdb(depth); { LMDBStore db(*_environment, name, false, false, integer_key_cmp); @@ -427,7 +427,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_commit_and_restore) TEST_F(PersistedAppendOnlyTreeTest, test_size) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); @@ -455,7 +455,7 @@ TEST_F(PersistedAppendOnlyTreeTest, test_size) TEST_F(PersistedAppendOnlyTreeTest, test_find_leaf_index) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); @@ -541,7 +541,7 @@ TEST_F(PersistedAppendOnlyTreeTest, test_find_leaf_index) TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); @@ -561,7 +561,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values) TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values_in_a_batch) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); @@ -581,7 +581,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_multiple_values_in_a_batch) TEST_F(PersistedAppendOnlyTreeTest, can_be_filled) { constexpr size_t depth = 3; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); @@ -613,7 +613,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_single_whilst_reading) std::vector paths(num_reads); { - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(8); @@ -647,7 +647,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_add_single_whilst_reading) TEST_F(PersistedAppendOnlyTreeTest, can_get_inserted_leaves) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); @@ -666,7 +666,7 @@ TEST_F(PersistedAppendOnlyTreeTest, can_get_inserted_leaves) TEST_F(PersistedAppendOnlyTreeTest, returns_sibling_path) { constexpr size_t depth = 4; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(1); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp index 52c651c9924..d824198d856 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp @@ -20,17 +20,17 @@ static std::vector VALUES = []() { return values; }(); -inline std::string randomString() +inline std::string random_string() { std::stringstream ss; ss << random_engine.get_random_uint256(); return ss.str(); } -inline std::string randomTempDirectory() +inline std::string random_temp_directory() { std::stringstream ss; - ss << "/tmp/lmdb/" << randomString(); + ss << "/tmp/lmdb/" << random_string(); return ss.str(); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp index 5ec63e05c2b..854c38d1882 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -38,7 +38,7 @@ class PersistedIndexedTreeTest : public testing::Test { void SetUp() override { // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers - _directory = randomTempDirectory(); + _directory = random_temp_directory(); std::filesystem::create_directories(_directory); _environment = std::make_unique(_directory, 1024, 2, 2); } @@ -210,7 +210,7 @@ void add_values(IndexedTree, Poseidon2 TEST_F(PersistedIndexedTreeTest, can_create) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); EXPECT_NO_THROW(Store store(name, depth, db)); Store store(name, depth, db); @@ -225,7 +225,7 @@ TEST_F(PersistedIndexedTreeTest, can_create) TEST_F(PersistedIndexedTreeTest, can_only_recreate_with_same_name_and_depth) { constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); @@ -238,7 +238,7 @@ TEST_F(PersistedIndexedTreeTest, test_size) index_t current_size = 2; ThreadPool workers(1); constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -257,7 +257,7 @@ TEST_F(PersistedIndexedTreeTest, indexed_tree_must_have_at_least_2_initial_size) index_t current_size = 1; ThreadPool workers(1); constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); EXPECT_THROW(TreeType(store, workers, current_size), std::runtime_error); @@ -268,7 +268,7 @@ TEST_F(PersistedIndexedTreeTest, reports_an_error_if_tree_is_overfilled) index_t current_size = 2; ThreadPool workers(1); constexpr size_t depth = 4; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -296,7 +296,7 @@ TEST_F(PersistedIndexedTreeTest, test_get_sibling_path) ThreadPool workers(1); constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -334,7 +334,7 @@ TEST_F(PersistedIndexedTreeTest, test_find_leaf_index) index_t initial_size = 2; ThreadPool workers(1); constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, initial_size); @@ -393,7 +393,7 @@ TEST_F(PersistedIndexedTreeTest, can_commit_and_restore) index_t current_size = 2; ThreadPool workers(1); constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); { LMDBStore db(*_environment, name, false, false, integer_key_cmp); @@ -458,12 +458,12 @@ TEST_F(PersistedIndexedTreeTest, test_batch_insert) ThreadPool multi_workers(8); NullifierMemoryTree memdb(depth, batch_size); - std::string name1 = randomString(); + std::string name1 = random_string(); LMDBStore db1(*_environment, name1, false, false, integer_key_cmp); Store store1(name1, depth, db1); auto tree1 = TreeType(store1, workers, batch_size); - std::string name2 = randomString(); + std::string name2 = random_string(); LMDBStore db2(*_environment, name2, false, false, integer_key_cmp); Store store2(name2, depth, db2); auto tree2 = TreeType(store2, workers, batch_size); @@ -529,7 +529,7 @@ TEST_F(PersistedIndexedTreeTest, reports_an_error_if_batch_contains_duplicate) index_t current_size = 2; ThreadPool workers(1); constexpr size_t depth = 10; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -590,7 +590,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory) ThreadPool workers(8); // Create a depth-3 indexed merkle tree constexpr size_t depth = 3; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -752,7 +752,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_tree) ThreadPool workers(1); // Create a depth-8 indexed merkle tree constexpr uint32_t depth = 8; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, current_size); @@ -805,7 +805,7 @@ TEST_F(PersistedIndexedTreeTest, can_add_single_whilst_reading) std::vector paths(num_reads); { - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); ThreadPool pool(8); @@ -842,7 +842,7 @@ TEST_F(PersistedIndexedTreeTest, test_indexed_memory_with_public_data_writes) ThreadPool workers(8); // Create a depth-3 indexed merkle tree constexpr size_t depth = 3; - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); CachedTreeStore store(name, depth, db); auto tree = @@ -1001,7 +1001,7 @@ TEST_F(PersistedIndexedTreeTest, returns_low_leaves) constexpr uint32_t depth = 8; ThreadPool workers(1); - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, 2); @@ -1025,7 +1025,7 @@ TEST_F(PersistedIndexedTreeTest, duplicates) constexpr uint32_t depth = 8; ThreadPool workers(1); - std::string name = randomString(); + std::string name = random_string(); LMDBStore db(*_environment, name, false, false, integer_key_cmp); Store store(name, depth, db); auto tree = TreeType(store, workers, 2); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp index 915d98b6368..ee81fec390d 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_store.test.cpp @@ -33,7 +33,7 @@ class LMDBStoreTest : public testing::Test { void SetUp() override { // setup with 1MB max db size, 1 max database and 2 maximum concurrent readers - _directory = randomTempDirectory(); + _directory = random_temp_directory(); std::filesystem::create_directories(_directory); _environment = std::make_unique(_directory, 1024, 2, 2); }