Merge defi-bridge-project into master (dependent CMake pj support, rollup/ removal, kesha's bigfield fix)

.circleci/config.yml

@@ -209,7 +209,7 @@ jobs:
           command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 3 stdlib_recursion_tests --gtest_filter=-*turbo*
       - *save_logs
 
-  tx-rollup-tests:
+  join-split-tests:
     docker:
       - image: aztecprotocol/alpine-build-image
     resource_class: small
@@ -218,55 +218,7 @@ jobs:
       - *setup_env
       - run:
           name: "Test"
-          command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 1 rollup_proofs_tx_rollup_tests --gtest_filter=-rollup_full_tests.*
-      - *save_logs
-
-  tx-rollup-full-tests:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Test"
-          command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 1 rollup_proofs_tx_rollup_tests --gtest_filter=rollup_full_tests.*
-      - *save_logs
-
-  root-rollup-tests:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Test"
-          command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 1 rollup_proofs_root_rollup_tests --gtest_filter=-root_rollup_full_tests.*
-      - *save_logs
-
-  root-rollup-full-tests:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Test"
-          command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 1 rollup_proofs_root_rollup_tests --gtest_filter=root_rollup_full_tests.*
-      - *save_logs
-
-  root-verifier-tests:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Test"
-          command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 3 rollup_proofs_root_verifier_tests
+          command: cond_spot_run_tests barretenberg-x86_64-linux-clang-assert 3 join_split_example_proofs_join_split_tests --gtest_filter=-*full_proof*
       - *save_logs
 
   benchmark-aggregator:
@@ -298,26 +250,26 @@ workflows:
       - honk-tests: *bb_test
       - barretenberg-tests: *bb_test
       - stdlib-primitives-tests: *bb_test
+<<<<<<< HEAD
       - stdlib-biggroup-tests: *bb_test
       - stdlib-recursion-turbo-tests: *bb_test
       - stdlib-recursion-ultra-tests: *bb_test
-      - tx-rollup-tests: *bb_test
-      - tx-rollup-full-tests: *bb_test
-      - root-rollup-tests: *bb_test
-      - root-rollup-full-tests: *bb_test
-      - root-verifier-tests: *bb_test
+=======
+      - stdlib-recursion-tests: *bb_test
+>>>>>>> origin/defi-bridge-project
+      - join-split-tests: *bb_test
       - benchmark-aggregator:
           requires:
             - barretenberg-tests
             - stdlib-primitives-tests
+<<<<<<< HEAD
             - stdlib-biggroup-tests
             - stdlib-recursion-turbo-tests
             - stdlib-recursion-ultra-tests
-            - tx-rollup-tests
-            - tx-rollup-full-tests
-            - root-verifier-tests
-            - root-rollup-tests
-            - root-rollup-full-tests
+=======
+            - stdlib-recursion-tests
+>>>>>>> origin/defi-bridge-project
+            - join-split-tests
           filters:
             branches:
               only:

README.md

@@ -17,8 +17,8 @@ The structured reference string contains monomials up to x^{2^20}. This SRS was
 RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git \
   && cd llvm-project && mkdir build-openmp && cd build-openmp \
   && cmake ../openmp -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLIBOMP_ENABLE_SHARED=OFF \
-  && make -j$(nproc) \
-  && make install \
+  && cmake --build . --parallel \
+  && cmake --build . --parallel --target install \
   && cd ../.. && rm -rf llvm-project
 ```
 
@@ -27,12 +27,13 @@ RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git
 Run the bootstrap script. (The bootstrap script will build both the native and wasm versions of barretenberg)
 
 ```
+cd cpp
 ./bootstrap.sh
 ```
 
 ### Parallelise the build
 
-Make sure your MAKEFLAGS environment variable is set to run jobs equal to number of cores. e.g. `MAKEFLAGS=-j$(nproc)`.
+Use the `--parallel` option to `cmake --build <path>` to parallelize builds. This is roughly equivalent to `make -j$(nproc)` but is more portable.
 
 ### Formatting
 
@@ -44,20 +45,20 @@ If you've installed the C++ Vscode extension you should configure it to format o
 Each module has its own tests. e.g. To build and run `ecc` tests:
 
 ```
-make ecc_tests
+cmake --build . --parallel --target ecc_tests
 ./bin/ecc_tests
 ```
 
 A shorthand for the above is:
 
 ```
-make run_ecc_tests
+cmake --build . --parallel --target run_ecc_tests
 ```
 
 Running the entire suite of tests using `ctest`:
 
 ```
-make test
+cmake --build . --parallel --target test
 ```
 
 You can run specific tests, e.g.
@@ -71,14 +72,14 @@ You can run specific tests, e.g.
 Some modules have benchmarks. The build targets are named `<module_name>_bench`. To build and run, for example `ecc` benchmarks.
 
 ```
-make ecc_bench
+cmake --build . --parallel --target ecc_bench
 ./src/aztec/ecc/ecc_bench
 ```
 
 A shorthand for the above is:
 
 ```
-make run_ecc_bench
+cmake --build . --parallel --target run_ecc_bench
 ```
 
 ### CMake Build Options
@@ -101,10 +102,10 @@ To build:
 ```
 mkdir build-wasm && cd build-wasm
 cmake -DTOOLCHAIN=wasm-linux-clang ..
-make barretenberg.wasm
+cmake --build . --parallel --target barretenberg.wasm
 ```
 
-The resulting wasm binary will be at `./src/aztec/barretenberg.wasm`.
+The resulting wasm binary will be at `./build-wasm/bin/barretenberg.wasm`.
 
 To run the tests, you'll need to install `wasmtime`.
 
@@ -115,7 +116,7 @@ curl https://wasmtime.dev/install.sh -sSf | bash
 Tests can be built and run like:
 
 ```
-make ecc_tests
+cmake --build . --parallel --target ecc_tests
 wasmtime --dir=.. ./bin/ecc_tests
 ```
 
@@ -125,11 +126,11 @@ To build:
 ```
 mkdir build-fuzzing && cd build-fuzzing
 cmake -DTOOLCHAIN=x86_64-linux-clang -DFUZZING=ON ..
-make
+cmake --build . --parallel
 ```
 Fuzzing build turns off building tests and benchmarks, since they are incompatible with libfuzzer interface.
 
 To turn on address sanitizer add `-DADDRESS_SANITIZER=ON`. Note that address sanitizer can be used to explore crashes.
 Sometimes you might have to specify the address of llvm-symbolizer. You have to do it with `export ASAN_SYMBOLIZER_PATH=<PATH_TO_SYMBOLIZER>`.
 For undefined behaviour sanitizer `-DUNDEFINED_BEHAVIOUR_SANITIZER=ON`.
-Note that the fuzzer can be orders of magnitude slower with ASan (2-3x slower) or UBSan on, so it is best to run a non-sanitized build first, minimize the testcase and then run it for a bit of time with sanitizers.  
+Note that the fuzzer can be orders of magnitude slower with ASan (2-3x slower) or UBSan on, so it is best to run a non-sanitized build first, minimize the testcase and then run it for a bit of time with sanitizers.

cpp/CMakeLists.txt

@@ -47,6 +47,7 @@ if(FUZZING)
 
     set(WASM OFF)
     set(BENCHMARKS OFF)
+    set(MULTITHREADING OFF)
     set(TESTING OFF)
 endif()
 
@@ -84,4 +85,4 @@ include(cmake/gtest.cmake)
 include(cmake/benchmark.cmake)
 include(cmake/module.cmake)
 
-add_subdirectory(src/aztec)
+add_subdirectory(src/aztec)

cpp/bootstrap.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
-set -e
+set -eu
 
 # Clean.
 rm -rf ./build
 rm -rf ./build-wasm
+rm -rf ./src/wasi-sdk-*
 
 # Install formatting git hook.
 HOOKS_DIR=$(git rev-parse --git-path hooks)
@@ -16,12 +17,12 @@ chmod +x $HOOKS_DIR/pre-commit
 
 # Determine system.
 if [[ "$OSTYPE" == "darwin"* ]]; then
-    OS=macos
+  OS=macos
 elif [[ "$OSTYPE" == "linux-gnu" ]]; then
-    OS=linux
+  OS=linux
 else
-    echo "Unknown OS: $OSTYPE"
-    exit 1
+  echo "Unknown OS: $OSTYPE"
+  exit 1
 fi
 
 # Download ignition transcripts.
@@ -32,31 +33,30 @@ cd ..
 
 # Pick native toolchain file.
 if [ "$OS" == "macos" ]; then
-    export BREW_PREFIX=$(brew --prefix)
-    # Ensure we have toolchain.
-    if [ ! "$?" -eq 0 ] || [ ! -f "$BREW_PREFIX/opt/llvm/bin/clang++" ]; then
-        echo "Default clang not sufficient. Install homebrew, and then: brew install llvm libomp clang-format"
-        exit 1
-    fi
-    ARCH=$(uname -m)
-    if [ "$ARCH" = "arm64" ]; then
-        TOOLCHAIN=arm-apple-clang
-    else
-        TOOLCHAIN=x86_64-apple-clang
-    fi
+  export BREW_PREFIX=$(brew --prefix)
+  # Ensure we have toolchain.
+  if [ ! "$?" -eq 0 ] || [ ! -f "$BREW_PREFIX/opt/llvm/bin/clang++" ]; then
+    echo "Default clang not sufficient. Install homebrew, and then: brew install llvm libomp clang-format"
+    exit 1
+  fi
+  ARCH=$(uname -m)
+  if [ "$ARCH" = "arm64" ]; then
+    TOOLCHAIN=arm-apple-clang
+  else
+    TOOLCHAIN=x86_64-apple-clang
+  fi
 else
-    TOOLCHAIN=x86_64-linux-clang
+  TOOLCHAIN=x86_64-linux-clang
 fi
 
 # Build native.
 mkdir -p build && cd build
 cmake -DCMAKE_BUILD_TYPE=RelWithAssert -DTOOLCHAIN=$TOOLCHAIN ..
-make -j$(getconf _NPROCESSORS_ONLN) $@
+cmake --build . --parallel ${@/#/--target }
 cd ..
 
 # Install the webassembly toolchain.
 WASI_VERSION=12
-rm -rf ./src/wasi-sdk-*
 cd ./src
 curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$WASI_VERSION/wasi-sdk-$WASI_VERSION.0-$OS.tar.gz | tar zxfv -
 cd ..
@@ -65,4 +65,4 @@ cd ..
 mkdir -p build-wasm && cd build-wasm
 cmake -DTOOLCHAIN=wasm-linux-clang ..
 cmake --build . --parallel --target barretenberg.wasm
-cd ..
+cd ..

cpp/cmake/arch.cmake

@@ -6,5 +6,5 @@ if(WASM)
 endif()
 
 if(NOT WASM AND NOT APPLE)
-    add_compile_options(-march=skylake-avx512)
+    add_compile_options(-march=skylake)
 endif()

cpp/cmake/toolchains/wasm-linux-clang.cmake

@@ -9,7 +9,10 @@ set(CMAKE_SYSTEM_VERSION 1)
 set(CMAKE_SYSTEM_PROCESSOR wasm32)
 set(triple wasm32-wasi)
 
-set(WASI_SDK_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/src/wasi-sdk-12.0")
+if (NOT WASI_SDK_PREFIX)
+  # can be set by a dependent project
+  set(WASI_SDK_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/src/wasi-sdk-12.0")
+endif()
 set(CMAKE_C_COMPILER ${WASI_SDK_PREFIX}/bin/clang)
 set(CMAKE_CXX_COMPILER ${WASI_SDK_PREFIX}/bin/clang++)
 set(CMAKE_AR ${WASI_SDK_PREFIX}/bin/llvm-ar CACHE STRING "wasi-sdk build")

cpp/dockerfiles/Dockerfile.arm64-linux-gcc

@@ -1,6 +1,6 @@
 FROM aztecprotocol/crosstool-ng-arm64:latest
 WORKDIR /usr/src/barretenberg/cpp
 COPY . .
-RUN mkdir build && cd build && cmake -DTOOLCHAIN=arm64-linux-gcc .. && make -j$(nproc)
+RUN mkdir build && cd build && cmake -DTOOLCHAIN=arm64-linux-gcc .. && cmake --build . --parallel
 RUN cd build && qemu-aarch64 ./test/barretenberg_tests
 ENTRYPOINT /bin/bash

cpp/dockerfiles/Dockerfile.wasm-linux-clang

@@ -7,7 +7,7 @@ WORKDIR /usr/src/barretenberg/cpp
 COPY . .
 # Build both honk_tests barretenberg.wasm
 # This ensures that we aren't using features that would be incompatible with WASM for Honk
-RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && make -j$(nproc) honk_tests barretenberg.wasm
+RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && cmake --build . --parallel --target honk_tests --target barretenberg.wasm
 
 FROM alpine:3.17
 COPY --from=builder /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm

cpp/dockerfiles/Dockerfile.x86_64-linux-clang

@@ -9,15 +9,13 @@ RUN apk update \
         git \
         curl \
         perl
+
 WORKDIR /usr/src/barretenberg/cpp
+
 COPY . .
-# Only build binaries that are needed upstream.
-RUN mkdir build && cd build && cmake .. && make -j$(nproc) db_cli rollup_cli tx_factory keygen
+# Build the entire project, as we want to check everything builds under clang
+RUN mkdir build && cd build && cmake .. && cmake --build . --parallel
 
 FROM alpine:3.17
 RUN apk update && apk add openmp
 COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db
-COPY --from=builder /usr/src/barretenberg/cpp/build/bin/db_cli /usr/src/barretenberg/cpp/build/bin/db_cli
-COPY --from=builder /usr/src/barretenberg/cpp/build/bin/rollup_cli /usr/src/barretenberg/cpp/build/bin/rollup_cli
-COPY --from=builder /usr/src/barretenberg/cpp/build/bin/tx_factory /usr/src/barretenberg/cpp/build/bin/tx_factory
-COPY --from=builder /usr/src/barretenberg/cpp/build/bin/keygen /usr/src/barretenberg/cpp/build/bin/keygen

cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert

@@ -14,7 +14,7 @@ WORKDIR /usr/src/barretenberg/cpp
 
 COPY . .
 # Build everything to ensure everything builds. All tests will be run from the result of this build.
-RUN mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=RelWithAssert -DCI=ON .. && make -j$(nproc)
+RUN mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=RelWithAssert -DCI=ON .. && cmake --build . --parallel
 
 FROM alpine:3.17
 RUN apk update && apk add curl openmp

cpp/dockerfiles/Dockerfile.x86_64-linux-gcc

@@ -8,10 +8,9 @@ RUN apk update \
         curl
 WORKDIR /usr/src/barretenberg/cpp
 COPY . .
-# Build the entire project (not just rollup_cli and db_cli), as we want to check everything builds under gcc.
-RUN mkdir build && cd build && cmake -DTOOLCHAIN=x86_64-linux-gcc -DCI=ON .. && make -j$(nproc)
+# Build the entire project, as we want to check everything builds under gcc.
+RUN mkdir build && cd build && cmake -DTOOLCHAIN=x86_64-linux-gcc -DCI=ON .. && cmake --build . --parallel
 
 FROM alpine:3.17
 RUN apk update && apk add libstdc++ libgomp
-COPY --from=builder /usr/src/barretenberg/cpp/build/bin/db_cli /usr/src/barretenberg/cpp/build/bin/db_cli
-COPY --from=builder /usr/src/barretenberg/cpp/build/bin/rollup_cli /usr/src/barretenberg/cpp/build/bin/rollup_cli
+COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db

cpp/docs/Fuzzing.md

@@ -82,7 +82,7 @@ Build with coverage instrumentation:
 mkdir build-coverage/
 cd build-coverage/
 cmake -DFUZZING=ON -DCMAKE_CXX_FLAGS="-fprofile-instr-generate -fcoverage-mapping" ..
-make -j$(nproc)
+cmake --build . --parallel
 ```
 
 Then run the fuzzer on the corpus and generate the HTML coverage reports: