feat: Encapsulated UltraHonk Vanilla IVC (#10900)

This adds a class that does IVC proving via recursion for the UltraHonk
proof system.

barretenberg/cpp/src/CMakeLists.txt

@@ -61,10 +61,10 @@ if (ENABLE_PIC AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   add_subdirectory(barretenberg/world_state_napi)
 endif()
 
-add_subdirectory(barretenberg/client_ivc)
 add_subdirectory(barretenberg/bb)
 add_subdirectory(barretenberg/boomerang_value_detection)
 add_subdirectory(barretenberg/circuit_checker)
+add_subdirectory(barretenberg/client_ivc)
 add_subdirectory(barretenberg/commitment_schemes)
 add_subdirectory(barretenberg/commitment_schemes_recursion)
 add_subdirectory(barretenberg/common)
@@ -87,6 +87,7 @@ add_subdirectory(barretenberg/relations)
 add_subdirectory(barretenberg/serialize)
 add_subdirectory(barretenberg/solidity_helpers)
 add_subdirectory(barretenberg/srs)
+add_subdirectory(barretenberg/ultra_vanilla_client_ivc)
 add_subdirectory(barretenberg/stdlib)
 add_subdirectory(barretenberg/stdlib_circuit_builders)
 add_subdirectory(barretenberg/sumcheck)
@@ -139,6 +140,7 @@ set(BARRETENBERG_TARGET_OBJECTS
     $<TARGET_OBJECTS:protogalaxy_objects>
     $<TARGET_OBJECTS:relations_objects>
     $<TARGET_OBJECTS:srs_objects>
+    $<TARGET_OBJECTS:ultra_vanilla_client_ivc_objects>
     $<TARGET_OBJECTS:stdlib_aes128_objects>
     $<TARGET_OBJECTS:stdlib_blake2s_objects>
     $<TARGET_OBJECTS:stdlib_blake3s_objects>

barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp

@@ -55,6 +55,7 @@ template <class Curve> class CommitmentKey {
     scalar_multiplication::pippenger_runtime_state<Curve> pippenger_runtime_state;
     std::shared_ptr<srs::factories::CrsFactory<Curve>> crs_factory;
     std::shared_ptr<srs::factories::ProverCrs<Curve>> srs;
+    size_t dyadic_size;
 
     CommitmentKey() = delete;
 
@@ -69,6 +70,7 @@ template <class Curve> class CommitmentKey {
         : pippenger_runtime_state(get_num_needed_srs_points(num_points))
         , crs_factory(srs::get_crs_factory<Curve>())
         , srs(crs_factory->get_prover_crs(get_num_needed_srs_points(num_points)))
+        , dyadic_size(get_num_needed_srs_points(num_points))
     {}
 
     // Note: This constructor is to be used only by Plonk; For Honk the srs lives in the CommitmentKey

barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.test.cpp

@@ -44,6 +44,8 @@ template <typename RecursiveFlavor> class RecursiveVerifierTest : public testing
     using RecursiveVerifier = UltraRecursiveVerifier_<RecursiveFlavor>;
     using VerificationKey = typename RecursiveVerifier::VerificationKey;
 
+    using AggState = aggregation_state<typename RecursiveFlavor::Curve>;
+    using VerifierOutput = bb::stdlib::recursion::honk::UltraRecursiveVerifierOutput<RecursiveFlavor>;
     /**
      * @brief Create a non-trivial arbitrary inner circuit, the proof of which will be recursively verified
      *
@@ -252,11 +254,9 @@ template <typename RecursiveFlavor> class RecursiveVerifierTest : public testing
         OuterBuilder outer_circuit;
         RecursiveVerifier verifier{ &outer_circuit, verification_key };
 
-        aggregation_state<typename RecursiveFlavor::Curve> agg_obj =
-            init_default_aggregation_state<OuterBuilder, typename RecursiveFlavor::Curve>(outer_circuit);
-        bb::stdlib::recursion::honk::UltraRecursiveVerifierOutput<RecursiveFlavor> output =
-            verifier.verify_proof(inner_proof, agg_obj);
-        aggregation_state<typename RecursiveFlavor::Curve> pairing_points = output.agg_obj;
+        AggState agg_obj = init_default_aggregation_state<OuterBuilder, typename RecursiveFlavor::Curve>(outer_circuit);
+        VerifierOutput output = verifier.verify_proof(inner_proof, agg_obj);
+        AggState pairing_points = output.agg_obj;
         if constexpr (HasIPAAccumulator<OuterFlavor>) {
             outer_circuit.add_ipa_claim(output.ipa_opening_claim.get_witness_indices());
             outer_circuit.ipa_proof = convert_stdlib_proof_to_native(output.ipa_proof);

barretenberg/cpp/src/barretenberg/ultra_honk/ultra_prover.cpp

@@ -4,6 +4,14 @@
 #include "barretenberg/ultra_honk/oink_prover.hpp"
 namespace bb {
 
+template <IsUltraFlavor Flavor>
+UltraProver_<Flavor>::UltraProver_(const std::shared_ptr<DeciderPK>& proving_key,
+                                   const std::shared_ptr<CommitmentKey>& commitment_key)
+    : proving_key(std::move(proving_key))
+    , transcript(std::make_shared<Transcript>())
+    , commitment_key(commitment_key)
+{}
+
 /**
  * @brief Create UltraProver_ from a decider proving key.
  *

barretenberg/cpp/src/barretenberg/ultra_honk/ultra_prover.hpp

@@ -37,6 +37,8 @@ template <IsUltraFlavor Flavor_> class UltraProver_ {
 
     std::shared_ptr<CommitmentKey> commitment_key;
 
+    UltraProver_(const std::shared_ptr<DeciderPK>&, const std::shared_ptr<CommitmentKey>&);
+
     explicit UltraProver_(const std::shared_ptr<DeciderPK>&,
                           const std::shared_ptr<Transcript>& transcript = std::make_shared<Transcript>());
 

barretenberg/cpp/src/barretenberg/ultra_vanilla_client_ivc/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(ultra_vanilla_client_ivc stdlib_honk_verifier)

barretenberg/cpp/src/barretenberg/ultra_vanilla_client_ivc/mock_circuit_producer.hpp

@@ -0,0 +1,90 @@
+
+#include "barretenberg/common/op_count.hpp"
+#include "barretenberg/goblin/mock_circuits.hpp"
+#include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
+#include "barretenberg/ultra_honk/ultra_verifier.hpp"
+#include "barretenberg/ultra_vanilla_client_ivc/ultra_vanilla_client_ivc.hpp"
+
+using namespace bb;
+
+namespace {
+
+/**
+ * @brief Manage the construction of mock app/kernel circuits for the private function execution setting
+ * @details Per the medium complexity benchmark spec, the first app circuit is size 2^19. Subsequent app and kernel
+ * circuits are size 2^17. Circuits produced are alternatingly app and kernel. Mock databus data is passed between the
+ * circuits in a manor conistent with the real architecture in order to facilitate testing of databus consistency
+ * checks.
+ */
+class PrivateFunctionExecutionMockCircuitProducer {
+    using ClientCircuit = UltraVanillaClientIVC::ClientCircuit;
+    using Flavor = MegaFlavor;
+    using VerificationKey = Flavor::VerificationKey;
+
+    size_t circuit_counter = 0;
+
+    bool large_first_app = true; // if true, first app is 2^19, else 2^17
+
+  public:
+    PrivateFunctionExecutionMockCircuitProducer(bool large_first_app = true)
+        : large_first_app(large_first_app)
+    {}
+
+    /**
+     * @brief Create the next circuit (app/kernel) in a mocked private function execution stack
+     */
+    ClientCircuit create_next_circuit(UltraVanillaClientIVC& ivc, bool force_is_kernel = false)
+    {
+        circuit_counter++;
+
+        // Assume only every second circuit is a kernel, unless force_is_kernel == true
+        bool is_kernel = (circuit_counter % 2 == 0) || force_is_kernel;
+
+        ClientCircuit circuit{ ivc.goblin.op_queue };
+        if (is_kernel) {
+            GoblinMockCircuits::construct_mock_folding_kernel(circuit); // construct mock base logic
+            mock_databus.populate_kernel_databus(circuit);              // populate databus inputs/outputs
+            ivc.complete_kernel_circuit_logic(circuit);                 // complete with recursive verifiers etc
+        } else {
+            bool use_large_circuit = large_first_app && (circuit_counter == 1);         // first circuit is size 2^19
+            GoblinMockCircuits::construct_mock_app_circuit(circuit, use_large_circuit); // construct mock app
+            mock_databus.populate_app_databus(circuit);                                 // populate databus outputs
+        }
+        return circuit;
+    }
+
+    /**
+     * @brief Tamper with databus data to facilitate failure testing
+     */
+    void tamper_with_databus() { mock_databus.tamper_with_app_return_data(); }
+
+    /**
+     * @brief Compute and return the verification keys for a mocked private function execution IVC
+     * @details For testing/benchmarking only. This method is robust at the cost of being extremely inefficient. It
+     * simply executes a full IVC for a given number of circuits and stores the verification keys along the way. (In
+     * practice these VKs will be known to a client prover in advance).
+     *
+     * @param num_circuits
+     * @param trace_structure Trace structuring must be known in advance because it effects the VKs
+     * @return set of num_circuits-many verification keys
+     */
+    auto precompute_verification_keys(const size_t num_circuits, TraceSettings trace_settings)
+    {
+        UltraVanillaClientIVC ivc{
+            trace_settings
+        }; // temporary IVC instance needed to produce the complete kernel circuits
+
+        std::vector<std::shared_ptr<VerificationKey>> vkeys;
+
+        for (size_t idx = 0; idx < num_circuits; ++idx) {
+            ClientCircuit circuit = create_next_circuit(ivc); // create the next circuit
+            ivc.accumulate(circuit);                          // accumulate the circuit
+            vkeys.emplace_back(ivc.honk_vk);                  // save the VK for the circuit
+        }
+        circuit_counter = 0; // reset the internal circuit counter back to 0
+
+        return vkeys;
+    }
+};
+
+} // namespace

barretenberg/cpp/src/barretenberg/ultra_vanilla_client_ivc/ultra_vanilla_client_ivc.cpp

@@ -0,0 +1,91 @@
+#include "barretenberg/ultra_vanilla_client_ivc/ultra_vanilla_client_ivc.hpp"
+#include "barretenberg/ultra_honk/oink_prover.hpp"
+
+namespace bb {
+
+void UltraVanillaClientIVC::accumulate(Circuit& circuit, const Proof& proof, const std::shared_ptr<VK>& vk)
+{
+    RecursiveVerifier verifier{ &circuit, std::make_shared<RecursiveVK>(&circuit, vk) };
+    Accumulator agg_obj = stdlib::recursion::init_default_aggregation_state<Circuit, stdlib::bn254<Circuit>>(circuit);
+    accumulator = verifier.verify_proof(proof, agg_obj).agg_obj;
+}
+
+HonkProof UltraVanillaClientIVC::prove(CircuitSource<Flavor>& source, const bool cache_vks)
+{
+    for (size_t step = 0; step < source.num_circuits(); step++) {
+        auto [circuit, vk] = source.next();
+        if (step == 0) {
+            accumulator_indices = stdlib::recursion::init_default_agg_obj_indices(circuit);
+        } else {
+            accumulate(circuit, previous_proof, previous_vk);
+            accumulator_indices = accumulator.get_witness_indices();
+        }
+
+        circuit.add_pairing_point_accumulator(accumulator_indices);
+        accumulator_value = { accumulator.P0.get_value(), accumulator.P1.get_value() };
+
+        auto proving_key = std::make_shared<PK>(circuit);
+
+        if (step < source.num_circuits() - 1) {
+            UltraProver prover{ proving_key, commitment_key };
+            previous_proof = prover.construct_proof();
+        } else {
+            // TODO(https://github.com/AztecProtocol/barretenberg/issues/1176) Use UltraZKProver when it exists
+            UltraProver prover{ proving_key, commitment_key };
+            previous_proof = prover.construct_proof();
+        }
+
+        previous_vk = vk ? vk : std::make_shared<VK>(proving_key->proving_key);
+        if (cache_vks) {
+            vk_cache.push_back(previous_vk);
+        }
+    }
+    return previous_proof;
+};
+
+bool UltraVanillaClientIVC::verify(const Proof& proof, const std::shared_ptr<VK>& vk)
+{
+
+    UltraVerifier verifer{ vk };
+    bool verified = verifer.verify_proof(proof);
+    vinfo("proof verified: ", verified);
+
+    using VerifierCommitmentKey = typename Flavor::VerifierCommitmentKey;
+    auto pcs_verification_key = std::make_shared<VerifierCommitmentKey>();
+    verified &= pcs_verification_key->pairing_check(accumulator_value[0], accumulator_value[1]);
+    vinfo("pairing verified: ", verified);
+    return verified;
+}
+
+/**
+ * @brief Construct and verify a proof for the IVC
+ * @note Use of this method only makes sense when the prover and verifier are the same entity, e.g. in
+ * development/testing.
+ *
+ */
+bool UltraVanillaClientIVC::prove_and_verify(CircuitSource<Flavor>& source, const bool cache_vks)
+{
+    auto start = std::chrono::steady_clock::now();
+    prove(source, cache_vks);
+    auto end = std::chrono::steady_clock::now();
+    auto diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
+    vinfo("time to call UltraVanillaClientIVC::prove: ", diff.count(), " ms.");
+
+    start = end;
+    bool verified = verify(previous_proof, previous_vk);
+    end = std::chrono::steady_clock::now();
+
+    diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
+    vinfo("time to verify UltraVanillaClientIVC proof: ", diff.count(), " ms.");
+
+    return verified;
+}
+
+std::vector<std::shared_ptr<UltraFlavor::VerificationKey>> UltraVanillaClientIVC::compute_vks(
+    CircuitSource<Flavor>& source)
+{
+    prove_and_verify(source, /*cache_vks=*/true);
+    return vk_cache;
+};
+
+} // namespace bb

barretenberg/cpp/src/barretenberg/ultra_vanilla_client_ivc/ultra_vanilla_client_ivc.hpp

@@ -0,0 +1,114 @@
+#pragma once
+
+#include "barretenberg/plonk_honk_shared/execution_trace/execution_trace_usage_tracker.hpp"
+#include "barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp"
+#include "barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.hpp"
+#include "barretenberg/stdlib/plonk_recursion/aggregation_state/aggregation_state.hpp"
+#include "barretenberg/ultra_honk/ultra_prover.hpp"
+#include "barretenberg/ultra_honk/ultra_verifier.hpp"
+#include <algorithm>
+namespace bb {
+
+/**
+ * @brief A function that produces a set of circuits and possibly their precomputed vks
+ * @details One has the option of not providing vks--just provide nullptr instead.
+ *    This class is introduced as an experiment. We _could_ just use vectors of vks and shared_ptrs, but this limits
+ * applicability of the class because, in practice, we don't have sufficient memory to store all circuit builders at
+ * once.  The idea is this class is applicable in both situations we care about testing via mocks (cf the test file for
+ * UltraVanillaClientIVC, which implements a source of mock circuits), and IVC of circuits written in Noir, where the
+ * source (not yet implemented) is ACIR and partial witnesses which are processed by our DSL code, expanding blackbox
+ * function calls.
+ * @todo Relocate this at the appropriate time, if it does become a standard interface.
+ */
+template <typename Flavor,
+          typename Builder = typename Flavor::CircuitBuilder,
+          typename VK = typename Flavor::VerificationKey>
+
+class CircuitSource {
+  public:
+    struct Output {
+        Builder circuit;
+        std::shared_ptr<VK> vk;
+    };
+
+    virtual Output next();
+    virtual size_t num_circuits() const;
+};
+
+/**
+ * @brief A class encapsulating multiple sequential steps of the IVC scheme that arises most naturally from recursive
+ * proof verification.
+ *
+ * @details "Vanilla" is in the colloquial sense of meaning "plain". "Client" refers to the fact that this is intended
+ * for executing proof construction in constrained environments.
+ */
+class UltraVanillaClientIVC {
+
+  public:
+    using Flavor = UltraFlavor;
+    using FF = Flavor::FF;
+    using Circuit = UltraCircuitBuilder;
+    using PK = DeciderProvingKey_<Flavor>;
+    using VK = UltraFlavor::VerificationKey;
+    using Proof = HonkProof;
+
+    using RecursiveFlavor = UltraRecursiveFlavor_<Circuit>;
+    using RecursiveVerifier = stdlib::recursion::honk::UltraRecursiveVerifier_<RecursiveFlavor>;
+    using RecursiveVK = RecursiveFlavor::VerificationKey;
+    using Curve = stdlib::bn254<Circuit>;
+    using Accumulator = stdlib::recursion::aggregation_state<Curve>;
+
+    /**
+     * @brief Append a recursive verifier and update the accumulator.
+     */
+    void accumulate(Circuit&, const Proof&, const std::shared_ptr<VK>&);
+
+  public:
+    std::shared_ptr<CommitmentKey<curve::BN254>> commitment_key;
+    Proof previous_proof;
+    std::shared_ptr<VK> previous_vk;
+    Accumulator accumulator;
+    std::array<curve::BN254::AffineElement, 2> accumulator_value;
+    PairingPointAccumulatorIndices accumulator_indices;
+    std::vector<std::shared_ptr<VK>> vk_cache;
+
+    UltraVanillaClientIVC(const size_t dyadic_size = 1 << 20)
+        : commitment_key(std::make_shared<CommitmentKey<curve::BN254>>(dyadic_size))
+    {}
+
+    /**
+     * @brief Iterate through all circuits and prove each, appending a recursive verifier of the previous proof after
+     * the first step.
+     * @param source A source of circuits, possibly accompanied by precomputed verification keys.
+     * @param cache_vks If true, case the verification key that is computed.
+     * @return HonkProof A proof of the final circuit which through recursive verification, demonstrates that all
+     * circuits were satisfied, or one of them was not satisfied, depending on whether it verifies or does not verify.
+     */
+    HonkProof prove(CircuitSource<Flavor>& source, const bool cache_vks = false);
+
+    /**
+     * @brief Verify an IVC proof.
+     * @details This verifies the final proof, including (natively) checking the pairing of the two points in the final
+     * accumulator.
+     *
+     * @param proof
+     * @param vk
+     * @return true All circuits provided have been satisfied.
+     * @return false Some circuit provided was not satisfied.
+     */
+    bool verify(const Proof& proof, const std::shared_ptr<VK>& vk);
+
+    /**
+     * @brief Prove and then verify the proof. This is used for testing.
+     */
+    bool prove_and_verify(CircuitSource<Flavor>& source, const bool cache_vks = false);
+
+    /**
+     * @brief Compute the verification key of each circuit provided by the source.
+     * @details This runs a full IVC prover. Our public interface provides a faster but more brittle method via dummy
+     * witnesses. This is a useful fallback that we might want for debugging. Currently it is used to test the prover
+     * flow that using precomputed verification keys.
+     */
+    std::vector<std::shared_ptr<VK>> compute_vks(CircuitSource<Flavor>& source);
+};
+} // namespace bb