feat: unify all acir recursion constraints based on RecursionConstrai…

…nt and proof_type (#7993)

This PR moves us towards a model where the type of recursive verifier
(plonk/honk, eventually IVC) can be specified from noir by setting a
`proof_type` constant input to `std::verify_proof()`. The new mechanism
has been integrated into the `verify_honk_proof` test program but not
yet into the protocol circuits. (The method defaults to Plonk recursion
so this is not a breaking change).

Other changes/updates:
- All types of recursion are specified through `RecursionConstraint` (no
more `HonkRecursionConstraint`)
- Move handling of recursion_constraints and honk_recursion_constraints
into individual methods `process_plonk/honk_recursion_constraints()` in
acir_format for greater clarity. (More cleanup along these lines to
come)
- Move the gate count tracking functionality to a class `GateCounter`

---------

Co-authored-by: sirasistant <sirasistant@gmail.com>
Co-authored-by: Maxim Vezenov <mvezenov@gmail.com>

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.hpp

@@ -94,7 +94,7 @@ struct AcirFormat {
     std::vector<MultiScalarMul> multi_scalar_mul_constraints;
     std::vector<EcAdd> ec_add_constraints;
     std::vector<RecursionConstraint> recursion_constraints;
-    std::vector<HonkRecursionConstraint> honk_recursion_constraints;
+    std::vector<RecursionConstraint> honk_recursion_constraints;
     std::vector<BigIntFromLeBytes> bigint_from_le_bytes_constraints;
     std::vector<BigIntToLeBytes> bigint_to_le_bytes_constraints;
     std::vector<BigIntOperation> bigint_operations;
@@ -204,4 +204,49 @@ void build_constraints(
                                             // circuit. This distinction is needed to not add the default
                                             // aggregation object when we're not using the honk RV.
 
+/**
+ * @brief Utility class for tracking the gate count of acir constraints
+ *
+ */
+template <typename Builder> class GateCounter {
+  public:
+    GateCounter(Builder* builder, bool collect_gates_per_opcode)
+        : builder(builder)
+        , collect_gates_per_opcode(collect_gates_per_opcode)
+    {}
+
+    size_t compute_diff()
+    {
+        if (!collect_gates_per_opcode) {
+            return 0;
+        }
+        size_t new_gate_count = builder->get_num_gates();
+        size_t diff = new_gate_count - prev_gate_count;
+        prev_gate_count = new_gate_count;
+        return diff;
+    }
+
+    void track_diff(std::vector<size_t>& gates_per_opcode, size_t opcode_index)
+    {
+        if (collect_gates_per_opcode) {
+            gates_per_opcode[opcode_index] = compute_diff();
+        }
+    }
+
+  private:
+    Builder* builder;
+    bool collect_gates_per_opcode;
+    size_t prev_gate_count{};
+};
+
+void process_plonk_recursion_constraints(Builder& builder,
+                                         AcirFormat& constraint_system,
+                                         bool has_valid_witness_assignments,
+                                         GateCounter<Builder>& gate_counter);
+void process_honk_recursion_constraints(Builder& builder,
+                                        AcirFormat& constraint_system,
+                                        bool has_valid_witness_assignments,
+                                        bool honk_recursion,
+                                        GateCounter<Builder>& gate_counter);
+
 } // namespace acir_format

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_integration.test.cpp

@@ -543,4 +543,26 @@ TEST_F(AcirIntegrationTest, DISABLED_UpdateAcirCircuit)
     EXPECT_TRUE(prove_and_verify_honk<Flavor>(circuit));
 }
 
+/**
+ * @brief Test recursive honk recursive verification
+ *
+ */
+TEST_F(AcirIntegrationTest, DISABLED_HonkRecursion)
+{
+    using Flavor = UltraFlavor;
+    using Builder = Flavor::CircuitBuilder;
+
+    std::string test_name = "verify_honk_proof"; // arbitrary program with RAM gates
+    // Note: honk_recursion set to false here because the selection of the honk recursive verifier is indicated by the
+    // proof_type field of the constraint generated from noir.
+    auto acir_program = get_program_data_from_test_file(test_name,
+                                                        /*honk_recursion=*/false);
+
+    // Construct a bberg circuit from the acir representation
+    auto circuit = acir_format::create_circuit<Builder>(acir_program.constraints, 0, acir_program.witness);
+
+    EXPECT_TRUE(CircuitChecker::check(circuit));
+    EXPECT_TRUE(prove_and_verify_honk<Flavor>(circuit));
+}
+
 #endif

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_to_constraint_buf.cpp

@@ -415,27 +415,34 @@ void handle_blackbox_func_call(Program::Opcode::BlackBoxFuncCall const& arg,
                 });
                 af.original_opcode_indices.keccak_permutations.push_back(opcode_index);
             } else if constexpr (std::is_same_v<T, Program::BlackBoxFuncCall::RecursiveAggregation>) {
-                if (honk_recursion) { // if we're using the honk recursive verifier
-                    auto c = HonkRecursionConstraint{
-                        .key = map(arg.verification_key, [](auto& e) { return get_witness_from_function_input(e); }),
-                        .proof = map(arg.proof, [](auto& e) { return get_witness_from_function_input(e); }),
-                        .public_inputs =
-                            map(arg.public_inputs, [](auto& e) { return get_witness_from_function_input(e); }),
-                    };
+
+                auto input_key = get_witness_from_function_input(arg.key_hash);
+
+                auto proof_type_in = arg.proof_type;
+                // TODO(https://github.com/AztecProtocol/barretenberg/issues/1074): Eventually arg.proof_type will be
+                // the only means for setting the proof type. use of honk_recursion flag in this context can go away
+                // once all noir programs (e.g. protocol circuits) are updated to use the new pattern.
+                if (honk_recursion && proof_type_in != HONK_RECURSION) {
+                    proof_type_in = HONK_RECURSION;
+                }
+
+                auto c = RecursionConstraint{
+                    .key = map(arg.verification_key, [](auto& e) { return get_witness_from_function_input(e); }),
+                    .proof = map(arg.proof, [](auto& e) { return get_witness_from_function_input(e); }),
+                    .public_inputs = map(arg.public_inputs, [](auto& e) { return get_witness_from_function_input(e); }),
+                    .key_hash = input_key,
+                    .proof_type = proof_type_in,
+                };
+                // Add the recursion constraint to the appropriate container based on proof type
+                if (c.proof_type == PLONK_RECURSION) {
+                    af.recursion_constraints.push_back(c);
+                    af.original_opcode_indices.recursion_constraints.push_back(opcode_index);
+                } else if (c.proof_type == HONK_RECURSION) {
                     af.honk_recursion_constraints.push_back(c);
                     af.original_opcode_indices.honk_recursion_constraints.push_back(opcode_index);
                 } else {
-                    auto input_key = get_witness_from_function_input(arg.key_hash);
-
-                    auto c = RecursionConstraint{
-                        .key = map(arg.verification_key, [](auto& e) { return get_witness_from_function_input(e); }),
-                        .proof = map(arg.proof, [](auto& e) { return get_witness_from_function_input(e); }),
-                        .public_inputs =
-                            map(arg.public_inputs, [](auto& e) { return get_witness_from_function_input(e); }),
-                        .key_hash = input_key,
-                    };
-                    af.recursion_constraints.push_back(c);
-                    af.original_opcode_indices.recursion_constraints.push_back(opcode_index);
+                    info("Invalid PROOF_TYPE in RecursionConstraint!");
+                    ASSERT(false);
                 }
             } else if constexpr (std::is_same_v<T, Program::BlackBoxFuncCall::BigIntFromLeBytes>) {
                 af.bigint_from_le_bytes_constraints.push_back(BigIntFromLeBytes{

barretenberg/cpp/src/barretenberg/dsl/acir_format/honk_recursion_constraint.cpp

@@ -27,7 +27,7 @@ using aggregation_state_ct = bb::stdlib::recursion::aggregation_state<bn254>;
  * @param proof_fields
  */
 void create_dummy_vkey_and_proof(Builder& builder,
-                                 const HonkRecursionConstraint& input,
+                                 const RecursionConstraint& input,
                                  std::vector<field_ct>& key_fields,
                                  std::vector<field_ct>& proof_fields)
 {
@@ -36,16 +36,15 @@ void create_dummy_vkey_and_proof(Builder& builder,
     // Set vkey->circuit_size correctly based on the proof size
     size_t num_frs_comm = bb::field_conversion::calc_num_bn254_frs<UltraFlavor::Commitment>();
     size_t num_frs_fr = bb::field_conversion::calc_num_bn254_frs<UltraFlavor::FF>();
-    assert((input.proof.size() - HonkRecursionConstraint::inner_public_input_offset -
-            UltraFlavor::NUM_WITNESS_ENTITIES * num_frs_comm - UltraFlavor::NUM_ALL_ENTITIES * num_frs_fr -
-            2 * num_frs_comm) %
+    assert((input.proof.size() - HONK_RECURSION_PUBLIC_INPUT_OFFSET - UltraFlavor::NUM_WITNESS_ENTITIES * num_frs_comm -
+            UltraFlavor::NUM_ALL_ENTITIES * num_frs_fr - 2 * num_frs_comm) %
                (num_frs_comm + num_frs_fr * UltraFlavor::BATCHED_RELATION_PARTIAL_LENGTH) ==
            0);
     // Note: this computation should always result in log_circuit_size = CONST_PROOF_SIZE_LOG_N
-    auto log_circuit_size = (input.proof.size() - HonkRecursionConstraint::inner_public_input_offset -
-                             UltraFlavor::NUM_WITNESS_ENTITIES * num_frs_comm -
-                             UltraFlavor::NUM_ALL_ENTITIES * num_frs_fr - 2 * num_frs_comm) /
-                            (num_frs_comm + num_frs_fr * UltraFlavor::BATCHED_RELATION_PARTIAL_LENGTH);
+    auto log_circuit_size =
+        (input.proof.size() - HONK_RECURSION_PUBLIC_INPUT_OFFSET - UltraFlavor::NUM_WITNESS_ENTITIES * num_frs_comm -
+         UltraFlavor::NUM_ALL_ENTITIES * num_frs_fr - 2 * num_frs_comm) /
+        (num_frs_comm + num_frs_fr * UltraFlavor::BATCHED_RELATION_PARTIAL_LENGTH);
     // First key field is circuit size
     builder.assert_equal(builder.add_variable(1 << log_circuit_size), key_fields[0].witness_index);
     // Second key field is number of public inputs
@@ -73,7 +72,7 @@ void create_dummy_vkey_and_proof(Builder& builder,
         offset += 4;
     }
 
-    offset = HonkRecursionConstraint::inner_public_input_offset;
+    offset = HONK_RECURSION_PUBLIC_INPUT_OFFSET;
     // first 3 things
     builder.assert_equal(builder.add_variable(1 << log_circuit_size), proof_fields[0].witness_index);
     builder.assert_equal(builder.add_variable(input.public_inputs.size()), proof_fields[1].witness_index);
@@ -151,14 +150,16 @@ void create_dummy_vkey_and_proof(Builder& builder,
  *       or we need non-witness data to be provided as metadata in the ACIR opcode
  */
 AggregationObjectIndices create_honk_recursion_constraints(Builder& builder,
-                                                           const HonkRecursionConstraint& input,
+                                                           const RecursionConstraint& input,
                                                            AggregationObjectIndices input_aggregation_object_indices,
                                                            bool has_valid_witness_assignments)
 {
     using Flavor = UltraRecursiveFlavor_<Builder>;
     using RecursiveVerificationKey = Flavor::VerificationKey;
     using RecursiveVerifier = bb::stdlib::recursion::honk::UltraRecursiveVerifier_<Flavor>;
 
+    ASSERT(input.proof_type == HONK_RECURSION);
+
     // Construct an in-circuit representation of the verification key.
     // For now, the v-key is a circuit constant and is fixed for the circuit.
     // (We may need a separate recursion opcode for this to vary, or add more config witnesses to this opcode)
@@ -179,7 +180,7 @@ AggregationObjectIndices create_honk_recursion_constraints(Builder& builder,
         auto field = field_ct::from_witness_index(&builder, idx);
         proof_fields.emplace_back(field);
         i++;
-        if (i == HonkRecursionConstraint::inner_public_input_offset) {
+        if (i == HONK_RECURSION_PUBLIC_INPUT_OFFSET) {
             for (const auto& idx : input.public_inputs) {
                 auto field = field_ct::from_witness_index(&builder, idx);
                 proof_fields.emplace_back(field);

barretenberg/cpp/src/barretenberg/dsl/acir_format/honk_recursion_constraint.hpp

@@ -1,4 +1,5 @@
 #pragma once
+#include "barretenberg/dsl/acir_format/recursion_constraint.hpp"
 #include "barretenberg/stdlib/primitives/bigfield/bigfield.hpp"
 #include <vector>
 
@@ -7,55 +8,12 @@ using Builder = bb::UltraCircuitBuilder;
 
 using namespace bb;
 
-/**
- * @brief HonkRecursionConstraint struct contains information required to recursively verify a proof!
- *
- * @details The recursive verifier algorithm produces an 'aggregation object' representing 2 G1 points, expressed as 16
- * witness values. The smart contract Verifier must be aware of this aggregation object in order to complete the full
- * recursive verification. If the circuit verifies more than 1 proof, the recursion algorithm will update a pre-existing
- * aggregation object (`input_aggregation_object`).
- *
- * @details We currently require that the inner circuit being verified only has a single public input. If more are
- * required, the outer circuit can hash them down to 1 input.
- *
- * @param verification_key_data The inner circuit vkey. Is converted into circuit witness values (internal to the
- * backend)
- * @param proof The honk proof. Is converted into circuit witness values (internal to the backend)
- * @param is_aggregation_object_nonzero A flag to tell us whether the circuit has already recursively verified proofs
- * (and therefore an aggregation object is present)
- * @param public_input The index of the single public input
- * @param input_aggregation_object Witness indices of pre-existing aggregation object (if it exists)
- * @param output_aggregation_object Witness indices of the aggregation object produced by recursive verification
- * @param nested_aggregation_object Public input indices of an aggregation object inside the proof.
- *
- * @note If input_aggregation_object witness indices are all zero, we interpret this to mean that the inner proof does
- * NOT contain a previously recursively verified proof
- * @note nested_aggregation_object is used for cases where the proof being verified contains an aggregation object in
- * its public inputs! If this is the case, we record the public input locations in `nested_aggregation_object`. If the
- * inner proof is of a circuit that does not have a nested aggregation object, these values are all zero.
- *
- * To outline the interaction between the input_aggergation_object and the nested_aggregation_object take the following
- * example: If we have a circuit that verifies 2 proofs A and B, the recursion constraint for B will have an
- * input_aggregation_object that points to the aggregation output produced by verifying A. If circuit B also verifies a
- * proof, in the above example the recursion constraint for verifying B will have a nested object that describes the
- * aggregation object in B’s public inputs as well as an input aggregation object that points to the object produced by
- * the previous recursion constraint in the circuit (the one that verifies A)
- *
- * TODO(https://github.com/AztecProtocol/barretenberg/issues/996): Update these comments for Honk.
- */
-struct HonkRecursionConstraint {
-    // In Honk, the proof starts with circuit_size, num_public_inputs, and pub_input_offset. We use this offset to keep
-    // track of where the public inputs start.
-    static constexpr size_t inner_public_input_offset = 3;
-    std::vector<uint32_t> key;
-    std::vector<uint32_t> proof;
-    std::vector<uint32_t> public_inputs;
-
-    friend bool operator==(HonkRecursionConstraint const& lhs, HonkRecursionConstraint const& rhs) = default;
-};
+// In Honk, the proof starts with circuit_size, num_public_inputs, and pub_input_offset. We use this offset to keep
+// track of where the public inputs start.
+static constexpr size_t HONK_RECURSION_PUBLIC_INPUT_OFFSET = 3;
 
 AggregationObjectIndices create_honk_recursion_constraints(Builder& builder,
-                                                           const HonkRecursionConstraint& input,
+                                                           const RecursionConstraint& input,
                                                            AggregationObjectIndices input_aggregation_object,
                                                            bool has_valid_witness_assignments = false);
 

barretenberg/cpp/src/barretenberg/dsl/acir_format/honk_recursion_constraint.test.cpp

@@ -138,7 +138,7 @@ class AcirHonkRecursionConstraint : public ::testing::Test {
      */
     Builder create_outer_circuit(std::vector<Builder>& inner_circuits)
     {
-        std::vector<HonkRecursionConstraint> honk_recursion_constraints;
+        std::vector<RecursionConstraint> honk_recursion_constraints;
 
         size_t witness_offset = 0;
         std::vector<fr, ContainerSlabAllocator<fr>> witness;
@@ -155,7 +155,7 @@ class AcirHonkRecursionConstraint : public ::testing::Test {
 
             std::vector<fr> proof_witnesses = inner_proof;
             // where the inner public inputs start (after circuit_size, num_pub_inputs, pub_input_offset)
-            const size_t inner_public_input_offset = 3;
+            const size_t inner_public_input_offset = HONK_RECURSION_PUBLIC_INPUT_OFFSET;
             // - Save the public inputs so that we can set their values.
             // - Then truncate them from the proof because the ACIR API expects proofs without public inputs
             std::vector<fr> inner_public_input_values(
@@ -206,10 +206,12 @@ class AcirHonkRecursionConstraint : public ::testing::Test {
                 inner_public_inputs.push_back(static_cast<uint32_t>(i + public_input_start_idx));
             }
 
-            HonkRecursionConstraint honk_recursion_constraint{
+            RecursionConstraint honk_recursion_constraint{
                 .key = key_indices,
                 .proof = proof_indices,
                 .public_inputs = inner_public_inputs,
+                .key_hash = 0, // not used
+                .proof_type = HONK_RECURSION,
             };
             honk_recursion_constraints.push_back(honk_recursion_constraint);
 

barretenberg/cpp/src/barretenberg/dsl/acir_format/recursion_constraint.hpp

@@ -6,6 +6,10 @@
 
 namespace acir_format {
 
+// Used to specify the type of recursive verifier via the proof_type specified by the RecursiveAggregation opcode from
+// ACIR
+enum PROOF_TYPE { PLONK_RECURSION, HONK_RECURSION };
+
 using namespace bb::plonk;
 using Builder = bb::UltraCircuitBuilder;
 
@@ -43,6 +47,7 @@ using Builder = bb::UltraCircuitBuilder;
  * aggregation object in B’s public inputs as well as an input aggregation object that points to the object produced by
  * the previous recursion constraint in the circuit (the one that verifies A)
  *
+ * TODO(https://github.com/AztecProtocol/barretenberg/issues/996): Create similar comments for Honk.
  */
 struct RecursionConstraint {
     // An aggregation state is represented by two G1 affine elements. Each G1 point has
@@ -52,6 +57,7 @@ struct RecursionConstraint {
     std::vector<uint32_t> proof;
     std::vector<uint32_t> public_inputs;
     uint32_t key_hash;
+    uint32_t proof_type;
 
     friend bool operator==(RecursionConstraint const& lhs, RecursionConstraint const& rhs) = default;
 };

barretenberg/cpp/src/barretenberg/dsl/acir_format/recursion_constraint.test.cpp

@@ -209,6 +209,7 @@ Builder create_outer_circuit(std::vector<Builder>& inner_circuits)
             .proof = proof_indices,
             .public_inputs = inner_public_inputs,
             .key_hash = key_hash_start_idx,
+            .proof_type = PLONK_RECURSION,
         };
         recursion_constraints.push_back(recursion_constraint);