feat: removed redundant scalar muls from the verifiers using shplemini (

#9392)

* Reduced the number of scalar multiplications to be performed by the
native and recursive verifiers running shplemini
* Slightly re-shuffled the entities in Translator and ECCVM, so that
entitied to be shifted and their shifts form contiguous ranges
* This is useful for amortizing the verification costs in the case of ZK
sumcheck
* The Translator recursive verifier circuit is now around 820K gates as
opposed to 1700K. For other Flavors, the numbers are not as dramatic,
but there's still around -10% in scalar muls and the sizes of recursive
verifiers.

barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp

@@ -330,3 +330,85 @@ TEST_F(IPATest, ShpleminiIPAWithShift)
 
     EXPECT_EQ(result, true);
 }
+/**
+ * @brief Test the behaviour of the method ShpleminiVerifier::remove_shifted_commitments
+ *
+ */
+TEST_F(IPATest, ShpleminiIPAShiftsRemoval)
+{
+    using IPA = IPA<Curve>;
+    using ShplonkProver = ShplonkProver_<Curve>;
+    using ShpleminiVerifier = ShpleminiVerifier_<Curve>;
+    using GeminiProver = GeminiProver_<Curve>;
+
+    const size_t n = 8;
+    const size_t log_n = 3;
+
+    // Generate multilinear polynomials, their commitments (genuine and mocked) and evaluations (genuine) at a random
+    // point.
+    auto mle_opening_point = this->random_evaluation_point(log_n); // sometimes denoted 'u'
+    auto poly1 = Polynomial::random(n);
+    auto poly2 = Polynomial::random(n, /*shiftable*/ 1);
+    auto poly3 = Polynomial::random(n, /*shiftable*/ 1);
+    auto poly4 = Polynomial::random(n);
+
+    Commitment commitment1 = this->commit(poly1);
+    Commitment commitment2 = this->commit(poly2);
+    Commitment commitment3 = this->commit(poly3);
+    Commitment commitment4 = this->commit(poly4);
+
+    std::vector<Commitment> unshifted_commitments = { commitment1, commitment2, commitment3, commitment4 };
+    std::vector<Commitment> shifted_commitments = { commitment2, commitment3 };
+    auto eval1 = poly1.evaluate_mle(mle_opening_point);
+    auto eval2 = poly2.evaluate_mle(mle_opening_point);
+    auto eval3 = poly3.evaluate_mle(mle_opening_point);
+    auto eval4 = poly4.evaluate_mle(mle_opening_point);
+
+    auto eval2_shift = poly2.evaluate_mle(mle_opening_point, true);
+    auto eval3_shift = poly3.evaluate_mle(mle_opening_point, true);
+
+    auto prover_transcript = NativeTranscript::prover_init_empty();
+
+    // Run the full prover PCS protocol:
+
+    // Compute:
+    // - (d+1) opening pairs: {r, \hat{a}_0}, {-r^{2^i}, a_i}, i = 0, ..., d-1
+    // - (d+1) Fold polynomials Fold_{r}^(0), Fold_{-r}^(0), and Fold^(i), i = 0, ..., d-1
+    auto prover_opening_claims = GeminiProver::prove(n,
+                                                     RefArray{ poly1, poly2, poly3, poly4 },
+                                                     RefArray{ poly2, poly3 },
+                                                     mle_opening_point,
+                                                     this->ck(),
+                                                     prover_transcript);
+
+    const auto opening_claim = ShplonkProver::prove(this->ck(), prover_opening_claims, prover_transcript);
+    IPA::compute_opening_proof(this->ck(), opening_claim, prover_transcript);
+
+    // the index of the first commitment to a polynomial to be shifted in the union of unshifted_commitments and
+    // shifted_commitments. in our case, it is poly2
+    const size_t to_be_shifted_commitments_start = 1;
+    // the index of the first commitment to a shifted polynomial in the union of unshifted_commitments and
+    // shifted_commitments. in our case, it is the second occurence of poly2
+    const size_t shifted_commitments_start = 4;
+    // number of shifted polynomials
+    const size_t num_shifted_commitments = 2;
+    const RepeatedCommitmentsData repeated_commitments =
+        RepeatedCommitmentsData(to_be_shifted_commitments_start, shifted_commitments_start, num_shifted_commitments);
+    // since commitments to poly2, poly3 and their shifts are the same group elements, we simply combine the scalar
+    // multipliers of commitment2 and commitment3 in one place and remove the entries of the commitments and scalars
+    // vectors corresponding to the "shifted" commitment
+    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+
+    auto batch_opening_claim = ShpleminiVerifier::compute_batch_opening_claim(n,
+                                                                              RefVector(unshifted_commitments),
+                                                                              RefVector(shifted_commitments),
+                                                                              RefArray{ eval1, eval2, eval3, eval4 },
+                                                                              RefArray{ eval2_shift, eval3_shift },
+                                                                              mle_opening_point,
+                                                                              this->vk()->get_g1_identity(),
+                                                                              verifier_transcript,
+                                                                              repeated_commitments);
+
+    auto result = IPA::reduce_verify_batch_opening_claim(batch_opening_claim, this->vk(), verifier_transcript);
+    EXPECT_EQ(result, true);
+}

barretenberg/cpp/src/barretenberg/commitment_schemes/kzg/kzg.test.cpp

@@ -318,6 +318,7 @@ TYPED_TEST(KZGTest, ShpleminiKzgWithShiftAndConcatenation)
                                                        mle_opening_point,
                                                        this->vk()->get_g1_identity(),
                                                        verifier_transcript,
+                                                       {},
                                                        /* libra commitments = */ {},
                                                        /* libra evaluations = */ {},
                                                        to_vector_of_ref_vectors(concatenation_groups_commitments),
@@ -327,5 +328,99 @@ TYPED_TEST(KZGTest, ShpleminiKzgWithShiftAndConcatenation)
 
     EXPECT_EQ(this->vk()->pairing_check(pairing_points[0], pairing_points[1]), true);
 }
+TYPED_TEST(KZGTest, ShpleminiKzgShiftsRemoval)
+{
+    using ShplonkProver = ShplonkProver_<TypeParam>;
+    using GeminiProver = GeminiProver_<TypeParam>;
+    using ShpleminiVerifier = ShpleminiVerifier_<TypeParam>;
+    using KZG = KZG<TypeParam>;
+    using Fr = typename TypeParam::ScalarField;
+    using Commitment = typename TypeParam::AffineElement;
+    using Polynomial = typename bb::Polynomial<Fr>;
+
+    const size_t n = 16;
+    const size_t log_n = 4;
+    // Generate multilinear polynomials, their commitments (genuine and mocked) and evaluations (genuine) at a random
+    // point.
+    auto mle_opening_point = this->random_evaluation_point(log_n); // sometimes denoted 'u'
+    auto poly1 = Polynomial::random(n);
+    auto poly2 = Polynomial::random(n, 1);
+    auto poly3 = Polynomial::random(n, 1);
+    auto poly4 = Polynomial::random(n);
+
+    Commitment commitment1 = this->commit(poly1);
+    Commitment commitment2 = this->commit(poly2);
+    Commitment commitment3 = this->commit(poly3);
+    Commitment commitment4 = this->commit(poly4);
+    std::vector<Commitment> unshifted_commitments = { commitment1, commitment2, commitment3, commitment4 };
+    std::vector<Commitment> shifted_commitments = { commitment2, commitment3 };
+    auto eval1 = poly1.evaluate_mle(mle_opening_point);
+    auto eval2 = poly2.evaluate_mle(mle_opening_point);
+    auto eval3 = poly3.evaluate_mle(mle_opening_point);
+    auto eval4 = poly4.evaluate_mle(mle_opening_point);
+    auto eval2_shift = poly2.evaluate_mle(mle_opening_point, true);
+    auto eval3_shift = poly3.evaluate_mle(mle_opening_point, true);
+
+    // Collect multilinear evaluations for input to prover
+    // std::vector<Fr> multilinear_evaluations = { eval1, eval2, eval3, eval4, eval2_shift, eval3_shift };
+
+    auto prover_transcript = NativeTranscript::prover_init_empty();
+
+    // Run the full prover PCS protocol:
+
+    // Compute:
+    // - (d+1) opening pairs: {r, \hat{a}_0}, {-r^{2^i}, a_i}, i = 0, ..., d-1
+    // - (d+1) Fold polynomials Fold_{r}^(0), Fold_{-r}^(0), and Fold^(i), i = 0, ..., d-1
+    auto prover_opening_claims = GeminiProver::prove(n,
+                                                     RefArray{ poly1, poly2, poly3, poly4 },
+                                                     RefArray{ poly2, poly3 },
+                                                     mle_opening_point,
+                                                     this->ck(),
+                                                     prover_transcript);
+
+    // Shplonk prover output:
+    // - opening pair: (z_challenge, 0)
+    // - witness: polynomial Q - Q_z
+    const auto opening_claim = ShplonkProver::prove(this->ck(), prover_opening_claims, prover_transcript);
+
+    // KZG prover:
+    // - Adds commitment [W] to transcript
+    KZG::compute_opening_proof(this->ck(), opening_claim, prover_transcript);
+
+    // Run the full verifier PCS protocol with genuine opening claims (genuine commitment, genuine evaluation)
+
+    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    // the index of the first commitment to a polynomial to be shifted in the union of unshifted_commitments and
+    // shifted_commitments. in our case, it is poly2
+    const size_t to_be_shifted_commitments_start = 1;
+    // the index of the first commitment to a shifted polynomial in the union of unshifted_commitments and
+    // shifted_commitments. in our case, it is the second occurence of poly2
+    const size_t shifted_commitments_start = 4;
+    // number of shifted polynomials
+    const size_t num_shifted_commitments = 2;
+    // since commitments to poly2, poly3 and their shifts are the same group elements, we simply combine the scalar
+    // multipliers of commitment2 and commitment3 in one place and remove the entries of the commitments and scalars
+    // vectors corresponding to the "shifted" commitment
+    const RepeatedCommitmentsData repeated_commitments =
+        RepeatedCommitmentsData(to_be_shifted_commitments_start, shifted_commitments_start, num_shifted_commitments);
+
+    // Gemini verifier output:
+    // - claim: d+1 commitments to Fold_{r}^(0), Fold_{-r}^(0), Fold^(l), d+1 evaluations a_0_pos, a_l, l = 0:d-1
+    const auto batch_opening_claim =
+        ShpleminiVerifier::compute_batch_opening_claim(n,
+                                                       RefVector(unshifted_commitments),
+                                                       RefVector(shifted_commitments),
+                                                       RefArray{ eval1, eval2, eval3, eval4 },
+                                                       RefArray{ eval2_shift, eval3_shift },
+                                                       mle_opening_point,
+                                                       this->vk()->get_g1_identity(),
+                                                       verifier_transcript,
+                                                       repeated_commitments);
+
+    const auto pairing_points = KZG::reduce_verify_batch_opening_claim(batch_opening_claim, verifier_transcript);
+
+    // Final pairing check: e([Q] - [Q_z] + z[W], [1]_2) = e([W], [x]_2)
+    EXPECT_EQ(this->vk()->pairing_check(pairing_points[0], pairing_points[1]), true);
+}
 
 } // namespace bb

barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp

@@ -4,6 +4,7 @@
 #include "barretenberg/commitment_schemes/gemini/gemini_impl.hpp"
 #include "barretenberg/commitment_schemes/shplonk/shplonk.hpp"
 #include "barretenberg/commitment_schemes/verification_key.hpp"
+#include "barretenberg/flavor/repeated_commitments_data.hpp"
 #include "barretenberg/transcript/transcript.hpp"
 
 namespace bb {
@@ -132,6 +133,7 @@ template <typename Curve> class ShpleminiVerifier_ {
         const std::vector<Fr>& multivariate_challenge,
         const Commitment& g1_identity,
         const std::shared_ptr<Transcript>& transcript,
+        const RepeatedCommitmentsData& repeated_commitments = {},
         RefSpan<Commitment> libra_univariate_commitments = {},
         const std::vector<Fr>& libra_univariate_evaluations = {},
         const std::vector<RefVector<Commitment>>& concatenation_group_commitments = {},
@@ -288,6 +290,8 @@ template <typename Curve> class ShpleminiVerifier_ {
         commitments.emplace_back(g1_identity);
         scalars.emplace_back(constant_term_accumulator);
 
+        remove_repeated_commitments(commitments, scalars, repeated_commitments, has_zk);
+
         // For ZK flavors, the sumcheck output contains the evaluations of Libra univariates that submitted to the
         // ShpleminiVerifier, otherwise this argument is set to be empty
         if (has_zk) {
@@ -493,13 +497,93 @@ template <typename Curve> class ShpleminiVerifier_ {
         }
     }
 
+    /**
+     * @brief Combines scalars of repeating commitments to reduce the number of scalar multiplications performed by the
+     * verifier.
+     *
+     * @details The Shplemini verifier gets the access to multiple groups of commitments, some of which are duplicated
+     * because they correspond to polynomials whose shifts also evaluated or used in concatenation groups in
+     * Translator. This method combines the scalars associated with these repeating commitments, reducing the total
+     * number of scalar multiplications required during the verification.
+     *
+     * More specifically, the Shplemini verifier receives two or three groups of commitments: get_unshifted() and
+     * get_to_be_shifted() in the case of Ultra, Mega, and ECCVM Flavors; and get_unshifted_without_concatenated(),
+     * get_to_be_shifted(), and get_groups_to_be_concatenated() in the case of the TranslatorFlavor. The commitments are
+     * then placed in this specific order in a BatchOpeningClaim object containing a vector of commitments and a vector
+     * of scalars. The ranges with repeated commitments belong to the Flavors. This method iterates over these ranges
+     * and sums the scalar multipliers corresponding to the same group element. After combining the scalars, we erase
+     * corresponding entries in both vectors.
+     *
+     */
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1151) Avoid erasing vector elements.
+    static void remove_repeated_commitments(std::vector<Commitment>& commitments,
+                                            std::vector<Fr>& scalars,
+                                            const RepeatedCommitmentsData& repeated_commitments,
+                                            bool has_zk)
+    {
+        // We started populating commitments and scalars by adding Shplonk:Q commitmment and the corresponding scalar
+        // factor 1. In the case of ZK, we also added Gemini:masking_poly_comm before populating the vector with
+        // commitments to prover polynomials
+        const size_t offset = has_zk ? 2 : 1;
+
+        // Extract the indices from the container, which is normally created in a given Flavor
+        const size_t& first_range_to_be_shifted_start = repeated_commitments.first_range_to_be_shifted_start + offset;
+        const size_t& first_range_shifted_start = repeated_commitments.first_range_shifted_start + offset;
+        const size_t& first_range_size = repeated_commitments.first_range_size;
+
+        const size_t& second_range_to_be_shifted_start = repeated_commitments.second_range_to_be_shifted_start + offset;
+        const size_t& second_range_shifted_start = repeated_commitments.second_range_shifted_start + offset;
+        const size_t& second_range_size = repeated_commitments.second_range_size;
+
+        // Iterate over the first range of to-be-shifted scalars and their shifted counterparts
+        for (size_t i = 0; i < first_range_size; i++) {
+            size_t idx_to_be_shifted = i + first_range_to_be_shifted_start;
+            size_t idx_shifted = i + first_range_shifted_start;
+            scalars[idx_to_be_shifted] = scalars[idx_to_be_shifted] + scalars[idx_shifted];
+        }
+
+        // Iterate over the second range of to-be-shifted precomputed scalars and their shifted counterparts (if
+        // provided)
+        for (size_t i = 0; i < second_range_size; i++) {
+            size_t idx_to_be_shifted = i + second_range_to_be_shifted_start;
+            size_t idx_shifted = i + second_range_shifted_start;
+            scalars[idx_to_be_shifted] = scalars[idx_to_be_shifted] + scalars[idx_shifted];
+        }
+
+        if (second_range_shifted_start > first_range_shifted_start) {
+            // Erase the shifted scalars and commitments from the second range (if provided)
+            for (size_t i = 0; i < second_range_size; ++i) {
+                scalars.erase(scalars.begin() + static_cast<std::ptrdiff_t>(second_range_shifted_start));
+                commitments.erase(commitments.begin() + static_cast<std::ptrdiff_t>(second_range_shifted_start));
+            }
+
+            // Erase the shifted scalars and commitments from the first range
+            for (size_t i = 0; i < first_range_size; ++i) {
+                scalars.erase(scalars.begin() + static_cast<std::ptrdiff_t>(first_range_shifted_start));
+                commitments.erase(commitments.begin() + static_cast<std::ptrdiff_t>(first_range_shifted_start));
+            }
+        } else {
+            // Erase the shifted scalars and commitments from the first range
+            for (size_t i = 0; i < first_range_size; ++i) {
+                scalars.erase(scalars.begin() + static_cast<std::ptrdiff_t>(first_range_shifted_start));
+                commitments.erase(commitments.begin() + static_cast<std::ptrdiff_t>(first_range_shifted_start));
+            }
+            // Erase the shifted scalars and commitments from the second range (if provided)
+            for (size_t i = 0; i < second_range_size; ++i) {
+                scalars.erase(scalars.begin() + static_cast<std::ptrdiff_t>(second_range_shifted_start));
+                commitments.erase(commitments.begin() + static_cast<std::ptrdiff_t>(second_range_shifted_start));
+            }
+        }
+    }
+
     /**
      * @brief Add the opening data corresponding to Libra masking univariates to the batched opening claim
      *
      * @details After verifying ZK Sumcheck, the verifier has to validate the claims about the evaluations of Libra
-     * univariates used to mask Sumcheck round univariates. To minimize the overhead of such openings, we continue the
-     * Shplonk batching started in Gemini, i.e. we add new claims multiplied by a suitable power of the Shplonk batching
-     * challenge and re-use the evaluation challenge sampled to prove the evaluations of Gemini polynomials.
+     * univariates used to mask Sumcheck round univariates. To minimize the overhead of such openings, we continue
+     * the Shplonk batching started in Gemini, i.e. we add new claims multiplied by a suitable power of the Shplonk
+     * batching challenge and re-use the evaluation challenge sampled to prove the evaluations of Gemini
+     * polynomials.
      *
      * @param commitments
      * @param scalars
@@ -541,8 +625,8 @@ template <typename Curve> class ShpleminiVerifier_ {
         if constexpr (!Curve::is_stdlib_type) {
             Fr::batch_invert(denominators);
         }
-        // add Libra commitments to the vector of commitments; compute corresponding scalars and the correction to the
-        // constant term
+        // add Libra commitments to the vector of commitments; compute corresponding scalars and the correction to
+        // the constant term
         for (const auto [libra_univariate_commitment, denominator, libra_univariate_evaluation] :
              zip_view(libra_univariate_commitments, denominators, libra_univariate_evaluations)) {
             commitments.push_back(std::move(libra_univariate_commitment));

barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.test.cpp

@@ -324,6 +324,7 @@ TYPED_TEST(ShpleminiTest, ShpleminiWithMaskingLibraUnivariates)
                                                                               mle_opening_point,
                                                                               this->vk()->get_g1_identity(),
                                                                               verifier_transcript,
+                                                                              {},
                                                                               RefVector(libra_commitments),
                                                                               libra_evaluations);