Merge pull request #5 from osmosis-labs/adam/batch-verfy-backport

chore: batch verify backport

.github/workflows/e2e.yml

@@ -2,7 +2,7 @@ name: e2e
 # Runs the CI end-to-end test network on all pushes to main or release branches
 # and every pull request, but only if any Go files have been changed.
 on:
-  workflow_dispatch:  # allow running workflow manually
+  workflow_dispatch: # allow running workflow manually
   pull_request:
   push:
     branches:
@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.21'
+          go-version: "1.21"
       - uses: actions/checkout@v4
       - uses: technote-space/get-diff-action@v6
         with:

crypto/batch/batch.go

@@ -0,0 +1,32 @@
+package batch
+
+import (
+	"github.com/cometbft/cometbft/crypto"
+	"github.com/cometbft/cometbft/crypto/ed25519"
+	"github.com/cometbft/cometbft/crypto/sr25519"
+)
+
+// CreateBatchVerifier checks if a key type implements the batch verifier interface.
+// Currently only ed25519 & sr25519 supports batch verification.
+func CreateBatchVerifier(pk crypto.PubKey) (crypto.BatchVerifier, bool) {
+	switch pk.Type() {
+	case ed25519.KeyType:
+		return ed25519.NewBatchVerifier(), true
+	case sr25519.KeyType:
+		return sr25519.NewBatchVerifier(), true
+	}
+
+	// case where the key does not support batch verification
+	return nil, false
+}
+
+// SupportsBatchVerifier checks if a key type implements the batch verifier
+// interface.
+func SupportsBatchVerifier(pk crypto.PubKey) bool {
+	switch pk.Type() {
+	case ed25519.KeyType, sr25519.KeyType:
+		return true
+	}
+
+	return false
+}

crypto/crypto.go

@@ -40,3 +40,15 @@ type Symmetric interface {
 	Encrypt(plaintext []byte, secret []byte) (ciphertext []byte)
 	Decrypt(ciphertext []byte, secret []byte) (plaintext []byte, err error)
 }
+
+// If a new key type implements batch verification,
+// the key type must be registered in github.com/tendermint/tendermint/crypto/batch
+type BatchVerifier interface {
+	// Add appends an entry into the BatchVerifier.
+	Add(key PubKey, message, signature []byte) error
+	// Verify verifies all the entries in the BatchVerifier, and returns
+	// if every signature in the batch is valid, and a vector of bools
+	// indicating the verification status of each signature (in the order
+	// that signatures were added to the batch).
+	Verify() (bool, []bool)
+}

crypto/ed25519/bench_test.go

@@ -1,9 +1,12 @@
 package ed25519
 
 import (
+	"fmt"
 	"io"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/cometbft/cometbft/crypto"
 	"github.com/cometbft/cometbft/crypto/internal/benchmarking"
 )
@@ -24,3 +27,42 @@ func BenchmarkVerification(b *testing.B) {
 	priv := GenPrivKey()
 	benchmarking.BenchmarkVerification(b, priv)
 }
+
+func BenchmarkVerifyBatch(b *testing.B) {
+	msg := []byte("BatchVerifyTest")
+
+	for _, sigsCount := range []int{1, 8, 64, 1024} {
+		sigsCount := sigsCount
+		b.Run(fmt.Sprintf("sig-count-%d", sigsCount), func(b *testing.B) {
+			// Pre-generate all of the keys, and signatures, but do not
+			// benchmark key-generation and signing.
+			pubs := make([]crypto.PubKey, 0, sigsCount)
+			sigs := make([][]byte, 0, sigsCount)
+			for i := 0; i < sigsCount; i++ {
+				priv := GenPrivKey()
+				sig, _ := priv.Sign(msg)
+				pubs = append(pubs, priv.PubKey().(PubKey))
+				sigs = append(sigs, sig)
+			}
+			b.ResetTimer()
+
+			b.ReportAllocs()
+			// NOTE: dividing by n so that metrics are per-signature
+			for i := 0; i < b.N/sigsCount; i++ {
+				// The benchmark could just benchmark the Verify()
+				// routine, but there is non-trivial overhead associated
+				// with BatchVerifier.Add(), which should be included
+				// in the benchmark.
+				v := NewBatchVerifier()
+				for i := 0; i < sigsCount; i++ {
+					err := v.Add(pubs[i], msg, sigs[i])
+					require.NoError(b, err)
+				}
+
+				if ok, _ := v.Verify(); !ok {
+					b.Fatal("signature set failed batch verification")
+				}
+			}
+		})
+	}
+}

crypto/ed25519/ed25519.go

@@ -3,10 +3,12 @@ package ed25519
 import (
 	"bytes"
 	"crypto/subtle"
+	"errors"
 	"fmt"
 	"io"
 
-	"golang.org/x/crypto/ed25519"
+	"github.com/oasisprotocol/curve25519-voi/primitives/ed25519"
+	"github.com/oasisprotocol/curve25519-voi/primitives/ed25519/extra/cache"
 
 	"github.com/cometbft/cometbft/crypto"
 	"github.com/cometbft/cometbft/crypto/tmhash"
@@ -15,7 +17,19 @@ import (
 
 //-------------------------------------
 
-var _ crypto.PrivKey = PrivKey{}
+var (
+	_ crypto.PrivKey       = PrivKey{}
+	_ crypto.BatchVerifier = &BatchVerifier{}
+
+	// curve25519-voi's Ed25519 implementation supports configurable
+	// verification behavior, and tendermint uses the ZIP-215 verification
+	// semantics.
+	verifyOptions = &ed25519.Options{
+		Verify: ed25519.VerifyOptionsZIP_215,
+	}
+
+	cachingVerifier = cache.NewVerifier(cache.NewLRUCache(cacheSize))
+)
 
 const (
 	PrivKeyName = "tendermint/PrivKeyEd25519"
@@ -32,6 +46,14 @@ const (
 	SeedSize = 32
 
 	KeyType = "ed25519"
+
+	// cacheSize is the number of public keys that will be cached in
+	// an expanded format for repeated signature verification.
+	//
+	// TODO/perf: Either this should exclude single verification, or be
+	// tuned to `> validatorSize + maxTxnsPerBlock` to avoid cache
+	// thrashing.
+	cacheSize = 4096
 )
 
 func init() {
@@ -105,14 +127,12 @@ func GenPrivKey() PrivKey {
 
 // genPrivKey generates a new ed25519 private key using the provided reader.
 func genPrivKey(rand io.Reader) PrivKey {
-	seed := make([]byte, SeedSize)
-
-	_, err := io.ReadFull(rand, seed)
+	_, priv, err := ed25519.GenerateKey(rand)
 	if err != nil {
 		panic(err)
 	}
 
-	return PrivKey(ed25519.NewKeyFromSeed(seed))
+	return PrivKey(priv)
 }
 
 // GenPrivKeyFromSecret hashes the secret with SHA2, and uses
@@ -129,7 +149,7 @@ func GenPrivKeyFromSecret(secret []byte) PrivKey {
 
 var _ crypto.PubKey = PubKey{}
 
-// PubKeyEd25519 implements crypto.PubKey for the Ed25519 signature scheme.
+// PubKey implements crypto.PubKey for the Ed25519 signature scheme.
 type PubKey []byte
 
 // Address is the SHA256-20 of the raw pubkey bytes.
@@ -151,7 +171,7 @@ func (pubKey PubKey) VerifySignature(msg []byte, sig []byte) bool {
 		return false
 	}
 
-	return ed25519.Verify(ed25519.PublicKey(pubKey), msg, sig)
+	return cachingVerifier.VerifyWithOptions(ed25519.PublicKey(pubKey), msg, sig, verifyOptions)
 }
 
 func (pubKey PubKey) String() string {
@@ -169,3 +189,40 @@ func (pubKey PubKey) Equals(other crypto.PubKey) bool {
 
 	return false
 }
+
+//-------------------------------------
+
+// BatchVerifier implements batch verification for ed25519.
+type BatchVerifier struct {
+	*ed25519.BatchVerifier
+}
+
+func NewBatchVerifier() crypto.BatchVerifier {
+	return &BatchVerifier{ed25519.NewBatchVerifier()}
+}
+
+func (b *BatchVerifier) Add(key crypto.PubKey, msg, signature []byte) error {
+	pkEd, ok := key.(PubKey)
+	if !ok {
+		return fmt.Errorf("pubkey is not Ed25519")
+	}
+
+	pkBytes := pkEd.Bytes()
+
+	if l := len(pkBytes); l != PubKeySize {
+		return fmt.Errorf("pubkey size is incorrect; expected: %d, got %d", PubKeySize, l)
+	}
+
+	// check that the signature is the correct length
+	if len(signature) != SignatureSize {
+		return errors.New("invalid signature")
+	}
+
+	cachingVerifier.AddWithOptions(b.BatchVerifier, ed25519.PublicKey(pkBytes), msg, signature, verifyOptions)
+
+	return nil
+}
+
+func (b *BatchVerifier) Verify() (bool, []bool) {
+	return b.BatchVerifier.Verify(crypto.CReader())
+}

crypto/ed25519/ed25519_test.go

@@ -11,7 +11,6 @@ import (
 )
 
 func TestSignAndValidateEd25519(t *testing.T) {
-
 	privKey := ed25519.GenPrivKey()
 	pubKey := privKey.PubKey()
 
@@ -28,3 +27,28 @@ func TestSignAndValidateEd25519(t *testing.T) {
 
 	assert.False(t, pubKey.VerifySignature(msg, sig))
 }
+
+func TestBatchSafe(t *testing.T) {
+	v := ed25519.NewBatchVerifier()
+
+	for i := 0; i <= 38; i++ {
+		priv := ed25519.GenPrivKey()
+		pub := priv.PubKey()
+
+		var msg []byte
+		if i%2 == 0 {
+			msg = []byte("easter")
+		} else {
+			msg = []byte("egg")
+		}
+
+		sig, err := priv.Sign(msg)
+		require.NoError(t, err)
+
+		err = v.Add(pub, msg, sig)
+		require.NoError(t, err)
+	}
+
+	ok, _ := v.Verify()
+	require.True(t, ok)
+}

crypto/sr25519/batch.go

@@ -0,0 +1,46 @@
+package sr25519
+
+import (
+	"fmt"
+
+	"github.com/oasisprotocol/curve25519-voi/primitives/sr25519"
+
+	"github.com/cometbft/cometbft/crypto"
+)
+
+var _ crypto.BatchVerifier = &BatchVerifier{}
+
+// BatchVerifier implements batch verification for sr25519.
+type BatchVerifier struct {
+	*sr25519.BatchVerifier
+}
+
+func NewBatchVerifier() crypto.BatchVerifier {
+	return &BatchVerifier{sr25519.NewBatchVerifier()}
+}
+
+func (b *BatchVerifier) Add(key crypto.PubKey, msg, signature []byte) error {
+	pk, ok := key.(PubKey)
+	if !ok {
+		return fmt.Errorf("sr25519: pubkey is not sr25519")
+	}
+
+	var srpk sr25519.PublicKey
+	if err := srpk.UnmarshalBinary(pk); err != nil {
+		return fmt.Errorf("sr25519: invalid public key: %w", err)
+	}
+
+	var sig sr25519.Signature
+	if err := sig.UnmarshalBinary(signature); err != nil {
+		return fmt.Errorf("sr25519: unable to decode signature: %w", err)
+	}
+
+	st := signingCtx.NewTranscriptBytes(msg)
+	b.BatchVerifier.Add(&srpk, st, &sig)
+
+	return nil
+}
+
+func (b *BatchVerifier) Verify() (bool, []bool) {
+	return b.BatchVerifier.Verify(crypto.CReader())
+}

crypto/sr25519/bench_test.go

@@ -1,9 +1,12 @@
 package sr25519
 
 import (
+	"fmt"
 	"io"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/cometbft/cometbft/crypto"
 	"github.com/cometbft/cometbft/crypto/internal/benchmarking"
 )
@@ -24,3 +27,42 @@ func BenchmarkVerification(b *testing.B) {
 	priv := GenPrivKey()
 	benchmarking.BenchmarkVerification(b, priv)
 }
+
+func BenchmarkVerifyBatch(b *testing.B) {
+	msg := []byte("BatchVerifyTest")
+
+	for _, sigsCount := range []int{1, 8, 64, 1024} {
+		sigsCount := sigsCount
+		b.Run(fmt.Sprintf("sig-count-%d", sigsCount), func(b *testing.B) {
+			// Pre-generate all of the keys, and signatures, but do not
+			// benchmark key-generation and signing.
+			pubs := make([]crypto.PubKey, 0, sigsCount)
+			sigs := make([][]byte, 0, sigsCount)
+			for i := 0; i < sigsCount; i++ {
+				priv := GenPrivKey()
+				sig, _ := priv.Sign(msg)
+				pubs = append(pubs, priv.PubKey().(PubKey))
+				sigs = append(sigs, sig)
+			}
+			b.ResetTimer()
+
+			b.ReportAllocs()
+			// NOTE: dividing by n so that metrics are per-signature
+			for i := 0; i < b.N/sigsCount; i++ {
+				// The benchmark could just benchmark the Verify()
+				// routine, but there is non-trivial overhead associated
+				// with BatchVerifier.Add(), which should be included
+				// in the benchmark.
+				v := NewBatchVerifier()
+				for i := 0; i < sigsCount; i++ {
+					err := v.Add(pubs[i], msg, sigs[i])
+					require.NoError(b, err)
+				}
+
+				if ok, _ := v.Verify(); !ok {
+					b.Fatal("signature set failed batch verification")
+				}
+			}
+		})
+	}
+}