From 5e1f8672f25ea591f030159b1407f98624265862 Mon Sep 17 00:00:00 2001 From: hacheigriega Date: Thu, 5 Sep 2024 22:59:26 -0400 Subject: [PATCH] feat: progress on tree constructions --- app/app.go | 1 + cmd/sedad/utils/merkle.go | 80 ++++++++++++++ cmd/sedad/utils/seda_keys.go | 22 ++++ go.mod | 5 +- go.sum | 8 +- x/batching/keeper/abci.go | 152 ++++++++++++++++++--------- x/batching/keeper/keeper.go | 26 +++++ x/batching/types/expected_keepers.go | 5 + x/pubkey/client/cli/tx.go | 21 ++-- 9 files changed, 249 insertions(+), 71 deletions(-) create mode 100644 cmd/sedad/utils/merkle.go create mode 100644 cmd/sedad/utils/seda_keys.go diff --git a/app/app.go b/app/app.go index f4e53764..f291fc9e 100644 --- a/app/app.go +++ b/app/app.go @@ -659,6 +659,7 @@ func NewApp( authtypes.NewModuleAddress(batchingtypes.ModuleName).String(), app.StakingKeeper, app.WasmStorageKeeper, + app.PubKeyKeeper, contractKeeper, app.WasmKeeper, authcodec.NewBech32Codec(sdk.GetConfig().GetBech32ValidatorAddrPrefix()), diff --git a/cmd/sedad/utils/merkle.go b/cmd/sedad/utils/merkle.go new file mode 100644 index 00000000..5455f126 --- /dev/null +++ b/cmd/sedad/utils/merkle.go @@ -0,0 +1,80 @@ +// Largely taken from the CometBFT repo. +// Source: https://github.com/cometbft/cometbft/blob/main/crypto/merkle/tree.go +package utils + +import ( + "bytes" + "crypto/sha256" + "hash" + "math/bits" + + "github.com/cometbft/cometbft/crypto/tmhash" +) + +// TODO: make these have a large predefined capacity +var ( + leafPrefix = []byte{0} + innerPrefix = []byte{1} +) + +// HashFromByteSlices computes a Merkle tree where the leaves are the byte slice, +// in the provided order. It follows RFC-6962. +func HashFromByteSlices(items [][]byte) []byte { + return hashFromByteSlices(sha256.New(), items) +} + +func hashFromByteSlices(sha hash.Hash, items [][]byte) []byte { + switch len(items) { + case 0: + return emptyHash() + case 1: + return leafHashOpt(sha, items[0]) + default: + k := getSplitPoint(int64(len(items))) + a := hashFromByteSlices(sha, items[:k]) + b := hashFromByteSlices(sha, items[k:]) + + var left, right []byte + if bytes.Compare(a, b) == -1 { + left, right = a, b + } else { + right, left = a, b + } + return innerHashOpt(sha, left, right) + } +} + +// returns tmhash() +func emptyHash() []byte { + return tmhash.Sum([]byte{}) +} + +// returns tmhash(0x00 || leaf) +func leafHashOpt(s hash.Hash, leaf []byte) []byte { + s.Reset() + s.Write(leafPrefix) + s.Write(leaf) + return s.Sum(nil) +} + +func innerHashOpt(s hash.Hash, left []byte, right []byte) []byte { + s.Reset() + s.Write(innerPrefix) + s.Write(left) + s.Write(right) + return s.Sum(nil) +} + +// getSplitPoint returns the largest power of 2 less than length +func getSplitPoint(length int64) int64 { + if length < 1 { + panic("Trying to split a tree with size < 1") + } + uLength := uint(length) + bitlen := bits.Len(uLength) + k := int64(1 << uint(bitlen-1)) + if k == length { + k >>= 1 + } + return k +} diff --git a/cmd/sedad/utils/seda_keys.go b/cmd/sedad/utils/seda_keys.go new file mode 100644 index 00000000..8a6d9564 --- /dev/null +++ b/cmd/sedad/utils/seda_keys.go @@ -0,0 +1,22 @@ +package utils + +import ( + cmtcrypto "github.com/cometbft/cometbft/crypto" + "github.com/cometbft/cometbft/crypto/secp256k1" +) + +const ( + SEDAKeysIndexSecp256k1 = 0 +) + +// SEDAKeysGenerators is a map from SEDA Key Index to the +// corresponding private key generator function. +var SEDAKeysGenerators = map[uint32]PrivKeyGenerator{ + SEDAKeysIndexSecp256k1: secp256k1GenPrivKey, +} + +type PrivKeyGenerator func() cmtcrypto.PrivKey + +func secp256k1GenPrivKey() cmtcrypto.PrivKey { + return secp256k1.GenPrivKey() +} diff --git a/go.mod b/go.mod index 82d6bff5..86c5e719 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,9 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 + github.com/txaty/go-merkletree v0.2.2 go.uber.org/mock v0.4.0 + golang.org/x/crypto v0.25.0 golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0 google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d google.golang.org/grpc v1.64.1 @@ -224,11 +226,10 @@ require ( go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.25.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sync v0.7.0 // indirect + golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.23.0 // indirect golang.org/x/term v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect diff --git a/go.sum b/go.sum index 51c0f325..61d772a4 100644 --- a/go.sum +++ b/go.sum @@ -254,6 +254,8 @@ github.com/adlio/schema v1.3.3 h1:oBJn8I02PyTB466pZO1UZEn1TV5XLlifBSyMrmHl/1I= github.com/adlio/schema v1.3.3/go.mod h1:1EsRssiv9/Ce2CMzq5DoL7RiMshhuigQxrR4DMV9fHg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/agiledragon/gomonkey/v2 v2.11.0 h1:5oxSgA+tC1xuGsrIorR+sYiziYltmJyEZ9qA25b6l5U= +github.com/agiledragon/gomonkey/v2 v2.11.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -1087,6 +1089,8 @@ github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/txaty/go-merkletree v0.2.2 h1:K5bHDFK+Q3KK+gEJeyTOECKuIwl/LVo4CI+cm0/p34g= +github.com/txaty/go-merkletree v0.2.2/go.mod h1:w5HPEu7ubNw5LzS+91m+1/GtuZcWHKiPU3vEGi+ThJM= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= @@ -1320,8 +1324,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/x/batching/keeper/abci.go b/x/batching/keeper/abci.go index 12e1ef38..e39220bd 100644 --- a/x/batching/keeper/abci.go +++ b/x/batching/keeper/abci.go @@ -1,17 +1,16 @@ package keeper import ( - "bytes" "encoding/binary" "encoding/hex" "encoding/json" - "fmt" - "sort" - - "github.com/cometbft/cometbft/crypto/merkle" + "errors" + "cosmossdk.io/collections" sdk "github.com/cosmos/cosmos-sdk/types" + "golang.org/x/crypto/sha3" + "github.com/sedaprotocol/seda-chain/cmd/sedad/utils" "github.com/sedaprotocol/seda-chain/x/batching/types" tallytypes "github.com/sedaprotocol/seda-chain/x/tally/types" ) @@ -33,17 +32,16 @@ func (k Keeper) EndBlock(ctx sdk.Context) (err error) { batch, err := k.ConstructBatch(ctx) if err != nil { - panic(err) + return err } - fmt.Println(batch) err = k.SetBatch(ctx, batch) if err != nil { - panic(err) + return err } err = k.IncrementCurrentBatchNum(ctx) if err != nil { - panic(err) + return err } return nil @@ -54,76 +52,126 @@ func (k Keeper) ConstructBatch(ctx sdk.Context) (types.Batch, error) { if err != nil { return types.Batch{}, err } + dataRootHex, err := k.ConstructDataResultTree(ctx) + if err != nil { + return types.Batch{}, err + } + valRootHex, err := k.ConstructValidatorTree(ctx) + if err != nil { + return types.Batch{}, err + } - // Construct data result tree. + return types.Batch{ + BatchNumber: curBatchNum, + BlockHeight: ctx.BlockHeight(), + DataResultRoot: dataRootHex, + ValidatorRoot: valRootHex, + BlockTime: ctx.BlockTime(), + }, nil +} + +// ConstructDataResultTree constructs a data result tree based on the +// batching-ready data results returned from the core contract and +// returns a hex-encoded tree root. +func (k Keeper) ConstructDataResultTree(ctx sdk.Context) (string, error) { coreContract, err := k.wasmStorageKeeper.GetCoreContractAddr(ctx) if err != nil { - return types.Batch{}, err + return "", err } // TODO: Deal with offset and limits. (#313) queryRes, err := k.wasmViewKeeper.QuerySmart(ctx, coreContract, []byte(`{"get_data_results_by_status":{"status": "tallied", "offset": 0, "limit": 100}}`)) if err != nil { - return types.Batch{}, err + return "", err } if string(queryRes) == "[]" { - return types.Batch{}, err + return "", err } var dataResults []tallytypes.DataResult err = json.Unmarshal(queryRes, &dataResults) if err != nil { - return types.Batch{}, err + return "", err } - var dataLeaves [][]byte + leaves := make([][]byte, len(dataResults)) for _, res := range dataResults { resHash, err := hex.DecodeString(res.ID) if err != nil { - return types.Batch{}, err + return "", err } - dataLeaves = append(dataLeaves, resHash) + leaves = append(leaves, resHash) } - sort.Slice(dataLeaves, func(i, j int) bool { - if bytes.Compare(dataLeaves[i], dataLeaves[j]) == -1 { - return true - } - return false - }) - dataRoot := merkle.HashFromByteSlices(dataLeaves) - dataRootHex := hex.EncodeToString(dataRoot) - - // Construct validator tree. - var valLeaves [][]byte - err = k.stakingKeeper.IterateLastValidatorPowers(ctx, func(addr sdk.ValAddress, power int64) (stop bool) { - // TODO construct with pubkey in pubkey module instead - buf := make([]byte, len(addr)+8) - copy(buf[:len(addr)], addr) - binary.BigEndian.PutUint64(buf[:len(addr)], uint64(power)) - - valLeaves = append(valLeaves, addr) - return false - }) + // TODO construct the whole tree. (and merkle proofs?) + // curRoot := merkle.HashFromByteSlices(leaves) + curRoot := utils.HashFromByteSlices(leaves) + prevRoot, err := k.GetPreviousDataResultRoot(ctx) if err != nil { - return types.Batch{}, err + return "", err } + // root := merkle.HashFromByteSlices([][]byte{prevRoot, curRoot}) + root := utils.HashFromByteSlices([][]byte{prevRoot, curRoot}) - // TODO subtrees based on keys + // TODO update data result status on contract - sort.Slice(valLeaves, func(i, j int) bool { - if bytes.Compare(valLeaves[i], valLeaves[j]) == -1 { - return true - } + return hex.EncodeToString(root), nil +} + +type validatorPower struct { + ValAddr sdk.ValAddress + Power int64 +} + +// ConstructValidatorTree constructs a validator tree based on the +// validators in the active set and their registered public keys. +func (k Keeper) ConstructValidatorTree(ctx sdk.Context) (string, error) { + var activeSet []validatorPower + err := k.stakingKeeper.IterateLastValidatorPowers(ctx, func(valAddr sdk.ValAddress, power int64) (stop bool) { + activeSet = append(activeSet, validatorPower{ValAddr: valAddr, Power: power}) return false }) - valRoot := merkle.HashFromByteSlices(valLeaves) - valRootHex := hex.EncodeToString(valRoot) + if err != nil { + return "", err + } - return types.Batch{ - BatchNumber: curBatchNum, - BlockHeight: ctx.BlockHeight(), - DataResultRoot: dataRootHex, - ValidatorRoot: valRootHex, - BlockTime: ctx.BlockTime(), - }, nil + var leaves [][]byte + var votes []types.Vote + for _, vp := range activeSet { + pubKey, err := k.pubKeyKeeper.GetValidatorKeyAtIndex(ctx, vp.ValAddr, utils.SEDAKeysIndexSecp256k1) + if err != nil { + if errors.Is(err, collections.ErrNotFound) { + continue // TODO check + } + } + + // Construct a leaf content and hash it. + pkBytes := pubKey.Bytes() + buf := make([]byte, len(pkBytes)+8) + copy(buf[:len(pkBytes)], pkBytes) + binary.BigEndian.PutUint64(buf[len(pkBytes):], uint64(vp.Power)) + + hash := sha3.New256() + hash.Write(buf) + hashed := hash.Sum(nil) + + leaves = append(leaves, hashed) + votes = append(votes, types.Vote{ + ValidatorAddr: vp.ValAddr.String(), + VotingPower: vp.Power, + Signatures: []*types.Signature{{ + Scheme: utils.SEDAKeysIndexSecp256k1, + Signature: "", + PublicKey: pubKey.String(), + MerkleProof: "", // TODO populate this + }}, + }) + } + + // TODO construct the whole tree and populate merkle proof fields. + // secp256k1Root := merkle.HashFromByteSlices(leaves) + // root := merkle.HashFromByteSlices([][]byte{{}, secp256k1Root}) + secp256k1Root := utils.HashFromByteSlices(leaves) + root := utils.HashFromByteSlices([][]byte{{}, secp256k1Root}) + + return hex.EncodeToString(root), nil } diff --git a/x/batching/keeper/keeper.go b/x/batching/keeper/keeper.go index 5f155f96..97f8a668 100644 --- a/x/batching/keeper/keeper.go +++ b/x/batching/keeper/keeper.go @@ -2,6 +2,7 @@ package keeper import ( "context" + "encoding/hex" "fmt" wasmtypes "github.com/CosmWasm/wasmd/x/wasm/types" @@ -20,6 +21,7 @@ import ( type Keeper struct { stakingKeeper types.StakingKeeper wasmStorageKeeper types.WasmStorageKeeper + pubKeyKeeper types.PubKeyKeeper wasmKeeper wasmtypes.ContractOpsKeeper wasmViewKeeper wasmtypes.ViewKeeper validatorAddressCodec addresscodec.Codec @@ -40,6 +42,7 @@ func NewKeeper( authority string, sk types.StakingKeeper, wsk types.WasmStorageKeeper, + pkk types.PubKeyKeeper, wk wasmtypes.ContractOpsKeeper, wvk wasmtypes.ViewKeeper, validatorAddressCodec addresscodec.Codec, @@ -49,6 +52,7 @@ func NewKeeper( k := Keeper{ stakingKeeper: sk, wasmStorageKeeper: wsk, + pubKeyKeeper: pkk, wasmKeeper: wk, wasmViewKeeper: wvk, validatorAddressCodec: validatorAddressCodec, @@ -95,6 +99,28 @@ func (k Keeper) GetBatch(ctx context.Context, batchNum uint64) (types.Batch, err return batch, nil } +// GetPreviousDataResultRoot returns the previous batch's data result +// tree root in byte slice. If there is no previous batch, it returns +// an empty byte slice. +func (k Keeper) GetPreviousDataResultRoot(ctx context.Context) ([]byte, error) { + curBatchNum, err := k.GetCurrentBatchNum(ctx) + if err != nil { + return nil, err + } + if curBatchNum == 0 { + return []byte{}, err + } + batch, err := k.batch.Get(ctx, curBatchNum-1) + if err != nil { + return nil, err + } + root, err := hex.DecodeString(batch.DataResultRoot) + if err != nil { + return nil, err + } + return root, nil +} + // IterateBatches iterates over the batches and performs a given // callback function. func (k Keeper) IterateBatches(ctx sdk.Context, callback func(types.Batch) (stop bool)) error { diff --git a/x/batching/types/expected_keepers.go b/x/batching/types/expected_keepers.go index d92f6648..14fb7ab3 100644 --- a/x/batching/types/expected_keepers.go +++ b/x/batching/types/expected_keepers.go @@ -5,6 +5,7 @@ import ( abci "github.com/cometbft/cometbft/abci/types" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" sdk "github.com/cosmos/cosmos-sdk/types" stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" ) @@ -18,3 +19,7 @@ type StakingKeeper interface { type WasmStorageKeeper interface { GetCoreContractAddr(ctx context.Context) (sdk.AccAddress, error) } + +type PubKeyKeeper interface { + GetValidatorKeyAtIndex(ctx context.Context, validatorAddr sdk.ValAddress, index uint32) (cryptotypes.PubKey, error) +} diff --git a/x/pubkey/client/cli/tx.go b/x/pubkey/client/cli/tx.go index 1d3ee8b5..64350300 100644 --- a/x/pubkey/client/cli/tx.go +++ b/x/pubkey/client/cli/tx.go @@ -8,7 +8,6 @@ import ( "github.com/spf13/cobra" cmtcrypto "github.com/cometbft/cometbft/crypto" - "github.com/cometbft/cometbft/crypto/secp256k1" cmtjson "github.com/cometbft/cometbft/libs/json" cmtos "github.com/cometbft/cometbft/libs/os" @@ -20,6 +19,7 @@ import ( cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec" "github.com/cosmos/cosmos-sdk/server" + "github.com/sedaprotocol/seda-chain/cmd/sedad/utils" "github.com/sedaprotocol/seda-chain/x/pubkey/types" ) @@ -75,10 +75,7 @@ func AddKey(ac address.Codec) *cobra.Command { return err } } else { - pks, err = generateSEDAKeys( - []privKeyGenerator{secp256k1GenPrivKey}, - filepath.Dir(serverCfg.PrivValidatorKeyFile()), - ) + pks, err = generateSEDAKeys(filepath.Dir(serverCfg.PrivValidatorKeyFile())) if err != nil { return err } @@ -156,21 +153,15 @@ func saveSEDAKeys(keys []IndexedPrivKey, dirPath string) error { return nil } -type privKeyGenerator func() cmtcrypto.PrivKey - -func secp256k1GenPrivKey() cmtcrypto.PrivKey { - return secp256k1.GenPrivKey() -} - // generateSEDAKeys generates SEDA keys given a list of private key // generators, saves them to the SEDA key file, and returns the resulting // index-public key pairs. Index is assigned incrementally in the order // of the given private key generators. The key file is stored in the // directory given by dirPath. -func generateSEDAKeys(generators []privKeyGenerator, dirPath string) ([]types.IndexedPubKey, error) { - keys := make([]IndexedPrivKey, len(generators)) - result := make([]types.IndexedPubKey, len(generators)) - for i, generator := range generators { +func generateSEDAKeys(dirPath string) ([]types.IndexedPubKey, error) { + keys := make([]IndexedPrivKey, len(utils.SEDAKeysGenerators)) + result := make([]types.IndexedPubKey, len(utils.SEDAKeysGenerators)) + for i, generator := range utils.SEDAKeysGenerators { privKey := generator() keys[i] = IndexedPrivKey{ Index: uint32(i),