From 97e1c31d40667d1020d39de747353725cce7ea7e Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 27 Apr 2020 17:07:58 +0200 Subject: [PATCH] store: simplehashfrommap (#6069) * bring simplehashfrommap from TM * bring simplemap as well * rename simplemap to merklemap * reduce byte to 8 * Update APIs and godoc Co-authored-by: Alexander Bezobchuk Co-authored-by: Aleksandr Bezobchuk --- store/rootmulti/merkle_map.go | 112 +++++++++++++++++++++++++++++ store/rootmulti/merkle_map_test.go | 50 +++++++++++++ store/rootmulti/store.go | 17 +++-- store/rootmulti/store_test.go | 3 +- 4 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 store/rootmulti/merkle_map.go create mode 100644 store/rootmulti/merkle_map_test.go diff --git a/store/rootmulti/merkle_map.go b/store/rootmulti/merkle_map.go new file mode 100644 index 000000000000..3f5d2ff4c9fb --- /dev/null +++ b/store/rootmulti/merkle_map.go @@ -0,0 +1,112 @@ +package rootmulti + +import ( + "bytes" + "encoding/binary" + "io" + + "github.com/tendermint/tendermint/crypto/merkle" + "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/libs/kv" +) + +// merkleMap defines a merkle-ized tree from a map. Leave values are treated as +// hash(key) | hash(value). Leaves are sorted before Merkle hashing. +type merkleMap struct { + kvs kv.Pairs + sorted bool +} + +func newMerkleMap() *merkleMap { + return &merkleMap{ + kvs: nil, + sorted: false, + } +} + +// set creates a kv.Pair from the provided key and value. The value is hashed prior +// to creating a kv.Pair. The created kv.Pair is appended to the merkleMap's slice +// of kv.Pairs. Whenever called, the merkleMap must be resorted. +func (sm *merkleMap) set(key string, value []byte) { + sm.sorted = false + + // The value is hashed, so you can check for equality with a cached value (say) + // and make a determination to fetch or not. + vhash := tmhash.Sum(value) + + sm.kvs = append(sm.kvs, kv.Pair{ + Key: []byte(key), + Value: vhash, + }) +} + +// hash returns the merkle root of items sorted by key. Note, it is unstable. +func (sm *merkleMap) hash() []byte { + sm.sort() + return hashKVPairs(sm.kvs) +} + +func (sm *merkleMap) sort() { + if sm.sorted { + return + } + + sm.kvs.Sort() + sm.sorted = true +} + +// kvPairs sorts the merkleMap kv.Pair objects and returns a copy as a slice. +func (sm *merkleMap) kvPairs() kv.Pairs { + sm.sort() + + kvs := make(kv.Pairs, len(sm.kvs)) + copy(kvs, sm.kvs) + + return kvs +} + +// kvPair defines a type alias for kv.Pair so that we can create bytes to hash +// when constructing the merkle root. Note, key and values are both length-prefixed. +type kvPair kv.Pair + +// bytes returns a byte slice representation of the kvPair where the key and value +// are length-prefixed. +func (kv kvPair) bytes() []byte { + var b bytes.Buffer + + err := encodeByteSlice(&b, kv.Key) + if err != nil { + panic(err) + } + + err = encodeByteSlice(&b, kv.Value) + if err != nil { + panic(err) + } + + return b.Bytes() +} + +func encodeByteSlice(w io.Writer, bz []byte) error { + var buf [8]byte + n := binary.PutUvarint(buf[:], uint64(len(bz))) + + _, err := w.Write(buf[:n]) + if err != nil { + return err + } + + _, err = w.Write(bz) + return err +} + +// hashKVPairs hashes a kvPair and creates a merkle tree where the leaves are +// byte slices. +func hashKVPairs(kvs kv.Pairs) []byte { + kvsH := make([][]byte, len(kvs)) + for i, kvp := range kvs { + kvsH[i] = kvPair(kvp).bytes() + } + + return merkle.SimpleHashFromByteSlices(kvsH) +} diff --git a/store/rootmulti/merkle_map_test.go b/store/rootmulti/merkle_map_test.go new file mode 100644 index 000000000000..06f02a62bc5b --- /dev/null +++ b/store/rootmulti/merkle_map_test.go @@ -0,0 +1,50 @@ +package rootmulti + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSimpleMap(t *testing.T) { + tests := []struct { + keys []string + values []string // each string gets converted to []byte in test + want string + }{ + {[]string{"key1"}, []string{"value1"}, "a44d3cc7daba1a4600b00a2434b30f8b970652169810d6dfa9fb1793a2189324"}, + {[]string{"key1"}, []string{"value2"}, "0638e99b3445caec9d95c05e1a3fc1487b4ddec6a952ff337080360b0dcc078c"}, + // swap order with 2 keys + { + []string{"key1", "key2"}, + []string{"value1", "value2"}, + "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3", + }, + { + []string{"key2", "key1"}, + []string{"value2", "value1"}, + "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3", + }, + // swap order with 3 keys + { + []string{"key1", "key2", "key3"}, + []string{"value1", "value2", "value3"}, + "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc", + }, + { + []string{"key1", "key3", "key2"}, + []string{"value1", "value3", "value2"}, + "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc", + }, + } + for i, tc := range tests { + db := newMerkleMap() + for i := 0; i < len(tc.keys); i++ { + db.set(tc.keys[i], []byte(tc.values[i])) + } + + got := db.hash() + assert.Equal(t, tc.want, fmt.Sprintf("%x", got), "Hash didn't match on tc %d", i) + } +} diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 032da04416e2..bc59f62316ee 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -6,9 +6,7 @@ import ( "strings" "github.com/pkg/errors" - abci "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/crypto/merkle" "github.com/tendermint/tendermint/crypto/tmhash" dbm "github.com/tendermint/tm-db" @@ -571,7 +569,7 @@ func (ci commitInfo) Hash() []byte { m[storeInfo.Name] = storeInfo.Hash() } - return merkle.SimpleHashFromMap(m) + return SimpleHashFromMap(m) } func (ci commitInfo) CommitID() types.CommitID { @@ -600,7 +598,7 @@ type storeCore struct { // Implements merkle.Hasher. func (si storeInfo) Hash() []byte { - // Doesn't write Name, since merkle.SimpleHashFromMap() will + // Doesn't write Name, since SimpleHashFromMap() will // include them via the keys. bz := si.Core.CommitID.Hash hasher := tmhash.New() @@ -705,3 +703,14 @@ func flushCommitInfo(db dbm.DB, version int64, cInfo commitInfo) { panic(fmt.Errorf("error on batch write %w", err)) } } + +// SimpleHashFromMap computes a merkle tree from sorted map and returns the merkle +// root. +func SimpleHashFromMap(m map[string][]byte) []byte { + mm := newMerkleMap() + for k, v := range m { + mm.set(k, v) + } + + return mm.hash() +} diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go index f798c4c76d27..a6f40dde3805 100644 --- a/store/rootmulti/store_test.go +++ b/store/rootmulti/store_test.go @@ -6,7 +6,6 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/crypto/merkle" dbm "github.com/tendermint/tm-db" "github.com/cosmos/cosmos-sdk/store/iavl" @@ -519,5 +518,5 @@ func hashStores(stores map[types.StoreKey]types.CommitKVStore) []byte { }, }.Hash() } - return merkle.SimpleHashFromMap(m) + return SimpleHashFromMap(m) }