From 26de5dd8a5dc863f13c6765a09d6602cff2e76b8 Mon Sep 17 00:00:00 2001 From: krhubert Date: Tue, 3 Sep 2019 15:16:48 +0200 Subject: [PATCH 1/2] Replace cnf structhash with custom one --- go.mod | 6 +- go.sum | 2 - hash/hash.go | 4 +- hash/structhash/structhash.go | 211 +++++++++++++++++++++++++++++ hash/structhash/structhash_test.go | 202 +++++++++++++++++++++++++++ 5 files changed, 416 insertions(+), 9 deletions(-) create mode 100644 hash/structhash/structhash.go create mode 100644 hash/structhash/structhash_test.go diff --git a/go.mod b/go.mod index ca9e04fcc..e84d06ac8 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/Microsoft/go-winio v0.4.12 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/btcsuite/btcd v0.0.0-20190807005414-4063feeff79a // indirect - github.com/cnf/structhash v0.0.0-20180104161610-62a607eb0224 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect github.com/cosmos/cosmos-sdk v0.36.0 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d @@ -38,8 +37,6 @@ require ( github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-isatty v0.0.8 // indirect github.com/mitchellh/go-homedir v1.1.0 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/mr-tron/base58 v1.1.1 github.com/mwitkow/go-proto-validators v0.0.0-20190212092829-1f388280e944 // indirect github.com/onsi/ginkgo v1.8.0 // indirect @@ -65,12 +62,11 @@ require ( github.com/tendermint/tendermint v0.32.2 github.com/tendermint/tm-db v0.1.1 github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5 - golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect + golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect golang.org/x/text v0.3.2 // indirect golang.org/x/tools v0.0.0-20190813142322-97f12d73768f // indirect - google.golang.org/appengine v1.4.0 // indirect google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64 // indirect google.golang.org/grpc v1.23.0 gopkg.in/go-playground/assert.v1 v1.2.1 // indirect diff --git a/go.sum b/go.sum index a7219cbfc..86541e96d 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,6 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cnf/structhash v0.0.0-20180104161610-62a607eb0224 h1:rnCKRrdSBqc061l0CDuYB+7X3w6w8IK/VCSChJXv62g= -github.com/cnf/structhash v0.0.0-20180104161610-62a607eb0224/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= diff --git a/hash/hash.go b/hash/hash.go index a7e212770..1d7572374 100644 --- a/hash/hash.go +++ b/hash/hash.go @@ -9,7 +9,7 @@ import ( "fmt" "hash" - "github.com/cnf/structhash" + "github.com/mesg-foundation/engine/hash/structhash" "github.com/mr-tron/base58" ) @@ -32,7 +32,7 @@ func (d *Digest) Sum(b []byte) Hash { // WriteObject adds an interface data to the running hash. // It never retruns an error. func (d *Digest) WriteObject(v interface{}) (int, error) { - return d.Write(structhash.Dump(v, 0)) + return d.Write(structhash.Dump(v)) } // A Hash is a type for representing common hash. diff --git a/hash/structhash/structhash.go b/hash/structhash/structhash.go new file mode 100644 index 000000000..9516eb48d --- /dev/null +++ b/hash/structhash/structhash.go @@ -0,0 +1,211 @@ +package structhash + +import ( + "bytes" + "crypto/md5" + "crypto/sha1" + "fmt" + "reflect" + "sort" + "strconv" + "strings" + "sync" + + "golang.org/x/crypto/sha3" +) + +var bufPool = sync.Pool{ + New: func() interface{} { + return &bytes.Buffer{} + }, +} + +// Sha1 takes a data structure and returns its md5 hash as string. +func Sha1(v interface{}) [sha1.Size]byte { + return sha1.Sum(serialize(v)) +} + +// Sha1 takes a data structure and returns its md5 hash as string. +func Sha3(v interface{}) [64]byte { + return sha3.Sum512(serialize(v)) +} + +// Md5 takes a data structure and returns its md5 hash. +func Md5(v interface{}) [md5.Size]byte { + return md5.Sum(serialize(v)) +} + +// Dump takes a data structure and returns its string representation. +func Dump(v interface{}) []byte { + return serialize(v) +} + +func serialize(v interface{}) []byte { + return []byte(valueToString(reflect.ValueOf(v))) +} + +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Complex64, reflect.Complex128: + c := v.Complex() + return real(c) == 0 && imag(c) == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + return false +} + +func valueToString(v reflect.Value) string { + buf := bufPool.Get().(*bytes.Buffer) + s := write(buf, v).String() + buf.Reset() + bufPool.Put(buf) + return s +} + +//nolint:gocyclo +func write(buf *bytes.Buffer, v reflect.Value) *bytes.Buffer { + switch v.Kind() { + case reflect.Bool: + buf.WriteString(strconv.FormatBool(v.Bool())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + buf.WriteString(strconv.FormatInt(v.Int(), 16)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + buf.WriteString(strconv.FormatUint(v.Uint(), 16)) + case reflect.Float32, reflect.Float64: + buf.WriteString(strconv.FormatFloat(v.Float(), 'e', -1, 64)) + case reflect.String: + buf.WriteString("\"" + v.String() + "\"") + case reflect.Interface: + if v.IsNil() { + buf.WriteString("nil") + return buf + } + if v.CanInterface() { + write(buf, v.Elem()) + } + case reflect.Struct: + vt := v.Type() + items := make([]string, 0) + for i := 0; i < v.NumField(); i++ { + sf := vt.Field(i) + to := parseTag(sf) + if to.skip || to.omitempty && isEmptyValue(v.Field(i)) { + continue + } + + str := valueToString(v.Field(i)) + // if field string == "" then it is chan,func or invalid type + // and skip it + if str == "" { + continue + } + items = append(items, to.name+":"+str) + } + sort.Strings(items) + + buf.WriteByte('{') + for i := range items { + if i != 0 { + buf.WriteByte(',') + } + buf.WriteString(items[i]) + } + buf.WriteByte('}') + case reflect.Map: + if v.IsNil() { + buf.WriteString("()nil") + return buf + } + buf.WriteByte('(') + + keys := v.MapKeys() + items := make([]string, len(keys)) + + // Extract and sort the keys. + for i, key := range keys { + items[i] = valueToString(key) + ":" + valueToString(v.MapIndex(key)) + } + sort.Strings(items) + + for i := range items { + if i > 0 { + buf.WriteByte(',') + } + buf.WriteString(items[i]) + } + buf.WriteByte(')') + case reflect.Slice: + if v.IsNil() { + buf.WriteString("[]nil") + return buf + } + fallthrough + case reflect.Array: + buf.WriteByte('[') + for i := 0; i < v.Len(); i++ { + if i != 0 { + buf.WriteByte(',') + } + write(buf, v.Index(i)) + } + buf.WriteByte(']') + case reflect.Ptr: + if v.IsNil() { + buf.WriteString("nil") + return buf + } + write(buf, v.Elem()) + case reflect.Complex64, reflect.Complex128: + c := v.Complex() + buf.WriteString(strconv.FormatFloat(real(c), 'e', -1, 64)) + buf.WriteString(strconv.FormatFloat(imag(c), 'e', -1, 64)) + buf.WriteString("i") + } + return buf +} + +// tagOptions is the string struct field's "hash" +type tagOptions struct { + name string + omitempty bool + skip bool +} + +// parseTag splits a struct field's hash tag into its name and +// comma-separated options. +func parseTag(f reflect.StructField) tagOptions { + tag := f.Tag.Get("hash") + if tag == "" || tag == "-" { + return tagOptions{skip: true} + } + + var to tagOptions + options := strings.Split(tag, ",") + for _, option := range options { + switch { + case option == "omitempty": + to.omitempty = true + case strings.HasPrefix(option, "name:"): + to.name = option[len("name:"):] + default: + panic(fmt.Sprintf("structhash: field %s with tag hash:%q has invalid option %q", f.Name, tag, option)) + } + } + + // skip fields without name + if to.name == "" { + return tagOptions{skip: true} + } + return to +} diff --git a/hash/structhash/structhash_test.go b/hash/structhash/structhash_test.go new file mode 100644 index 000000000..9547cbe0d --- /dev/null +++ b/hash/structhash/structhash_test.go @@ -0,0 +1,202 @@ +package structhash + +import ( + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSha1(t *testing.T) { + assert.Equal(t, "da39a3ee5e6b4b0d3255bfef95601890afd80709", fmt.Sprintf("%x", Sha1(nil))) +} + +func TestSha3(t *testing.T) { + assert.Equal(t, "a69f73cca23a9ac5c8b567dc185a756e97c982164fe25859e0d1dcc1475c80a615b2123af1f5f94c11e3e9402c3ac558f500199d95b6d3e301758586281dcd26", fmt.Sprintf("%x", Sha3(nil))) +} + +func TestMd5(t *testing.T) { + assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", fmt.Sprintf("%x", Md5(nil))) +} + +//nolint:megacheck +func TestDump(t *testing.T) { + tests := []struct { + v interface{} + s string + }{ + {nil, ""}, + {make(chan string), ""}, + {func() {}, ""}, + {false, "false"}, + {(*int)(nil), "nil"}, + {int(0), "0"}, + {uint(0), "0"}, + {0.0, "0e+00"}, + {"", `""`}, + {interface{}(0), "0"}, + {map[int]int(nil), "()nil"}, + {map[int]int{}, "()"}, + {map[int]int{0: 0}, "(0:0)"}, + {map[int]int{0: 0, 1: 1}, "(0:0,1:1)"}, + {[]int(nil), "[]nil"}, + {[]*int{nil}, "[nil]"}, + {[]int{}, "[]"}, + {[]int{0}, "[0]"}, + {[]int{0, 1}, "[0,1]"}, + {[0]int{}, "[]"}, + {[1]int{0}, "[0]"}, + {[2]int{0, 1}, "[0,1]"}, + {complex(0, 0), "0e+000e+00i"}, + {(*struct{})(nil), "nil"}, + {struct{}{}, "{}"}, + { + struct { + a chan int `hash:"name:a"` + }{}, + "{}", + }, + { + struct { + a int `hash:"omitempty"` + }{1}, + "{}", + }, + { + struct { + a interface{} `hash:"name:a"` + }{nil}, + "{a:nil}", + }, + { + struct { + A interface{} `hash:"name:A"` + }{0}, + "{A:0}", + }, + { + struct { + a int `hash:"name:a"` + }{0}, + "{a:0}", + }, + { + struct { + a int `hash:"name:a"` + b int `hash:"name:b"` + }{0, 1}, + "{a:0,b:1}", + }, + { + struct { + a struct { + a bool `hash:"name:a"` + } `hash:"name:a"` + }{a: struct { + a bool `hash:"name:a"` + }{a: false}}, + "{a:{a:false}}", + }, + { + struct { + a *struct { + b bool `hash:"name:b"` + } `hash:"name:a"` + }{a: &struct { + b bool `hash:"name:b"` + }{b: false}}, + "{a:{b:false}}", + }, + } + + for _, tt := range tests { + s := Dump(tt.v) + assert.Equalf(t, []byte(tt.s), s, "type %s: %v", reflect.TypeOf(tt.v), tt.v) + } +} + +//nolint:megacheck +func TestTag(t *testing.T) { + tests := []struct { + v interface{} + s string + }{ + { + struct { + a int `hash:"-"` + b uint `hash:"-"` + c bool `hash:"-"` + d string `hash:"-"` + e []int `hash:"-"` + f float64 `hash:"-"` + g complex128 `hash:"-"` + h interface{} `hash:"-"` + i *struct{} `hash:"-"` + j *[]uint `hash:"-"` + k chan int `hash:"-"` + }{}, + "{}", + }, + { + struct { + a int `hash:"name:a,omitempty"` + b uint `hash:"name:b,omitempty"` + c bool `hash:"name:c,omitempty"` + d string `hash:"name:d,omitempty"` + e []int `hash:"name:e,omitempty"` + f float64 `hash:"name:f,omitempty"` + g complex128 `hash:"name:g,omitempty"` + h interface{} `hash:"name:h,omitempty"` + i *struct{} `hash:"name:i,omitempty"` + j *[]uint `hash:"name:j,omitempty"` + k chan int `hash:"name:k,omitempty"` + }{}, + "{}", + }, + { + struct { + a int `hash:"name:b"` + }{}, + "{b:0}", + }, + { + struct { + a int `hash:"name:b"` + b int `hash:"name:a"` + }{0, 1}, + "{a:1,b:0}", + }, + { + struct { + a int `hash:"name:b,omitempty"` + b int `hash:"name:a,omitempty"` + }{0, 1}, + "{a:1}", + }, + { + struct { + a int + b int `hash:"-"` + c int `hash:"name:c"` + }{1, 2, 3}, + "{c:3}", + }, + } + + for _, tt := range tests { + assert.Equalf(t, []byte(tt.s), serialize(tt.v), "type %s: %v", reflect.TypeOf(tt.v), tt.v) + } +} + +func TestTagPanicInvalidOption(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("serialize did not panic with invalid option") + } + }() + + serialize(struct { + a int `hash:"name:a,omitempty,invalid"` + }{0}) +} From 1a9a273ec45764deed17fcdb6e4f24669ceb527a Mon Sep 17 00:00:00 2001 From: krhubert Date: Wed, 4 Sep 2019 07:55:47 +0200 Subject: [PATCH 2/2] Fix comments for linter --- hash/structhash/structhash.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hash/structhash/structhash.go b/hash/structhash/structhash.go index 9516eb48d..3b129985c 100644 --- a/hash/structhash/structhash.go +++ b/hash/structhash/structhash.go @@ -20,12 +20,12 @@ var bufPool = sync.Pool{ }, } -// Sha1 takes a data structure and returns its md5 hash as string. +// Sha1 takes a data structure and returns its sha1 hash as string. func Sha1(v interface{}) [sha1.Size]byte { return sha1.Sum(serialize(v)) } -// Sha1 takes a data structure and returns its md5 hash as string. +// Sha3 takes a data structure and returns its sha3 hash as string. func Sha3(v interface{}) [64]byte { return sha3.Sum512(serialize(v)) }