From fcf5d84e6a6cd98bc252df1af4eb51e697383689 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 15 Dec 2021 18:24:51 -0500 Subject: [PATCH] Fix Int indexes to make them sortable. (#114) --- go.mod | 2 +- go.sum | 1 - index.go | 47 ++++++++++++++----- index_test.go | 127 +++++++++++++++++++++++++++++++------------------- 4 files changed, 116 insertions(+), 61 deletions(-) diff --git a/go.mod b/go.mod index 242f5fa..1281d97 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/hashicorp/go-memdb -go 1.12 +go 1.13 require ( github.com/hashicorp/go-immutable-radix v1.3.0 diff --git a/go.sum b/go.sum index eaff521..2965e4b 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,6 @@ github.com/hashicorp/go-immutable-radix v1.3.0 h1:8exGP7ego3OmkfksihtSouGMZ+hQrh github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= diff --git a/index.go b/index.go index 7ff31ce..172a0e8 100644 --- a/index.go +++ b/index.go @@ -5,8 +5,8 @@ import ( "encoding/hex" "errors" "fmt" - "math/bits" "reflect" + "strconv" "strings" ) @@ -308,8 +308,7 @@ func (i *IntFieldIndex) FromObject(obj interface{}) (bool, []byte, error) { // Get the value and encode it val := fv.Int() - buf := make([]byte, size) - binary.PutVarint(buf, val) + buf := encodeInt(val, size) return true, buf, nil } @@ -331,26 +330,50 @@ func (i *IntFieldIndex) FromArgs(args ...interface{}) ([]byte, error) { } val := v.Int() - buf := make([]byte, size) - binary.PutVarint(buf, val) + buf := encodeInt(val, size) return buf, nil } +func encodeInt(val int64, size int) []byte { + buf := make([]byte, size) + + // This bit flips the sign bit on any sized signed twos-complement integer, + // which when truncated to a uint of the same size will bias the value such + // that the maximum negative int becomes 0, and the maximum positive int + // becomes the maximum positive uint. + scaled := val ^ int64(-1<<(size*8-1)) + + switch size { + case 1: + buf[0] = uint8(scaled) + case 2: + binary.BigEndian.PutUint16(buf, uint16(scaled)) + case 4: + binary.BigEndian.PutUint32(buf, uint32(scaled)) + case 8: + binary.BigEndian.PutUint64(buf, uint64(scaled)) + default: + panic(fmt.Sprintf("unsupported int size parameter: %d", size)) + } + + return buf +} + // IsIntType returns whether the passed type is a type of int and the number // of bytes needed to encode the type. func IsIntType(k reflect.Kind) (size int, okay bool) { switch k { case reflect.Int: - return binary.MaxVarintLen64, true + return strconv.IntSize / 8, true case reflect.Int8: - return 2, true + return 1, true case reflect.Int16: - return binary.MaxVarintLen16, true + return 2, true case reflect.Int32: - return binary.MaxVarintLen32, true + return 4, true case reflect.Int64: - return binary.MaxVarintLen64, true + return 8, true default: return 0, false } @@ -420,6 +443,8 @@ func encodeUInt(val uint64, size int) []byte { binary.BigEndian.PutUint32(buf, uint32(val)) case 8: binary.BigEndian.PutUint64(buf, val) + default: + panic(fmt.Sprintf("unsupported uint size parameter: %d", size)) } return buf @@ -430,7 +455,7 @@ func encodeUInt(val uint64, size int) []byte { func IsUintType(k reflect.Kind) (size int, okay bool) { switch k { case reflect.Uint: - return bits.UintSize / 8, true + return strconv.IntSize / 8, true case reflect.Uint8: return 1, true case reflect.Uint16: diff --git a/index_test.go b/index_test.go index efe3f53..303ecd9 100644 --- a/index_test.go +++ b/index_test.go @@ -578,7 +578,6 @@ func TestUUIDFieldIndex_PrefixFromArgs(t *testing.T) { if !bytes.Equal(uuidBuf[:9], val) { t.Fatalf("foo") } - } func BenchmarkUUIDFieldIndex_parseString(b *testing.B) { @@ -609,16 +608,16 @@ func generateUUID() ([]byte, string) { func TestIntFieldIndex_FromObject(t *testing.T) { obj := testObj() - eint := make([]byte, 10) - eint8 := make([]byte, 2) - eint16 := make([]byte, 3) - eint32 := make([]byte, 5) - eint64 := make([]byte, 10) - binary.PutVarint(eint, int64(obj.Int)) - binary.PutVarint(eint8, int64(obj.Int8)) - binary.PutVarint(eint16, int64(obj.Int16)) - binary.PutVarint(eint32, int64(obj.Int32)) - binary.PutVarint(eint64, obj.Int64) + eint := make([]byte, 8) + eint8 := make([]byte, 1) + eint16 := make([]byte, 2) + eint32 := make([]byte, 4) + eint64 := make([]byte, 8) + binary.BigEndian.PutUint64(eint, 1<<63+1) + eint8[0] = 0 + binary.BigEndian.PutUint16(eint16, 0) + binary.BigEndian.PutUint32(eint32, 0) + binary.BigEndian.PutUint64(eint64, 0) cases := []struct { Field string @@ -659,7 +658,6 @@ func TestIntFieldIndex_FromObject(t *testing.T) { t.Run(c.Field, func(t *testing.T) { indexer := IntFieldIndex{c.Field} ok, val, err := indexer.FromObject(obj) - if err != nil { if ok { t.Fatalf("okay and error") @@ -679,7 +677,6 @@ func TestIntFieldIndex_FromObject(t *testing.T) { if !bytes.Equal(val, c.Expected) { t.Fatalf("bad: %#v %#v", val, c.Expected) } - }) } } @@ -702,16 +699,16 @@ func TestIntFieldIndex_FromArgs(t *testing.T) { } obj := testObj() - eint := make([]byte, 10) - eint8 := make([]byte, 2) - eint16 := make([]byte, 3) - eint32 := make([]byte, 5) - eint64 := make([]byte, 10) - binary.PutVarint(eint, int64(obj.Int)) - binary.PutVarint(eint8, int64(obj.Int8)) - binary.PutVarint(eint16, int64(obj.Int16)) - binary.PutVarint(eint32, int64(obj.Int32)) - binary.PutVarint(eint64, obj.Int64) + eint := make([]byte, 8) + eint8 := make([]byte, 1) + eint16 := make([]byte, 2) + eint32 := make([]byte, 4) + eint64 := make([]byte, 8) + binary.BigEndian.PutUint64(eint, 1<<63+1) + eint8[0] = 0 + binary.BigEndian.PutUint16(eint16, 0) + binary.BigEndian.PutUint32(eint32, 0) + binary.BigEndian.PutUint64(eint64, 0) val, err := indexer.FromArgs(obj.Int) if err != nil { @@ -754,6 +751,46 @@ func TestIntFieldIndex_FromArgs(t *testing.T) { } } +func TestIntFieldIndexSortability(t *testing.T) { + testCases := []struct { + i8l int8 + i8r int8 + i16l int16 + i16r int16 + i32l int32 + i32r int32 + i64l int64 + i64r int64 + il int + ir int + expected int + name string + }{ + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "zero"}, + {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, "small eq"}, + {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, -1, "small lt"}, + {2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 1, "small gt"}, + {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, "small neg eq"}, + {-2, -1, -2, -1, -2, -1, -2, -1, -2, -1, -1, "small neg lt"}, + {-1, -2, -1, -2, -1, -2, -1, -2, -1, -2, 1, "small neg gt"}, + {-1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, "neg vs pos"}, + {-128, 127, -32768, 32767, -2147483648, 2147483647, -9223372036854775808, 9223372036854775807, -9223372036854775808, 9223372036854775807, -1, "max conditions"}, + {100, 127, 1000, 2000, 1000000000, 2000000000, 10000000000, 20000000000, 1000000000, 2000000000, -1, "large lt"}, + {100, 99, 1000, 999, 1000000000, 999999999, 10000000000, 9999999999, 1000000000, 999999999, 1, "large gt"}, + {126, 127, 255, 256, 65535, 65536, 4294967295, 4294967296, 65535, 65536, -1, "edge conditions"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + compareEncoded(t, &IntFieldIndex{"Foo"}, tc.i8l, tc.i8r, tc.expected) + compareEncoded(t, &IntFieldIndex{"Foo"}, tc.i16l, tc.i16r, tc.expected) + compareEncoded(t, &IntFieldIndex{"Foo"}, tc.i32l, tc.i32r, tc.expected) + compareEncoded(t, &IntFieldIndex{"Foo"}, tc.i64l, tc.i64r, tc.expected) + compareEncoded(t, &IntFieldIndex{"Foo"}, tc.il, tc.ir, tc.expected) + }) + } +} + func TestUintFieldIndex_FromObject(t *testing.T) { obj := testObj() @@ -807,7 +844,6 @@ func TestUintFieldIndex_FromObject(t *testing.T) { t.Run(c.Field, func(t *testing.T) { indexer := UintFieldIndex{c.Field} ok, val, err := indexer.FromObject(obj) - if err != nil { if ok { t.Fatalf("okay and error") @@ -827,7 +863,6 @@ func TestUintFieldIndex_FromObject(t *testing.T) { if !bytes.Equal(val, c.Expected) { t.Fatalf("bad: %#v %#v", val, c.Expected) } - }) } } @@ -928,25 +963,23 @@ func TestUIntFieldIndexSortability(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - compareEncoded(t, tc.u8l, tc.u8r, tc.expected) - compareEncoded(t, tc.u16l, tc.u16r, tc.expected) - compareEncoded(t, tc.u32l, tc.u32r, tc.expected) - compareEncoded(t, tc.u64l, tc.u64r, tc.expected) - compareEncoded(t, tc.ul, tc.ur, tc.expected) + compareEncoded(t, &UintFieldIndex{"Foo"}, tc.u8l, tc.u8r, tc.expected) + compareEncoded(t, &UintFieldIndex{"Foo"}, tc.u16l, tc.u16r, tc.expected) + compareEncoded(t, &UintFieldIndex{"Foo"}, tc.u32l, tc.u32r, tc.expected) + compareEncoded(t, &UintFieldIndex{"Foo"}, tc.u64l, tc.u64r, tc.expected) + compareEncoded(t, &UintFieldIndex{"Foo"}, tc.ul, tc.ur, tc.expected) }) } } -func compareEncoded(t *testing.T, l interface{}, r interface{}, expected int) { - indexer := UintFieldIndex{"Foo"} - +func compareEncoded(t *testing.T, indexer Indexer, l interface{}, r interface{}, expected int) { lBytes, err := indexer.FromArgs(l) if err != nil { - t.Fatalf("unable to encode: %d", l) + t.Fatalf("unable to encode %d: %s", l, err) } rBytes, err := indexer.FromArgs(r) if err != nil { - t.Fatalf("unable to encode: %d", r) + t.Fatalf("unable to encode %d: %s", r, err) } if bytes.Compare(lBytes, rBytes) != expected { @@ -1113,21 +1146,19 @@ func TestFieldSetIndex_FromArgs(t *testing.T) { } } -var ( - // A conditional that checks if TestObject.Bar == 42 - conditional = func(obj interface{}) (bool, error) { - test, ok := obj.(*TestObject) - if !ok { - return false, fmt.Errorf("Expect only TestObj types") - } - - if test.Bar != 42 { - return false, nil - } +// A conditional that checks if TestObject.Bar == 42 +var conditional = func(obj interface{}) (bool, error) { + test, ok := obj.(*TestObject) + if !ok { + return false, fmt.Errorf("Expect only TestObj types") + } - return true, nil + if test.Bar != 42 { + return false, nil } -) + + return true, nil +} func TestConditionalIndex_FromObject(t *testing.T) { obj := testObj()