From 829b460281eea3dc411dd60297753b3cccc2b26e Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 25 May 2022 14:53:07 +0900 Subject: [PATCH 01/29] refactor charset helper func Signed-off-by: kevindiu --- .../core/ngt/handler/grpc/handler_test.go | 128 ++++++++---------- 1 file changed, 53 insertions(+), 75 deletions(-) diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index eae927b0d2..522d922985 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -20,7 +20,6 @@ package grpc import ( "context" "fmt" - "io/ioutil" "math" "reflect" "strconv" @@ -38,12 +37,11 @@ import ( "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/test/data/charset" "github.com/vdaas/vald/internal/test/data/vector" "github.com/vdaas/vald/internal/test/goleak" "github.com/vdaas/vald/pkg/agent/core/ngt/model" "github.com/vdaas/vald/pkg/agent/core/ngt/service" - "golang.org/x/text/encoding/japanese" - "golang.org/x/text/transform" ) func TestNew(t *testing.T) { @@ -542,7 +540,7 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.2: fail exists with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), searchId: "こんにちは", }, want: want{ @@ -553,7 +551,7 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.3: fail exists with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), searchId: "こんにちは", }, want: want{ @@ -565,7 +563,7 @@ func Test_server_Exists(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - searchId: utf8ToSjis("こんにちは"), + searchId: charset.Utf8ToSjis("こんにちは"), }, want: want{ code: codes.NotFound, @@ -575,8 +573,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.5: success exists with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), - searchId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), + searchId: charset.Utf8ToSjis("こんにちは"), }, want: want{}, }, @@ -584,8 +582,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.6: fail exists with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), - searchId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), + searchId: charset.Utf8ToSjis("こんにちは"), }, want: want{ code: codes.NotFound, @@ -596,7 +594,7 @@ func Test_server_Exists(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - searchId: utf8ToEucjp("こんにちは"), + searchId: charset.Utf8ToEucjp("こんにちは"), }, want: want{ code: codes.NotFound, @@ -606,8 +604,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.8: fail exists with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), - searchId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), + searchId: charset.Utf8ToEucjp("こんにちは"), }, want: want{ code: codes.NotFound, @@ -617,8 +615,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.9: success exists with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), - searchId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), + searchId: charset.Utf8ToEucjp("こんにちは"), }, want: want{}, }, @@ -1509,16 +1507,6 @@ func Test_server_Search(t *testing.T) { } } -func utf8ToSjis(s string) string { - b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())) - return string(b) -} - -func utf8ToEucjp(s string) string { - b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.EUCJP.NewEncoder())) - return string(b) -} - func Test_server_SearchByID(t *testing.T) { t.Parallel() @@ -1759,7 +1747,7 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.2: fail search with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), searchId: "こんにちは", }, want: want{ @@ -1771,7 +1759,7 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.3: fail search with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), searchId: "こんにちは", }, want: want{ @@ -1784,7 +1772,7 @@ func Test_server_SearchByID(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - searchId: utf8ToSjis("こんにちは"), + searchId: charset.Utf8ToSjis("こんにちは"), }, want: want{ resultSize: 0, @@ -1795,8 +1783,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.5: success search with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), - searchId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), + searchId: charset.Utf8ToSjis("こんにちは"), }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -1806,8 +1794,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.6: fail search with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), - searchId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), + searchId: charset.Utf8ToSjis("こんにちは"), }, want: want{ resultSize: 0, @@ -1819,7 +1807,7 @@ func Test_server_SearchByID(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - searchId: utf8ToEucjp("こんにちは"), + searchId: charset.Utf8ToEucjp("こんにちは"), }, want: want{ resultSize: 0, @@ -1830,8 +1818,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.8: fail search with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), - searchId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), + searchId: charset.Utf8ToEucjp("こんにちは"), }, want: want{ resultSize: 0, @@ -1842,8 +1830,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.9: success search with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), - searchId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), + searchId: charset.Utf8ToEucjp("こんにちは"), }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -7061,7 +7049,7 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.2: success update with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "こんにちは", @@ -7078,7 +7066,7 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.3: success update with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "こんにちは", @@ -7098,7 +7086,7 @@ func Test_server_Update(t *testing.T) { indexId: "こんにちは", req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: utf8ToSjis("こんにちは"), + Id: charset.Utf8ToSjis("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7112,27 +7100,27 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.5: success update with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: utf8ToSjis("こんにちは"), + Id: charset.Utf8ToSjis("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, }, }, want: want{ - wantUuid: utf8ToSjis("こんにちは"), + wantUuid: charset.Utf8ToSjis("こんにちは"), }, }, { name: "Boundary Value Testing case 3.6: fail update with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: utf8ToSjis("こんにちは"), + Id: charset.Utf8ToSjis("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7149,7 +7137,7 @@ func Test_server_Update(t *testing.T) { indexId: "こんにちは", req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: utf8ToEucjp("こんにちは"), + Id: charset.Utf8ToEucjp("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7163,10 +7151,10 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.8: fail update with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: utf8ToEucjp("こんにちは"), + Id: charset.Utf8ToEucjp("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7180,17 +7168,17 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.9: success update with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: utf8ToEucjp("こんにちは"), + Id: charset.Utf8ToEucjp("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, }, }, want: want{ - wantUuid: utf8ToEucjp("こんにちは"), + wantUuid: charset.Utf8ToEucjp("こんにちは"), }, }, { @@ -8198,16 +8186,6 @@ func Test_server_Remove(t *testing.T) { return s, nil } - utf8ToSjis := func(s string) string { - b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())) - return string(b) - } - - utf8ToEucjp := func(s string) string { - b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.EUCJP.NewEncoder())) - return string(b) - } - /* Remove test cases ( focus on ID(string), only test float32 ): - Equivalence Class Testing ( 1000 vectors inserted before a search ) @@ -8349,7 +8327,7 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.2: fail exists with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), removeId: "こんにちは", }, want: want{ @@ -8360,7 +8338,7 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.3: fail exists with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), removeId: "こんにちは", }, want: want{ @@ -8372,7 +8350,7 @@ func Test_server_Remove(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - removeId: utf8ToSjis("こんにちは"), + removeId: charset.Utf8ToSjis("こんにちは"), }, want: want{ code: codes.NotFound, @@ -8382,19 +8360,19 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.5: success exists with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), - removeId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), + removeId: charset.Utf8ToSjis("こんにちは"), }, want: want{ - wantUuid: utf8ToSjis("こんにちは"), + wantUuid: charset.Utf8ToSjis("こんにちは"), }, }, { name: "Boundary Value Testing case 3.6: fail exists with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), - removeId: utf8ToSjis("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), + removeId: charset.Utf8ToSjis("こんにちは"), }, want: want{ code: codes.NotFound, @@ -8405,7 +8383,7 @@ func Test_server_Remove(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - removeId: utf8ToEucjp("こんにちは"), + removeId: charset.Utf8ToEucjp("こんにちは"), }, want: want{ code: codes.NotFound, @@ -8415,8 +8393,8 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.8: fail exists with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: utf8ToSjis("こんにちは"), - removeId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToSjis("こんにちは"), + removeId: charset.Utf8ToEucjp("こんにちは"), }, want: want{ code: codes.NotFound, @@ -8426,11 +8404,11 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.9: success exists with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: utf8ToEucjp("こんにちは"), - removeId: utf8ToEucjp("こんにちは"), + indexId: charset.Utf8ToEucjp("こんにちは"), + removeId: charset.Utf8ToEucjp("こんにちは"), }, want: want{ - wantUuid: utf8ToEucjp("こんにちは"), + wantUuid: charset.Utf8ToEucjp("こんにちは"), }, }, { From 743b8b641b0ced84b559ef87c4da39879623dbe4 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 25 May 2022 15:07:15 +0900 Subject: [PATCH 02/29] refactor vector helper func Signed-off-by: kevindiu --- .../core/ngt/handler/grpc/handler_test.go | 114 ++++++------------ 1 file changed, 35 insertions(+), 79 deletions(-) diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 522d922985..026ab883c4 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -2443,22 +2443,6 @@ func Test_server_Insert(t *testing.T) { vqueueCfg = &config.VQueue{} ) - // functions to generator vectors for testing - genF32Vec := func(dist vector.Distribution, dim int) []float32 { - generator, _ := vector.Float32VectorGenerator(dist) - return generator(1, dim)[0] - } - genIntVec := func(dist vector.Distribution, dim int) []float32 { - generator, _ := vector.Uint8VectorGenerator(dist) - ivec := generator(1, dim)[0] - - vec := make([]float32, dim) - for i := 0; i < dim; i++ { - vec[i] = float32(ivec[i]) - } - return vec - } - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2584,7 +2568,7 @@ func Test_server_Insert(t *testing.T) { }(), func() test { invalidDim := intVecDim + 1 - ivec := genIntVec(vector.Gaussian, invalidDim) + ivec := vector.GenIntVec(vector.Gaussian, 1, invalidDim)[0] req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, @@ -2639,7 +2623,7 @@ func Test_server_Insert(t *testing.T) { }(), func() test { invalidDim := f32VecDim + 1 - ivec := genF32Vec(vector.Gaussian, invalidDim) + ivec := vector.GenF32Vec(vector.Gaussian, 1, invalidDim)[0] req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, @@ -2696,7 +2680,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genIntVec(vector.Gaussian, intVecDim), + Vector: vector.GenIntVec(vector.Gaussian, 1, intVecDim)[0], }, } @@ -2733,7 +2717,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genF32Vec(vector.Gaussian, f32VecDim), + Vector: vector.GenF32Vec(vector.Gaussian, 1, f32VecDim)[0], }, } @@ -2770,7 +2754,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genIntVec(vector.Uniform, intVecDim), + Vector: vector.GenIntVec(vector.Uniform, 1, intVecDim)[0], }, } @@ -2807,7 +2791,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genF32Vec(vector.Uniform, f32VecDim), + Vector: vector.GenF32Vec(vector.Uniform, 1, f32VecDim)[0], }, } @@ -2846,7 +2830,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genSameValueVec(intVecDim, 0), + Vector: vector.GenSameValueVec(intVecDim, 0), }, } @@ -2883,7 +2867,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genSameValueVec(f32VecDim, 0), + Vector: vector.GenSameValueVec(f32VecDim, 0), }, } @@ -2920,7 +2904,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genSameValueVec(intVecDim, math.MinInt), + Vector: vector.GenSameValueVec(intVecDim, math.MinInt), }, } @@ -2957,7 +2941,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genSameValueVec(f32VecDim, -math.MaxFloat32), + Vector: vector.GenSameValueVec(f32VecDim, -math.MaxFloat32), }, } @@ -2994,7 +2978,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genSameValueVec(intVecDim, math.MaxInt), + Vector: vector.GenSameValueVec(intVecDim, math.MaxInt), }, } @@ -3031,7 +3015,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genSameValueVec(f32VecDim, math.MaxFloat32), + Vector: vector.GenSameValueVec(f32VecDim, math.MaxFloat32), }, } @@ -3228,7 +3212,7 @@ func Test_server_Insert(t *testing.T) { req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: genSameValueVec(f32VecDim, nan), + Vector: vector.GenSameValueVec(f32VecDim, nan), }, } @@ -3419,7 +3403,7 @@ func Test_server_Insert(t *testing.T) { // Decision Table Testing func() test { - bVec := genIntVec(vector.Gaussian, intVecDim) // used in beforeFunc + bVec := vector.GenIntVec(vector.Gaussian, 1, intVecDim)[0] // used in beforeFunc req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -3560,7 +3544,7 @@ func Test_server_Insert(t *testing.T) { } }(), func() test { - bVec := genIntVec(vector.Gaussian, intVecDim) // use in beforeFunc + bVec := vector.GenIntVec(vector.Gaussian, 1, intVecDim)[0] // use in beforeFunc req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -3843,34 +3827,6 @@ func Test_server_StreamInsert(t *testing.T) { } } -func genF32Vec(dist vector.Distribution, num int, dim int) [][]float32 { - generator, _ := vector.Float32VectorGenerator(dist) - return generator(num, dim) -} - -func genIntVec(dist vector.Distribution, num int, dim int) [][]float32 { - generator, _ := vector.Uint8VectorGenerator(dist) - ivecs := generator(num, dim) - result := make([][]float32, num) - - for j, ivec := range ivecs { - vec := make([]float32, dim) - for i := 0; i < dim; i++ { - vec[i] = float32(ivec[i]) - } - result[j] = vec - } - return result -} - -func genSameValueVec(size int, val float32) []float32 { - v := make([]float32, size) - for i := 0; i < size; i++ { - v[i] = val - } - return v -} - type objectType int const ( @@ -3882,9 +3838,9 @@ func genMultiInsertReq(t objectType, dist vector.Distribution, num int, dim int, var vecs [][]float32 switch t { case Float: - vecs = genF32Vec(dist, num, dim) + vecs = vector.GenF32Vec(dist, num, dim) case Uint8: - vecs = genIntVec(dist, num, dim) + vecs = vector.GenIntVec(dist, num, dim) } req := &payload.Insert_MultiRequest{ @@ -4331,7 +4287,7 @@ func Test_server_MultiInsert(t *testing.T) { func() test { insertNum := 100 req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil) - req.Requests[0].Vector.Vector = genIntVec(vector.Gaussian, 1, intVecDim+1)[0] + req.Requests[0].Vector.Vector = vector.GenIntVec(vector.Gaussian, 1, intVecDim+1)[0] return test{ name: "Equivalence Class Testing case 3.1: Fail to MultiInsert 100 vector with 1 vector with different dimension (vector type is uint8)", @@ -4374,7 +4330,7 @@ func Test_server_MultiInsert(t *testing.T) { func() test { insertNum := 100 req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) - req.Requests[0].Vector.Vector = genF32Vec(vector.Gaussian, 1, f32VecDim+1)[0] + req.Requests[0].Vector.Vector = vector.GenF32Vec(vector.Gaussian, 1, f32VecDim+1)[0] return test{ name: "Equivalence Class Testing case 3.2: Fail to MultiInsert 100 vector with 1 vector with different dimension (vector type is float32)", @@ -4419,7 +4375,7 @@ func Test_server_MultiInsert(t *testing.T) { insertNum := 100 req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil) for i := 0; i < len(req.Requests)/2; i++ { - req.Requests[i].Vector.Vector = genIntVec(vector.Gaussian, 1, intVecDim+1)[0] + req.Requests[i].Vector.Vector = vector.GenIntVec(vector.Gaussian, 1, intVecDim+1)[0] } return test{ @@ -4464,7 +4420,7 @@ func Test_server_MultiInsert(t *testing.T) { insertNum := 100 req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) for i := 0; i < len(req.Requests)/2; i++ { - req.Requests[i].Vector.Vector = genF32Vec(vector.Gaussian, 1, f32VecDim+1)[0] + req.Requests[i].Vector.Vector = vector.GenF32Vec(vector.Gaussian, 1, f32VecDim+1)[0] } return test{ @@ -4595,7 +4551,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 1.1: Success to MultiInsert with 0 value vector (vector type is uint8)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, genSameValueVec(intVecDim, 0), nil), + reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(intVecDim, 0), nil), }, fields: fields{ name: name, @@ -4615,7 +4571,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 1.2: Success to MultiInsert with 0 value vector (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, genSameValueVec(f32VecDim, 0), nil), + reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, 0), nil), }, fields: fields{ name: name, @@ -4635,7 +4591,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 2.1: Success to MultiInsert with min value vector (vector type is uint8)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, genSameValueVec(intVecDim, math.MinInt), nil), + reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(intVecDim, math.MinInt), nil), }, fields: fields{ name: name, @@ -4655,7 +4611,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 2.2: Success to MultiInsert with min value vector (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, genSameValueVec(f32VecDim, -math.MaxFloat32), nil), + reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, -math.MaxFloat32), nil), }, fields: fields{ name: name, @@ -4675,7 +4631,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 3.1: Success to MultiInsert with max value vector (vector type is uint8)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, genSameValueVec(intVecDim, math.MaxUint8), nil), + reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(intVecDim, math.MaxUint8), nil), }, fields: fields{ name: name, @@ -4695,7 +4651,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 3.2: Success to MultiInsert with max value vector (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, genSameValueVec(f32VecDim, math.MaxFloat32), nil), + reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, math.MaxFloat32), nil), }, fields: fields{ name: name, @@ -5253,7 +5209,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 6.1: Success to MultiInsert with NaN value (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, genSameValueVec(f32VecDim, float32(math.NaN())), nil), + reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.NaN())), nil), }, fields: fields{ name: name, @@ -5273,7 +5229,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 6.2: Success to MultiInsert with +Inf value (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, genSameValueVec(f32VecDim, float32(math.Inf(+1.0))), nil), + reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.Inf(+1.0))), nil), }, fields: fields{ name: name, @@ -5293,7 +5249,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 6.3: Success to MultiInsert with -Inf value (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, genSameValueVec(f32VecDim, float32(math.Inf(-1.0))), nil), + reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.Inf(-1.0))), nil), }, fields: fields{ name: name, @@ -5313,7 +5269,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 6.4: Success to MultiInsert with -0 value (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, genSameValueVec(f32VecDim, float32(math.Copysign(0, -1.0))), nil), + reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.Copysign(0, -1.0))), nil), }, fields: fields{ name: name, @@ -6120,7 +6076,7 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, beforeFunc: func(t *testing.T, s *server) { - vecs := genF32Vec(vector.Gaussian, 2, f32VecDim) + vecs := vector.GenF32Vec(vector.Gaussian, 2, f32VecDim) for i := 0; i < 2; i++ { ir := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -6172,7 +6128,7 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, beforeFunc: func(t *testing.T, s *server) { - vecs := genF32Vec(vector.Gaussian, insertNum, f32VecDim) + vecs := vector.GenF32Vec(vector.Gaussian, insertNum, f32VecDim) for i, r := range req.Requests { ir := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -6215,7 +6171,7 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, beforeFunc: func(t *testing.T, s *server) { - vecs := genF32Vec(vector.Gaussian, 2, f32VecDim) + vecs := vector.GenF32Vec(vector.Gaussian, 2, f32VecDim) for i := 0; i < 2; i++ { ir := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -6267,7 +6223,7 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, beforeFunc: func(t *testing.T, s *server) { - vecs := genF32Vec(vector.Gaussian, insertNum, f32VecDim) + vecs := vector.GenF32Vec(vector.Gaussian, insertNum, f32VecDim) for i, r := range req.Requests { ir := &payload.Insert_Request{ Vector: &payload.Object_Vector{ From 05f872a80255f79694e2d0365e105edbf9eefcb3 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 25 May 2022 15:15:46 +0900 Subject: [PATCH 03/29] refactor request helper func Signed-off-by: kevindiu --- .../core/ngt/handler/grpc/handler_test.go | 261 +++++++----------- 1 file changed, 99 insertions(+), 162 deletions(-) diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 026ab883c4..40f5a768a1 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -38,6 +38,7 @@ import ( "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" "github.com/vdaas/vald/internal/test/data/charset" + "github.com/vdaas/vald/internal/test/data/request" "github.com/vdaas/vald/internal/test/data/vector" "github.com/vdaas/vald/internal/test/goleak" "github.com/vdaas/vald/pkg/agent/core/ngt/model" @@ -3827,38 +3828,6 @@ func Test_server_StreamInsert(t *testing.T) { } } -type objectType int - -const ( - Uint8 objectType = iota - Float -) - -func genMultiInsertReq(t objectType, dist vector.Distribution, num int, dim int, cfg *payload.Insert_Config) *payload.Insert_MultiRequest { - var vecs [][]float32 - switch t { - case Float: - vecs = vector.GenF32Vec(dist, num, dim) - case Uint8: - vecs = vector.GenIntVec(dist, num, dim) - } - - req := &payload.Insert_MultiRequest{ - Requests: make([]*payload.Insert_Request, num), - } - for i, vec := range vecs { - req.Requests[i] = &payload.Insert_Request{ - Vector: &payload.Object_Vector{ - Id: "uuid-" + strconv.Itoa(i+1), - Vector: vec, - }, - Config: cfg, - } - } - - return req -} - func Test_server_MultiInsert(t *testing.T) { t.Parallel() @@ -3922,24 +3891,6 @@ func Test_server_MultiInsert(t *testing.T) { } ) - // generate MultiInsert request with the same vector - genSameVecMultiInsertReq := func(num int, vec []float32, cfg *payload.Insert_Config) *payload.Insert_MultiRequest { - req := &payload.Insert_MultiRequest{ - Requests: make([]*payload.Insert_Request, num), - } - for i := 0; i < num; i++ { - req.Requests[i] = &payload.Insert_Request{ - Vector: &payload.Object_Vector{ - Id: "uuid-" + strconv.Itoa(i+1), - Vector: vec, - }, - Config: cfg, - } - } - - return req - } - genAlreadyExistsErr := func(uuid string, req *payload.Insert_MultiRequest, name, ip string) error { return status.WrapWithAlreadyExists(fmt.Sprintf("MultiInsert API uuids [%v] already exists", uuid), errors.ErrUUIDAlreadyExists(uuid), @@ -3953,20 +3904,6 @@ func Test_server_MultiInsert(t *testing.T) { }) } - genObjectLocations := func(num int, name string, ip string) *payload.Object_Locations { - result := &payload.Object_Locations{ - Locations: make([]*payload.Object_Location, num), - } - - for i := 0; i < num; i++ { - result.Locations[i] = &payload.Object_Location{ - Name: name, - Uuid: "uuid-" + strconv.Itoa(i+1), - Ips: []string{ip}, - } - } - return result - } defaultCheckFunc := func(w want, gotRes *payload.Object_Locations, err error) error { if w.containErr == nil { if !errors.Is(err, w.err) { @@ -4088,7 +4025,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Equivalence Class Testing case 1.1: Success to MultiInsert 1 vector (vector type is uint8)", args: args{ ctx: ctx, - reqs: genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil), + reqs: request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil), }, fields: fields{ name: name, @@ -4098,7 +4035,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -4108,7 +4045,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Equivalence Class Testing case 1.2: Success to MultiInsert 1 vector (vector type is float32)", args: args{ ctx: ctx, - reqs: genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil), + reqs: request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil), }, fields: fields{ name: name, @@ -4118,7 +4055,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -4128,7 +4065,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Equivalence Class Testing case 1.3: Success to MultiInsert 100 vector (vector type is uint8)", args: args{ ctx: ctx, - reqs: genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil), + reqs: request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil), }, fields: fields{ name: name, @@ -4138,7 +4075,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -4148,7 +4085,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Equivalence Class Testing case 1.4: Success to MultiInsert 100 vector (vector type is float32)", args: args{ ctx: ctx, - reqs: genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil), + reqs: request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil), }, fields: fields{ name: name, @@ -4158,7 +4095,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -4202,7 +4139,7 @@ func Test_server_MultiInsert(t *testing.T) { }, func() test { insertNum := 1 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim+1, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim+1, nil) return test{ name: "Equivalence Class Testing case 2.1: Fail to MultiInsert 1 vector with different dimension (vector type is uint8)", @@ -4244,7 +4181,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 1 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim+1, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim+1, nil) return test{ name: "Equivalence Class Testing case 2.2: Fail to MultiInsert 1 vector with different dimension (vector type is float32)", @@ -4286,7 +4223,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) req.Requests[0].Vector.Vector = vector.GenIntVec(vector.Gaussian, 1, intVecDim+1)[0] return test{ @@ -4329,7 +4266,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) req.Requests[0].Vector.Vector = vector.GenF32Vec(vector.Gaussian, 1, f32VecDim+1)[0] return test{ @@ -4373,7 +4310,7 @@ func Test_server_MultiInsert(t *testing.T) { func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Vector = vector.GenIntVec(vector.Gaussian, 1, intVecDim+1)[0] } @@ -4418,7 +4355,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Vector = vector.GenF32Vec(vector.Gaussian, 1, f32VecDim+1)[0] } @@ -4463,7 +4400,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim+1, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim+1, nil) return test{ name: "Equivalence Class Testing case 3.5: Fail to MultiInsert 100 vector with all vector with different dimension (vector type is uint8)", @@ -4505,7 +4442,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim+1, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim+1, nil) return test{ name: "Equivalence Class Testing case 3.6: Fail to MultiInsert 100 vector with all vector with different dimension (vector type is float32)", @@ -4551,7 +4488,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 1.1: Success to MultiInsert with 0 value vector (vector type is uint8)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(intVecDim, 0), nil), + reqs: request.GenSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(intVecDim, 0), nil), }, fields: fields{ name: name, @@ -4561,7 +4498,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -4571,7 +4508,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 1.2: Success to MultiInsert with 0 value vector (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, 0), nil), + reqs: request.GenSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, 0), nil), }, fields: fields{ name: name, @@ -4581,7 +4518,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -4591,7 +4528,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 2.1: Success to MultiInsert with min value vector (vector type is uint8)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(intVecDim, math.MinInt), nil), + reqs: request.GenSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(intVecDim, math.MinInt), nil), }, fields: fields{ name: name, @@ -4601,7 +4538,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -4611,7 +4548,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 2.2: Success to MultiInsert with min value vector (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, -math.MaxFloat32), nil), + reqs: request.GenSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, -math.MaxFloat32), nil), }, fields: fields{ name: name, @@ -4621,7 +4558,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -4631,7 +4568,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 3.1: Success to MultiInsert with max value vector (vector type is uint8)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(intVecDim, math.MaxUint8), nil), + reqs: request.GenSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(intVecDim, math.MaxUint8), nil), }, fields: fields{ name: name, @@ -4641,7 +4578,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -4651,7 +4588,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 3.2: Success to MultiInsert with max value vector (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, math.MaxFloat32), nil), + reqs: request.GenSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, math.MaxFloat32), nil), }, fields: fields{ name: name, @@ -4661,13 +4598,13 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) req.Requests[0].Vector.Id = "" uuids := make([]string, 0, len(req.Requests)) @@ -4711,7 +4648,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) req.Requests[0].Vector.Id = "" uuids := make([]string, 0, len(req.Requests)) @@ -4755,7 +4692,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Id = "" } @@ -4801,7 +4738,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Id = "" } @@ -4847,7 +4784,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) for i := 0; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = "" } @@ -4893,7 +4830,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) for i := 0; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = "" } @@ -4939,7 +4876,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) req.Requests[0].Vector.Vector = make([]float32, maxVecDim) return test{ @@ -4983,7 +4920,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) req.Requests[0].Vector.Vector = make([]float32, maxVecDim) return test{ @@ -5027,7 +4964,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Vector = make([]float32, maxVecDim) } @@ -5073,7 +5010,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Vector = make([]float32, maxVecDim) } @@ -5119,7 +5056,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genSameVecMultiInsertReq(insertNum, make([]float32, maxVecDim), nil) + req := request.GenSameVecMultiInsertReq(insertNum, make([]float32, maxVecDim), nil) return test{ name: "Boundary Value Testing case 5.5: Fail to MultiInsert with all vector with maximum dimension (vector type is uint8)", @@ -5162,7 +5099,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genSameVecMultiInsertReq(insertNum, make([]float32, maxVecDim), nil) + req := request.GenSameVecMultiInsertReq(insertNum, make([]float32, maxVecDim), nil) return test{ name: "Boundary Value Testing case 5.6: Fail to MultiInsert with all vector with maximum dimension (vector type is float32)", @@ -5209,7 +5146,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 6.1: Success to MultiInsert with NaN value (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.NaN())), nil), + reqs: request.GenSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.NaN())), nil), }, fields: fields{ name: name, @@ -5219,7 +5156,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -5229,7 +5166,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 6.2: Success to MultiInsert with +Inf value (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.Inf(+1.0))), nil), + reqs: request.GenSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.Inf(+1.0))), nil), }, fields: fields{ name: name, @@ -5239,7 +5176,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -5249,7 +5186,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 6.3: Success to MultiInsert with -Inf value (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.Inf(-1.0))), nil), + reqs: request.GenSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.Inf(-1.0))), nil), }, fields: fields{ name: name, @@ -5259,7 +5196,7 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -5269,7 +5206,7 @@ func Test_server_MultiInsert(t *testing.T) { name: "Boundary Value Testing case 6.4: Success to MultiInsert with -0 value (vector type is float32)", args: args{ ctx: ctx, - reqs: genSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.Copysign(0, -1.0))), nil), + reqs: request.GenSameVecMultiInsertReq(insertNum, vector.GenSameValueVec(f32VecDim, float32(math.Copysign(0, -1.0))), nil), }, fields: fields{ name: name, @@ -5279,13 +5216,13 @@ func Test_server_MultiInsert(t *testing.T) { streamConcurrency: 0, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) vid := req.Requests[0].Vector.Id req.Requests[0] = nil @@ -5329,7 +5266,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i] = nil @@ -5375,7 +5312,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests); i++ { req.Requests[i] = nil @@ -5421,7 +5358,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) vid := req.Requests[0].Vector.Id req.Requests[0].Vector.Vector = nil @@ -5465,7 +5402,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Vector = nil @@ -5511,7 +5448,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests); i++ { req.Requests[i].Vector.Vector = nil @@ -5557,7 +5494,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) vid := req.Requests[0].Vector.Id req.Requests[0].Vector.Vector = []float32{} @@ -5601,7 +5538,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Vector = []float32{} @@ -5647,7 +5584,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests); i++ { req.Requests[i].Vector.Vector = []float32{} @@ -5697,11 +5634,11 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: false, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) req.Requests[0].Vector.Id = req.Requests[1].Vector.Id // set want - w := genObjectLocations(insertNum, name, ip) + w := request.GenObjectLocations(insertNum, name, ip) w.Locations[0].Uuid = req.Requests[0].Vector.Id return test{ @@ -5727,13 +5664,13 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: false, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = req.Requests[0].Vector.Id } // set want - w := genObjectLocations(insertNum, name, ip) + w := request.GenObjectLocations(insertNum, name, ip) for _, l := range w.Locations { l.Uuid = req.Requests[0].Vector.Id } @@ -5761,11 +5698,11 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) req.Requests[0].Vector.Id = req.Requests[1].Vector.Id // set want - w := genObjectLocations(insertNum, name, ip) + w := request.GenObjectLocations(insertNum, name, ip) w.Locations[0].Uuid = req.Requests[0].Vector.Id // w.Locations[1].Uuid = dupID @@ -5792,13 +5729,13 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = req.Requests[0].Vector.Id } // set want - w := genObjectLocations(insertNum, name, ip) + w := request.GenObjectLocations(insertNum, name, ip) for _, l := range w.Locations { l.Uuid = req.Requests[0].Vector.Id } @@ -5826,7 +5763,7 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: false, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) req.Requests[0].Vector.Vector = req.Requests[1].Vector.Vector return test{ @@ -5842,7 +5779,7 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -5852,7 +5789,7 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: false, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Vector = req.Requests[0].Vector.Vector } @@ -5870,7 +5807,7 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -5880,7 +5817,7 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) req.Requests[0].Vector.Vector = req.Requests[1].Vector.Vector return test{ @@ -5896,7 +5833,7 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -5906,7 +5843,7 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Vector = req.Requests[0].Vector.Vector } @@ -5924,7 +5861,7 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), @@ -5933,12 +5870,12 @@ func Test_server_MultiInsert(t *testing.T) { cfg := &payload.Insert_Config{ SkipStrictExistCheck: false, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) req.Requests[0].Vector.Vector = req.Requests[1].Vector.Vector req.Requests[0].Vector.Id = req.Requests[1].Vector.Id // set want - w := genObjectLocations(insertNum, name, ip) + w := request.GenObjectLocations(insertNum, name, ip) w.Locations[0].Uuid = req.Requests[0].Vector.Id return test{ @@ -5964,14 +5901,14 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: false, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = req.Requests[0].Vector.Id req.Requests[i].Vector.Vector = req.Requests[0].Vector.Vector } // set want - w := genObjectLocations(insertNum, name, ip) + w := request.GenObjectLocations(insertNum, name, ip) for _, l := range w.Locations { l.Uuid = req.Requests[0].Vector.Id } @@ -5999,12 +5936,12 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) req.Requests[0].Vector.Vector = req.Requests[1].Vector.Vector req.Requests[0].Vector.Id = req.Requests[1].Vector.Id // set want - w := genObjectLocations(insertNum, name, ip) + w := request.GenObjectLocations(insertNum, name, ip) w.Locations[0].Uuid = req.Requests[0].Vector.Id return test{ @@ -6030,14 +5967,14 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = req.Requests[0].Vector.Id req.Requests[i].Vector.Vector = req.Requests[0].Vector.Vector } // set want - w := genObjectLocations(insertNum, name, ip) + w := request.GenObjectLocations(insertNum, name, ip) for _, l := range w.Locations { l.Uuid = req.Requests[0].Vector.Id } @@ -6061,7 +5998,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) return test{ name: "Decision Table Testing case 4.1: Fail to MultiInsert with 2 existed ID when SkipStrictExistCheck is false", @@ -6108,7 +6045,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) wantErrs := make([]error, 100) for i := 0; i < len(req.Requests); i++ { @@ -6156,7 +6093,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) return test{ name: "Decision Table Testing case 4.3: Fail to MultiInsert with 2 existed ID when SkipStrictExistCheck is true", @@ -6203,7 +6140,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) wantErrs := make([]error, 100) for i := 0; i < len(req.Requests); i++ { @@ -6251,7 +6188,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) return test{ name: "Decision Table Testing case 5.1: Success to MultiInsert with 2 existed vector when SkipStrictExistCheck is false", @@ -6288,13 +6225,13 @@ func Test_server_MultiInsert(t *testing.T) { } }, want: want{ - wantRes: genObjectLocations(100, name, ip), + wantRes: request.GenObjectLocations(100, name, ip), }, } }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) return test{ name: "Decision Table Testing case 5.2: Success to MultiInsert with all existed vector when SkipStrictExistCheck is false", @@ -6331,13 +6268,13 @@ func Test_server_MultiInsert(t *testing.T) { } }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) return test{ name: "Decision Table Testing case 5.3: Success to MultiInsert with 2 existed vector when SkipStrictExistCheck is true", @@ -6374,13 +6311,13 @@ func Test_server_MultiInsert(t *testing.T) { } }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) return test{ name: "Decision Table Testing case 5.4: Success to MultiInsert with all existed vector when SkipStrictExistCheck is true", @@ -6417,13 +6354,13 @@ func Test_server_MultiInsert(t *testing.T) { } }, want: want{ - wantRes: genObjectLocations(insertNum, name, ip), + wantRes: request.GenObjectLocations(insertNum, name, ip), }, } }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) return test{ name: "Decision Table Testing case 6.1: Fail to MultiInsert with 2 existed ID & vector when SkipStrictExistCheck is false", @@ -6466,7 +6403,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) wantErrs := make([]error, 100) for i := 0; i < len(req.Requests); i++ { @@ -6510,7 +6447,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) return test{ name: "Decision Table Testing case 6.3: Fail to MultiInsert with 2 existed ID & vector when SkipStrictExistCheck is true", @@ -6553,7 +6490,7 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := genMultiInsertReq(Float, vector.Gaussian, insertNum, f32VecDim, nil) + req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) wantErrs := make([]error, 100) for i := 0; i < len(req.Requests); i++ { @@ -6722,7 +6659,7 @@ func Test_server_Update(t *testing.T) { return nil, err } - // TODO: use genMultiInsertReq() + // TODO: use request.GenMultiInsertReq() reqs := make([]*payload.Insert_Request, insertNum) for i, v := range gen(insertNum, cfg.Dimension) { reqs[i] = &payload.Insert_Request{ From 7e374aad6e065a2b01b8642e09b1f22d70a3516c Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 25 May 2022 15:16:31 +0900 Subject: [PATCH 04/29] add missing files Signed-off-by: kevindiu --- internal/test/data/charset/convert.go | 19 +++++++++ internal/test/data/request/insert.go | 58 +++++++++++++++++++++++++++ internal/test/data/request/object.go | 22 ++++++++++ internal/test/data/vector/gen.go | 28 +++++++++++++ 4 files changed, 127 insertions(+) create mode 100644 internal/test/data/charset/convert.go create mode 100644 internal/test/data/request/insert.go create mode 100644 internal/test/data/request/object.go diff --git a/internal/test/data/charset/convert.go b/internal/test/data/charset/convert.go new file mode 100644 index 0000000000..a2aacd1f46 --- /dev/null +++ b/internal/test/data/charset/convert.go @@ -0,0 +1,19 @@ +package charset + +import ( + "io/ioutil" + "strings" + + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/transform" +) + +func Utf8ToSjis(s string) string { + b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())) + return string(b) +} + +func Utf8ToEucjp(s string) string { + b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.EUCJP.NewEncoder())) + return string(b) +} diff --git a/internal/test/data/request/insert.go b/internal/test/data/request/insert.go new file mode 100644 index 0000000000..cf846e4be0 --- /dev/null +++ b/internal/test/data/request/insert.go @@ -0,0 +1,58 @@ +package request + +import ( + "strconv" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/test/data/vector" +) + +type ObjectType int + +const ( + Uint8 ObjectType = iota + Float +) + +func GenMultiInsertReq(t ObjectType, dist vector.Distribution, num int, dim int, cfg *payload.Insert_Config) *payload.Insert_MultiRequest { + var vecs [][]float32 + switch t { + case Float: + vecs = vector.GenF32Vec(dist, num, dim) + case Uint8: + vecs = vector.GenIntVec(dist, num, dim) + } + + req := &payload.Insert_MultiRequest{ + Requests: make([]*payload.Insert_Request, num), + } + for i, vec := range vecs { + req.Requests[i] = &payload.Insert_Request{ + Vector: &payload.Object_Vector{ + Id: "uuid-" + strconv.Itoa(i+1), + Vector: vec, + }, + Config: cfg, + } + } + + return req +} + +// generate MultiInsert request with the same vector +func GenSameVecMultiInsertReq(num int, vec []float32, cfg *payload.Insert_Config) *payload.Insert_MultiRequest { + req := &payload.Insert_MultiRequest{ + Requests: make([]*payload.Insert_Request, num), + } + for i := 0; i < num; i++ { + req.Requests[i] = &payload.Insert_Request{ + Vector: &payload.Object_Vector{ + Id: "uuid-" + strconv.Itoa(i+1), + Vector: vec, + }, + Config: cfg, + } + } + + return req +} diff --git a/internal/test/data/request/object.go b/internal/test/data/request/object.go new file mode 100644 index 0000000000..a471cb9b74 --- /dev/null +++ b/internal/test/data/request/object.go @@ -0,0 +1,22 @@ +package request + +import ( + "strconv" + + "github.com/vdaas/vald/apis/grpc/v1/payload" +) + +func GenObjectLocations(num int, name string, ip string) *payload.Object_Locations { + result := &payload.Object_Locations{ + Locations: make([]*payload.Object_Location, num), + } + + for i := 0; i < num; i++ { + result.Locations[i] = &payload.Object_Location{ + Name: name, + Uuid: "uuid-" + strconv.Itoa(i+1), + Ips: []string{ip}, + } + } + return result +} diff --git a/internal/test/data/vector/gen.go b/internal/test/data/vector/gen.go index 55b14e3d17..b60ff6f8af 100644 --- a/internal/test/data/vector/gen.go +++ b/internal/test/data/vector/gen.go @@ -131,3 +131,31 @@ func gaussianDistributedUint8VectorGenerator(n, dim int, mean, sigma float64) [] } }) } + +func GenF32Vec(dist Distribution, num int, dim int) [][]float32 { + generator, _ := Float32VectorGenerator(dist) + return generator(num, dim) +} + +func GenIntVec(dist Distribution, num int, dim int) [][]float32 { + generator, _ := Uint8VectorGenerator(dist) + ivecs := generator(num, dim) + result := make([][]float32, num) + + for j, ivec := range ivecs { + vec := make([]float32, dim) + for i := 0; i < dim; i++ { + vec[i] = float32(ivec[i]) + } + result[j] = vec + } + return result +} + +func GenSameValueVec(size int, val float32) []float32 { + v := make([]float32, size) + for i := 0; i < size; i++ { + v[i] = val + } + return v +} From 847bbe2fdafc92384a054fea70d2c713f6b16fa5 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Thu, 26 May 2022 11:40:58 +0900 Subject: [PATCH 05/29] move charset to conv Signed-off-by: kevindiu --- internal/conv/conv.go | 15 +++ internal/test/data/charset/convert.go | 19 ---- .../core/ngt/handler/grpc/handler_test.go | 106 +++++++++--------- 3 files changed, 68 insertions(+), 72 deletions(-) delete mode 100644 internal/test/data/charset/convert.go diff --git a/internal/conv/conv.go b/internal/conv/conv.go index 2cf0f27c90..61685ed8c8 100644 --- a/internal/conv/conv.go +++ b/internal/conv/conv.go @@ -16,8 +16,13 @@ package conv import ( + "io/ioutil" "reflect" + "strings" "unsafe" + + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/transform" ) // Btoa converts from byte slice to string. @@ -48,3 +53,13 @@ func F32stos(fs []float32) (s string) { (*(*int)(unsafe.Pointer(uintptr(addr) + uintptr(16)))) = lf return Btoa(buf) } + +func Utf8ToSjis(s string) string { + b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())) + return string(b) +} + +func Utf8ToEucjp(s string) string { + b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.EUCJP.NewEncoder())) + return string(b) +} diff --git a/internal/test/data/charset/convert.go b/internal/test/data/charset/convert.go deleted file mode 100644 index a2aacd1f46..0000000000 --- a/internal/test/data/charset/convert.go +++ /dev/null @@ -1,19 +0,0 @@ -package charset - -import ( - "io/ioutil" - "strings" - - "golang.org/x/text/encoding/japanese" - "golang.org/x/text/transform" -) - -func Utf8ToSjis(s string) string { - b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())) - return string(b) -} - -func Utf8ToEucjp(s string) string { - b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.EUCJP.NewEncoder())) - return string(b) -} diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 40f5a768a1..138eccc904 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -30,6 +30,7 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/apis/grpc/v1/vald" "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/conv" "github.com/vdaas/vald/internal/core/algorithm/ngt" "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" @@ -37,7 +38,6 @@ import ( "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" - "github.com/vdaas/vald/internal/test/data/charset" "github.com/vdaas/vald/internal/test/data/request" "github.com/vdaas/vald/internal/test/data/vector" "github.com/vdaas/vald/internal/test/goleak" @@ -541,7 +541,7 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.2: fail exists with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), searchId: "こんにちは", }, want: want{ @@ -552,7 +552,7 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.3: fail exists with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), searchId: "こんにちは", }, want: want{ @@ -564,7 +564,7 @@ func Test_server_Exists(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - searchId: charset.Utf8ToSjis("こんにちは"), + searchId: conv.Utf8ToSjis("こんにちは"), }, want: want{ code: codes.NotFound, @@ -574,8 +574,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.5: success exists with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), - searchId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), + searchId: conv.Utf8ToSjis("こんにちは"), }, want: want{}, }, @@ -583,8 +583,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.6: fail exists with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), - searchId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), + searchId: conv.Utf8ToSjis("こんにちは"), }, want: want{ code: codes.NotFound, @@ -595,7 +595,7 @@ func Test_server_Exists(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - searchId: charset.Utf8ToEucjp("こんにちは"), + searchId: conv.Utf8ToEucjp("こんにちは"), }, want: want{ code: codes.NotFound, @@ -605,8 +605,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.8: fail exists with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), - searchId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), + searchId: conv.Utf8ToEucjp("こんにちは"), }, want: want{ code: codes.NotFound, @@ -616,8 +616,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.9: success exists with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), - searchId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), + searchId: conv.Utf8ToEucjp("こんにちは"), }, want: want{}, }, @@ -1748,7 +1748,7 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.2: fail search with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), searchId: "こんにちは", }, want: want{ @@ -1760,7 +1760,7 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.3: fail search with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), searchId: "こんにちは", }, want: want{ @@ -1773,7 +1773,7 @@ func Test_server_SearchByID(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - searchId: charset.Utf8ToSjis("こんにちは"), + searchId: conv.Utf8ToSjis("こんにちは"), }, want: want{ resultSize: 0, @@ -1784,8 +1784,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.5: success search with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), - searchId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), + searchId: conv.Utf8ToSjis("こんにちは"), }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -1795,8 +1795,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.6: fail search with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), - searchId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), + searchId: conv.Utf8ToSjis("こんにちは"), }, want: want{ resultSize: 0, @@ -1808,7 +1808,7 @@ func Test_server_SearchByID(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - searchId: charset.Utf8ToEucjp("こんにちは"), + searchId: conv.Utf8ToEucjp("こんにちは"), }, want: want{ resultSize: 0, @@ -1819,8 +1819,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.8: fail search with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), - searchId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), + searchId: conv.Utf8ToEucjp("こんにちは"), }, want: want{ resultSize: 0, @@ -1831,8 +1831,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.9: success search with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), - searchId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), + searchId: conv.Utf8ToEucjp("こんにちは"), }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -6942,7 +6942,7 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.2: success update with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "こんにちは", @@ -6959,7 +6959,7 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.3: success update with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "こんにちは", @@ -6979,7 +6979,7 @@ func Test_server_Update(t *testing.T) { indexId: "こんにちは", req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: charset.Utf8ToSjis("こんにちは"), + Id: conv.Utf8ToSjis("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -6993,27 +6993,27 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.5: success update with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: charset.Utf8ToSjis("こんにちは"), + Id: conv.Utf8ToSjis("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, }, }, want: want{ - wantUuid: charset.Utf8ToSjis("こんにちは"), + wantUuid: conv.Utf8ToSjis("こんにちは"), }, }, { name: "Boundary Value Testing case 3.6: fail update with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: charset.Utf8ToSjis("こんにちは"), + Id: conv.Utf8ToSjis("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7030,7 +7030,7 @@ func Test_server_Update(t *testing.T) { indexId: "こんにちは", req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: charset.Utf8ToEucjp("こんにちは"), + Id: conv.Utf8ToEucjp("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7044,10 +7044,10 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.8: fail update with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: charset.Utf8ToEucjp("こんにちは"), + Id: conv.Utf8ToEucjp("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7061,17 +7061,17 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.9: success update with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: charset.Utf8ToEucjp("こんにちは"), + Id: conv.Utf8ToEucjp("こんにちは"), Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, }, }, want: want{ - wantUuid: charset.Utf8ToEucjp("こんにちは"), + wantUuid: conv.Utf8ToEucjp("こんにちは"), }, }, { @@ -8220,7 +8220,7 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.2: fail exists with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), removeId: "こんにちは", }, want: want{ @@ -8231,7 +8231,7 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.3: fail exists with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), removeId: "こんにちは", }, want: want{ @@ -8243,7 +8243,7 @@ func Test_server_Remove(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - removeId: charset.Utf8ToSjis("こんにちは"), + removeId: conv.Utf8ToSjis("こんにちは"), }, want: want{ code: codes.NotFound, @@ -8253,19 +8253,19 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.5: success exists with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), - removeId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), + removeId: conv.Utf8ToSjis("こんにちは"), }, want: want{ - wantUuid: charset.Utf8ToSjis("こんにちは"), + wantUuid: conv.Utf8ToSjis("こんにちは"), }, }, { name: "Boundary Value Testing case 3.6: fail exists with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), - removeId: charset.Utf8ToSjis("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), + removeId: conv.Utf8ToSjis("こんにちは"), }, want: want{ code: codes.NotFound, @@ -8276,7 +8276,7 @@ func Test_server_Remove(t *testing.T) { args: args{ ctx: ctx, indexId: "こんにちは", - removeId: charset.Utf8ToEucjp("こんにちは"), + removeId: conv.Utf8ToEucjp("こんにちは"), }, want: want{ code: codes.NotFound, @@ -8286,8 +8286,8 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.8: fail exists with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: charset.Utf8ToSjis("こんにちは"), - removeId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToSjis("こんにちは"), + removeId: conv.Utf8ToEucjp("こんにちは"), }, want: want{ code: codes.NotFound, @@ -8297,11 +8297,11 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.9: success exists with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: charset.Utf8ToEucjp("こんにちは"), - removeId: charset.Utf8ToEucjp("こんにちは"), + indexId: conv.Utf8ToEucjp("こんにちは"), + removeId: conv.Utf8ToEucjp("こんにちは"), }, want: want{ - wantUuid: charset.Utf8ToEucjp("こんにちは"), + wantUuid: conv.Utf8ToEucjp("こんにちは"), }, }, { From a3dc38864202f949d89d6beb1369d9892d3de756 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Thu, 26 May 2022 15:24:32 +0900 Subject: [PATCH 06/29] return error from helper func Signed-off-by: kevindiu --- internal/test/data/request/insert.go | 12 +- internal/test/data/vector/gen.go | 19 +- .../core/ngt/handler/grpc/handler_test.go | 393 ++++++++++++++---- 3 files changed, 337 insertions(+), 87 deletions(-) diff --git a/internal/test/data/request/insert.go b/internal/test/data/request/insert.go index cf846e4be0..cb2a657d60 100644 --- a/internal/test/data/request/insert.go +++ b/internal/test/data/request/insert.go @@ -14,13 +14,17 @@ const ( Float ) -func GenMultiInsertReq(t ObjectType, dist vector.Distribution, num int, dim int, cfg *payload.Insert_Config) *payload.Insert_MultiRequest { +func GenMultiInsertReq(t ObjectType, dist vector.Distribution, num int, dim int, cfg *payload.Insert_Config) (*payload.Insert_MultiRequest, error) { var vecs [][]float32 + var err error switch t { case Float: - vecs = vector.GenF32Vec(dist, num, dim) + vecs, err = vector.GenF32Vec(dist, num, dim) case Uint8: - vecs = vector.GenIntVec(dist, num, dim) + vecs, err = vector.GenIntVec(dist, num, dim) + } + if err != nil { + return nil, err } req := &payload.Insert_MultiRequest{ @@ -36,7 +40,7 @@ func GenMultiInsertReq(t ObjectType, dist vector.Distribution, num int, dim int, } } - return req + return req, nil } // generate MultiInsert request with the same vector diff --git a/internal/test/data/vector/gen.go b/internal/test/data/vector/gen.go index b60ff6f8af..f8d28e5d05 100644 --- a/internal/test/data/vector/gen.go +++ b/internal/test/data/vector/gen.go @@ -132,13 +132,20 @@ func gaussianDistributedUint8VectorGenerator(n, dim int, mean, sigma float64) [] }) } -func GenF32Vec(dist Distribution, num int, dim int) [][]float32 { - generator, _ := Float32VectorGenerator(dist) - return generator(num, dim) +func GenF32Vec(dist Distribution, num int, dim int) ([][]float32, error) { + generator, err := Float32VectorGenerator(dist) + if err != nil { + return nil, err + } + return generator(num, dim), nil } -func GenIntVec(dist Distribution, num int, dim int) [][]float32 { - generator, _ := Uint8VectorGenerator(dist) +func GenIntVec(dist Distribution, num int, dim int) ([][]float32, error) { + generator, err := Uint8VectorGenerator(dist) + if err != nil { + return nil, err + } + ivecs := generator(num, dim) result := make([][]float32, num) @@ -149,7 +156,7 @@ func GenIntVec(dist Distribution, num int, dim int) [][]float32 { } result[j] = vec } - return result + return result, nil } func GenSameValueVec(size int, val float32) []float32 { diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 138eccc904..552dfb941a 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -2569,11 +2569,14 @@ func Test_server_Insert(t *testing.T) { }(), func() test { invalidDim := intVecDim + 1 - ivec := vector.GenIntVec(vector.Gaussian, 1, invalidDim)[0] + ivec, err := vector.GenIntVec(vector.Gaussian, 1, invalidDim) + if err != nil { + t.Error(err) + } req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: ivec, + Vector: ivec[0], }, } @@ -2624,11 +2627,14 @@ func Test_server_Insert(t *testing.T) { }(), func() test { invalidDim := f32VecDim + 1 - ivec := vector.GenF32Vec(vector.Gaussian, 1, invalidDim)[0] + ivec, err := vector.GenF32Vec(vector.Gaussian, 1, invalidDim) + if err != nil { + t.Error(err) + } req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: ivec, + Vector: ivec[0], }, } @@ -2678,10 +2684,15 @@ func Test_server_Insert(t *testing.T) { } }(), func() test { + ivec, err := vector.GenIntVec(vector.Gaussian, 1, intVecDim) + if err != nil { + t.Error(err) + } + req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: vector.GenIntVec(vector.Gaussian, 1, intVecDim)[0], + Vector: ivec[0], }, } @@ -2715,10 +2726,15 @@ func Test_server_Insert(t *testing.T) { } }(), func() test { + ivec, err := vector.GenF32Vec(vector.Gaussian, 1, f32VecDim) + if err != nil { + t.Error(err) + } + req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: vector.GenF32Vec(vector.Gaussian, 1, f32VecDim)[0], + Vector: ivec[0], }, } @@ -2752,10 +2768,15 @@ func Test_server_Insert(t *testing.T) { } }(), func() test { + ivec, err := vector.GenIntVec(vector.Uniform, 1, intVecDim) + if err != nil { + t.Error(err) + } + req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: vector.GenIntVec(vector.Uniform, 1, intVecDim)[0], + Vector: ivec[0], }, } @@ -2789,10 +2810,15 @@ func Test_server_Insert(t *testing.T) { } }(), func() test { + ivec, err := vector.GenF32Vec(vector.Uniform, 1, f32VecDim) + if err != nil { + t.Error(err) + } + req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ Id: id, - Vector: vector.GenF32Vec(vector.Uniform, 1, f32VecDim)[0], + Vector: ivec[0], }, } @@ -3404,7 +3430,10 @@ func Test_server_Insert(t *testing.T) { // Decision Table Testing func() test { - bVec := vector.GenIntVec(vector.Gaussian, 1, intVecDim)[0] // used in beforeFunc + bVecs, err := vector.GenIntVec(vector.Gaussian, 1, intVecDim) // used in beforeFunc + if err != nil { + t.Error(err) + } req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -3437,7 +3466,7 @@ func Test_server_Insert(t *testing.T) { }, }, beforeFunc: func(s *server) { - s.ngt.Insert(id, bVec) + s.ngt.Insert(id, bVecs[0]) }, want: want{ err: status.WrapWithAlreadyExists(fmt.Sprintf("Insert API uuid %s already exists", id), errors.ErrUUIDAlreadyExists(id), @@ -3545,7 +3574,10 @@ func Test_server_Insert(t *testing.T) { } }(), func() test { - bVec := vector.GenIntVec(vector.Gaussian, 1, intVecDim)[0] // use in beforeFunc + bVec, err := vector.GenIntVec(vector.Gaussian, 1, intVecDim) // use in beforeFunc + if err != nil { + t.Error(err) + } req := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -3578,7 +3610,7 @@ func Test_server_Insert(t *testing.T) { }, }, beforeFunc: func(s *server) { - s.ngt.Insert(id, bVec) + s.ngt.Insert(id, bVec[0]) }, want: want{ err: status.WrapWithAlreadyExists(fmt.Sprintf("Insert API uuid %s already exists", id), errors.ErrUUIDAlreadyExists(id), @@ -4021,11 +4053,16 @@ func Test_server_MultiInsert(t *testing.T) { tests := []test{ func() test { insertNum := 1 + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + if err != nil { + t.Error(err) + } + return test{ name: "Equivalence Class Testing case 1.1: Success to MultiInsert 1 vector (vector type is uint8)", args: args{ ctx: ctx, - reqs: request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil), + reqs: req, }, fields: fields{ name: name, @@ -4041,11 +4078,16 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 1 + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } + return test{ name: "Equivalence Class Testing case 1.2: Success to MultiInsert 1 vector (vector type is float32)", args: args{ ctx: ctx, - reqs: request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil), + reqs: req, }, fields: fields{ name: name, @@ -4061,11 +4103,16 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + if err != nil { + t.Error(err) + } + return test{ name: "Equivalence Class Testing case 1.3: Success to MultiInsert 100 vector (vector type is uint8)", args: args{ ctx: ctx, - reqs: request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil), + reqs: req, }, fields: fields{ name: name, @@ -4081,11 +4128,16 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } + return test{ name: "Equivalence Class Testing case 1.4: Success to MultiInsert 100 vector (vector type is float32)", args: args{ ctx: ctx, - reqs: request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil), + reqs: req, }, fields: fields{ name: name, @@ -4139,7 +4191,10 @@ func Test_server_MultiInsert(t *testing.T) { }, func() test { insertNum := 1 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim+1, nil) + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim+1, nil) + if err != nil { + t.Error(err) + } return test{ name: "Equivalence Class Testing case 2.1: Fail to MultiInsert 1 vector with different dimension (vector type is uint8)", @@ -4181,7 +4236,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 1 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim+1, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim+1, nil) + if err != nil { + t.Error(err) + } return test{ name: "Equivalence Class Testing case 2.2: Fail to MultiInsert 1 vector with different dimension (vector type is float32)", @@ -4223,8 +4281,16 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) - req.Requests[0].Vector.Vector = vector.GenIntVec(vector.Gaussian, 1, intVecDim+1)[0] + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + if err != nil { + t.Error(err) + } + + invalidVecs, err := vector.GenIntVec(vector.Gaussian, 1, intVecDim+1) + if err != nil { + t.Error(err) + } + req.Requests[0].Vector.Vector = invalidVecs[0] return test{ name: "Equivalence Class Testing case 3.1: Fail to MultiInsert 100 vector with 1 vector with different dimension (vector type is uint8)", @@ -4266,8 +4332,16 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) - req.Requests[0].Vector.Vector = vector.GenF32Vec(vector.Gaussian, 1, f32VecDim+1)[0] + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } + + invalidVecs, err := vector.GenF32Vec(vector.Gaussian, 1, f32VecDim+1) + if err != nil { + t.Error(err) + } + req.Requests[0].Vector.Vector = invalidVecs[0] return test{ name: "Equivalence Class Testing case 3.2: Fail to MultiInsert 100 vector with 1 vector with different dimension (vector type is float32)", @@ -4310,9 +4384,18 @@ func Test_server_MultiInsert(t *testing.T) { func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) - for i := 0; i < len(req.Requests)/2; i++ { - req.Requests[i].Vector.Vector = vector.GenIntVec(vector.Gaussian, 1, intVecDim+1)[0] + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + if err != nil { + t.Error(err) + } + + invalidCnt := len(req.Requests) / 2 + invalidVec, err := vector.GenIntVec(vector.Gaussian, invalidCnt, intVecDim+1) + if err != nil { + t.Error(err) + } + for i := 0; i < invalidCnt; i++ { + req.Requests[i].Vector.Vector = invalidVec[i] } return test{ @@ -4355,9 +4438,18 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) - for i := 0; i < len(req.Requests)/2; i++ { - req.Requests[i].Vector.Vector = vector.GenF32Vec(vector.Gaussian, 1, f32VecDim+1)[0] + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } + + invalidCnt := len(req.Requests) / 2 + invalidVec, err := vector.GenF32Vec(vector.Gaussian, invalidCnt, f32VecDim+1) + if err != nil { + t.Error(err) + } + for i := 0; i < invalidCnt; i++ { + req.Requests[i].Vector.Vector = invalidVec[i] } return test{ @@ -4400,7 +4492,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim+1, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim+1, nil) + if err != nil { + t.Error(err) + } return test{ name: "Equivalence Class Testing case 3.5: Fail to MultiInsert 100 vector with all vector with different dimension (vector type is uint8)", @@ -4442,7 +4537,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim+1, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim+1, nil) + if err != nil { + t.Error(err) + } return test{ name: "Equivalence Class Testing case 3.6: Fail to MultiInsert 100 vector with all vector with different dimension (vector type is float32)", @@ -4604,7 +4702,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + if err != nil { + t.Error(err) + } req.Requests[0].Vector.Id = "" uuids := make([]string, 0, len(req.Requests)) @@ -4648,7 +4749,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } req.Requests[0].Vector.Id = "" uuids := make([]string, 0, len(req.Requests)) @@ -4692,7 +4796,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + if err != nil { + t.Error(err) + } for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Id = "" } @@ -4738,7 +4845,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Id = "" } @@ -4784,7 +4894,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + if err != nil { + t.Error(err) + } for i := 0; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = "" } @@ -4830,7 +4943,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } for i := 0; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = "" } @@ -4876,7 +4992,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + if err != nil { + t.Error(err) + } req.Requests[0].Vector.Vector = make([]float32, maxVecDim) return test{ @@ -4920,7 +5039,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } req.Requests[0].Vector.Vector = make([]float32, maxVecDim) return test{ @@ -4964,7 +5086,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, intVecDim, nil) + if err != nil { + t.Error(err) + } for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Vector = make([]float32, maxVecDim) } @@ -5010,7 +5135,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Uint8, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Vector = make([]float32, maxVecDim) } @@ -5222,7 +5350,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } vid := req.Requests[0].Vector.Id req.Requests[0] = nil @@ -5266,7 +5397,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i] = nil @@ -5312,7 +5446,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests); i++ { req.Requests[i] = nil @@ -5358,7 +5495,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } vid := req.Requests[0].Vector.Id req.Requests[0].Vector.Vector = nil @@ -5402,7 +5542,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Vector = nil @@ -5448,7 +5591,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests); i++ { req.Requests[i].Vector.Vector = nil @@ -5494,7 +5640,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } vid := req.Requests[0].Vector.Id req.Requests[0].Vector.Vector = []float32{} @@ -5538,7 +5687,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests)/2; i++ { req.Requests[i].Vector.Vector = []float32{} @@ -5584,7 +5736,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } vid := req.Requests[0].Vector.Id for i := 0; i < len(req.Requests); i++ { req.Requests[i].Vector.Vector = []float32{} @@ -5634,7 +5789,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: false, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } req.Requests[0].Vector.Id = req.Requests[1].Vector.Id // set want @@ -5664,7 +5822,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: false, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = req.Requests[0].Vector.Id } @@ -5698,7 +5859,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } req.Requests[0].Vector.Id = req.Requests[1].Vector.Id // set want @@ -5729,7 +5893,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = req.Requests[0].Vector.Id } @@ -5763,7 +5930,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: false, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } req.Requests[0].Vector.Vector = req.Requests[1].Vector.Vector return test{ @@ -5789,7 +5959,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: false, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Vector = req.Requests[0].Vector.Vector } @@ -5817,7 +5990,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } req.Requests[0].Vector.Vector = req.Requests[1].Vector.Vector return test{ @@ -5843,7 +6019,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Vector = req.Requests[0].Vector.Vector } @@ -5870,7 +6049,10 @@ func Test_server_MultiInsert(t *testing.T) { cfg := &payload.Insert_Config{ SkipStrictExistCheck: false, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } req.Requests[0].Vector.Vector = req.Requests[1].Vector.Vector req.Requests[0].Vector.Id = req.Requests[1].Vector.Id @@ -5901,7 +6083,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: false, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = req.Requests[0].Vector.Id req.Requests[i].Vector.Vector = req.Requests[0].Vector.Vector @@ -5936,7 +6121,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } req.Requests[0].Vector.Vector = req.Requests[1].Vector.Vector req.Requests[0].Vector.Id = req.Requests[1].Vector.Id @@ -5967,7 +6155,10 @@ func Test_server_MultiInsert(t *testing.T) { SkipStrictExistCheck: true, } - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, cfg) + if err != nil { + t.Error(err) + } for i := 1; i < len(req.Requests); i++ { req.Requests[i].Vector.Id = req.Requests[0].Vector.Id req.Requests[i].Vector.Vector = req.Requests[0].Vector.Vector @@ -5998,7 +6189,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } return test{ name: "Decision Table Testing case 4.1: Fail to MultiInsert with 2 existed ID when SkipStrictExistCheck is false", @@ -6013,7 +6207,10 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, beforeFunc: func(t *testing.T, s *server) { - vecs := vector.GenF32Vec(vector.Gaussian, 2, f32VecDim) + vecs, err := vector.GenF32Vec(vector.Gaussian, 2, f32VecDim) + if err != nil { + t.Error(err) + } for i := 0; i < 2; i++ { ir := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -6045,7 +6242,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } wantErrs := make([]error, 100) for i := 0; i < len(req.Requests); i++ { @@ -6065,7 +6265,10 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, beforeFunc: func(t *testing.T, s *server) { - vecs := vector.GenF32Vec(vector.Gaussian, insertNum, f32VecDim) + vecs, err := vector.GenF32Vec(vector.Gaussian, insertNum, f32VecDim) + if err != nil { + t.Error(err) + } for i, r := range req.Requests { ir := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -6093,7 +6296,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } return test{ name: "Decision Table Testing case 4.3: Fail to MultiInsert with 2 existed ID when SkipStrictExistCheck is true", @@ -6108,7 +6314,10 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, beforeFunc: func(t *testing.T, s *server) { - vecs := vector.GenF32Vec(vector.Gaussian, 2, f32VecDim) + vecs, err := vector.GenF32Vec(vector.Gaussian, 2, f32VecDim) + if err != nil { + t.Error(err) + } for i := 0; i < 2; i++ { ir := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -6140,7 +6349,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } wantErrs := make([]error, 100) for i := 0; i < len(req.Requests); i++ { @@ -6160,7 +6372,10 @@ func Test_server_MultiInsert(t *testing.T) { svcOpts: defaultSvcOpts, }, beforeFunc: func(t *testing.T, s *server) { - vecs := vector.GenF32Vec(vector.Gaussian, insertNum, f32VecDim) + vecs, err := vector.GenF32Vec(vector.Gaussian, insertNum, f32VecDim) + if err != nil { + t.Error(err) + } for i, r := range req.Requests { ir := &payload.Insert_Request{ Vector: &payload.Object_Vector{ @@ -6188,7 +6403,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } return test{ name: "Decision Table Testing case 5.1: Success to MultiInsert with 2 existed vector when SkipStrictExistCheck is false", @@ -6231,7 +6449,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } return test{ name: "Decision Table Testing case 5.2: Success to MultiInsert with all existed vector when SkipStrictExistCheck is false", @@ -6274,7 +6495,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } return test{ name: "Decision Table Testing case 5.3: Success to MultiInsert with 2 existed vector when SkipStrictExistCheck is true", @@ -6317,7 +6541,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } return test{ name: "Decision Table Testing case 5.4: Success to MultiInsert with all existed vector when SkipStrictExistCheck is true", @@ -6360,7 +6587,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } return test{ name: "Decision Table Testing case 6.1: Fail to MultiInsert with 2 existed ID & vector when SkipStrictExistCheck is false", @@ -6403,7 +6633,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } wantErrs := make([]error, 100) for i := 0; i < len(req.Requests); i++ { @@ -6447,7 +6680,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } return test{ name: "Decision Table Testing case 6.3: Fail to MultiInsert with 2 existed ID & vector when SkipStrictExistCheck is true", @@ -6490,7 +6726,10 @@ func Test_server_MultiInsert(t *testing.T) { }(), func() test { insertNum := 100 - req := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + req, err := request.GenMultiInsertReq(request.Float, vector.Gaussian, insertNum, f32VecDim, nil) + if err != nil { + t.Error(err) + } wantErrs := make([]error, 100) for i := 0; i < len(req.Requests); i++ { From 7691ea4586840d2da0d1d78f8cca6ac055ca60a4 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Mon, 30 May 2022 16:29:40 +0900 Subject: [PATCH 07/29] refactor conv helper func Signed-off-by: kevindiu --- internal/conv/conv.go | 18 +- .../core/ngt/handler/grpc/handler_test.go | 196 +++++++++++------- 2 files changed, 130 insertions(+), 84 deletions(-) diff --git a/internal/conv/conv.go b/internal/conv/conv.go index 61685ed8c8..1d3f1219f4 100644 --- a/internal/conv/conv.go +++ b/internal/conv/conv.go @@ -54,12 +54,18 @@ func F32stos(fs []float32) (s string) { return Btoa(buf) } -func Utf8ToSjis(s string) string { - b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())) - return string(b) +func Utf8ToSjis(s string) (string, error) { + b, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())) + if err != nil { + return "", err + } + return string(b), nil } -func Utf8ToEucjp(s string) string { - b, _ := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.EUCJP.NewEncoder())) - return string(b) +func Utf8ToEucjp(s string) (string, error) { + b, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.EUCJP.NewEncoder())) + if err != nil { + return "", err + } + return string(b), nil } diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 552dfb941a..4119e678ba 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -368,6 +368,16 @@ func Test_server_Exists(t *testing.T) { insertNum = 1000 ) + jpStr := "こんにちは" + eucjpStr, err := conv.Utf8ToEucjp(jpStr) + if err != nil { + t.Error(err) + } + sjisStr, err := conv.Utf8ToSjis(jpStr) + if err != nil { + t.Error(err) + } + defaultNgtConfig := &config.NGT{ Dimension: 128, DistanceType: ngt.L2.String(), @@ -532,8 +542,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.1: success exists with utf-8 ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", - searchId: "こんにちは", + indexId: jpStr, + searchId: jpStr, }, want: want{}, }, @@ -541,8 +551,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.2: fail exists with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), - searchId: "こんにちは", + indexId: sjisStr, + searchId: jpStr, }, want: want{ code: codes.NotFound, @@ -552,8 +562,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.3: fail exists with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), - searchId: "こんにちは", + indexId: eucjpStr, + searchId: jpStr, }, want: want{ code: codes.NotFound, @@ -563,8 +573,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.4: fail exists with s-jis ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", - searchId: conv.Utf8ToSjis("こんにちは"), + indexId: jpStr, + searchId: sjisStr, }, want: want{ code: codes.NotFound, @@ -574,8 +584,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.5: success exists with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), - searchId: conv.Utf8ToSjis("こんにちは"), + indexId: sjisStr, + searchId: sjisStr, }, want: want{}, }, @@ -583,8 +593,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.6: fail exists with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), - searchId: conv.Utf8ToSjis("こんにちは"), + indexId: eucjpStr, + searchId: sjisStr, }, want: want{ code: codes.NotFound, @@ -594,8 +604,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.7: fail exists with euc-jp ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", - searchId: conv.Utf8ToEucjp("こんにちは"), + indexId: jpStr, + searchId: eucjpStr, }, want: want{ code: codes.NotFound, @@ -605,8 +615,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.8: fail exists with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), - searchId: conv.Utf8ToEucjp("こんにちは"), + indexId: sjisStr, + searchId: eucjpStr, }, want: want{ code: codes.NotFound, @@ -616,8 +626,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.9: success exists with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), - searchId: conv.Utf8ToEucjp("こんにちは"), + indexId: eucjpStr, + searchId: eucjpStr, }, want: want{}, }, @@ -1551,6 +1561,16 @@ func Test_server_SearchByID(t *testing.T) { insertNum = 1000 ) + jpStr := "こんにちは" + eucjpStr, err := conv.Utf8ToEucjp(jpStr) + if err != nil { + t.Error(err) + } + sjisStr, err := conv.Utf8ToSjis(jpStr) + if err != nil { + t.Error(err) + } + defaultNgtConfig := &config.NGT{ Dimension: 128, DistanceType: ngt.L2.String(), @@ -1737,8 +1757,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.1: success search with utf-8 ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", - searchId: "こんにちは", + indexId: jpStr, + searchId: jpStr, }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -1748,8 +1768,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.2: fail search with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), - searchId: "こんにちは", + indexId: sjisStr, + searchId: jpStr, }, want: want{ resultSize: 0, @@ -1760,8 +1780,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.3: fail search with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), - searchId: "こんにちは", + indexId: eucjpStr, + searchId: jpStr, }, want: want{ resultSize: 0, @@ -1772,8 +1792,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.4: fail search with s-jis ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", - searchId: conv.Utf8ToSjis("こんにちは"), + indexId: jpStr, + searchId: sjisStr, }, want: want{ resultSize: 0, @@ -1784,8 +1804,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.5: success search with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), - searchId: conv.Utf8ToSjis("こんにちは"), + indexId: sjisStr, + searchId: sjisStr, }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -1795,8 +1815,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.6: fail search with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), - searchId: conv.Utf8ToSjis("こんにちは"), + indexId: eucjpStr, + searchId: sjisStr, }, want: want{ resultSize: 0, @@ -1807,8 +1827,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.7: fail search with euc-jp ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", - searchId: conv.Utf8ToEucjp("こんにちは"), + indexId: jpStr, + searchId: eucjpStr, }, want: want{ resultSize: 0, @@ -1819,8 +1839,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.8: fail search with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), - searchId: conv.Utf8ToEucjp("こんにちは"), + indexId: sjisStr, + searchId: eucjpStr, }, want: want{ resultSize: 0, @@ -1831,8 +1851,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.9: success search with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), - searchId: conv.Utf8ToEucjp("こんにちは"), + indexId: eucjpStr, + searchId: eucjpStr, }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -6858,6 +6878,16 @@ func Test_server_Update(t *testing.T) { dimension = 128 ) + jpStr := "こんにちは" + eucjpStr, err := conv.Utf8ToEucjp(jpStr) + if err != nil { + t.Error(err) + } + sjisStr, err := conv.Utf8ToSjis(jpStr) + if err != nil { + t.Error(err) + } + defaultUpdateConfig := &payload.Update_Config{ SkipStrictExistCheck: true, } @@ -7164,27 +7194,27 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.1: success update with utf-8 ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", + indexId: jpStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: "こんにちは", + Id: jpStr, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, }, }, want: want{ - wantUuid: "こんにちは", + wantUuid: jpStr, }, }, { name: "Boundary Value Testing case 3.2: success update with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), + indexId: sjisStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: "こんにちは", + Id: jpStr, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7198,10 +7228,10 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.3: success update with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), + indexId: eucjpStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: "こんにちは", + Id: jpStr, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7215,10 +7245,10 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.4: fail update with s-jis ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", + indexId: jpStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: conv.Utf8ToSjis("こんにちは"), + Id: sjisStr, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7232,27 +7262,27 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.5: success update with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), + indexId: sjisStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: conv.Utf8ToSjis("こんにちは"), + Id: sjisStr, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, }, }, want: want{ - wantUuid: conv.Utf8ToSjis("こんにちは"), + wantUuid: sjisStr, }, }, { name: "Boundary Value Testing case 3.6: fail update with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), + indexId: eucjpStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: conv.Utf8ToSjis("こんにちは"), + Id: sjisStr, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7266,10 +7296,10 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.7: fail update with euc-jp ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", + indexId: jpStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: conv.Utf8ToEucjp("こんにちは"), + Id: eucjpStr, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7283,10 +7313,10 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.8: fail update with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), + indexId: sjisStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: conv.Utf8ToEucjp("こんにちは"), + Id: eucjpStr, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7300,17 +7330,17 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.9: success update with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), + indexId: eucjpStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: conv.Utf8ToEucjp("こんにちは"), + Id: eucjpStr, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, }, }, want: want{ - wantUuid: conv.Utf8ToEucjp("こんにちは"), + wantUuid: eucjpStr, }, }, { @@ -8270,6 +8300,16 @@ func Test_server_Remove(t *testing.T) { insertNum = 1000 ) + jpStr := "こんにちは" + eucjpStr, err := conv.Utf8ToEucjp(jpStr) + if err != nil { + t.Error(err) + } + sjisStr, err := conv.Utf8ToSjis(jpStr) + if err != nil { + t.Error(err) + } + defaultNgtConfig := &config.NGT{ Dimension: 128, DistanceType: ngt.L2.String(), @@ -8448,19 +8488,19 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.1: success exists with utf-8 ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", - removeId: "こんにちは", + indexId: jpStr, + removeId: jpStr, }, want: want{ - wantUuid: "こんにちは", + wantUuid: jpStr, }, }, { name: "Boundary Value Testing case 3.2: fail exists with utf-8 ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), - removeId: "こんにちは", + indexId: sjisStr, + removeId: jpStr, }, want: want{ code: codes.NotFound, @@ -8470,8 +8510,8 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.3: fail exists with utf-8 ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), - removeId: "こんにちは", + indexId: eucjpStr, + removeId: jpStr, }, want: want{ code: codes.NotFound, @@ -8481,8 +8521,8 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.4: fail exists with s-jis ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", - removeId: conv.Utf8ToSjis("こんにちは"), + indexId: jpStr, + removeId: sjisStr, }, want: want{ code: codes.NotFound, @@ -8492,19 +8532,19 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.5: success exists with s-jis ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), - removeId: conv.Utf8ToSjis("こんにちは"), + indexId: sjisStr, + removeId: sjisStr, }, want: want{ - wantUuid: conv.Utf8ToSjis("こんにちは"), + wantUuid: sjisStr, }, }, { name: "Boundary Value Testing case 3.6: fail exists with s-jis ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), - removeId: conv.Utf8ToSjis("こんにちは"), + indexId: eucjpStr, + removeId: sjisStr, }, want: want{ code: codes.NotFound, @@ -8514,8 +8554,8 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.7: fail exists with euc-jp ID from utf-8 index", args: args{ ctx: ctx, - indexId: "こんにちは", - removeId: conv.Utf8ToEucjp("こんにちは"), + indexId: jpStr, + removeId: eucjpStr, }, want: want{ code: codes.NotFound, @@ -8525,8 +8565,8 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.8: fail exists with euc-jp ID from s-jis index", args: args{ ctx: ctx, - indexId: conv.Utf8ToSjis("こんにちは"), - removeId: conv.Utf8ToEucjp("こんにちは"), + indexId: sjisStr, + removeId: eucjpStr, }, want: want{ code: codes.NotFound, @@ -8536,11 +8576,11 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.9: success exists with euc-jp ID from euc-jp index", args: args{ ctx: ctx, - indexId: conv.Utf8ToEucjp("こんにちは"), - removeId: conv.Utf8ToEucjp("こんにちは"), + indexId: eucjpStr, + removeId: eucjpStr, }, want: want{ - wantUuid: conv.Utf8ToEucjp("こんにちは"), + wantUuid: eucjpStr, }, }, { From 1cddad7550e679d38c9dbb3046a387db70dce400 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Tue, 31 May 2022 15:54:17 +0900 Subject: [PATCH 08/29] refactor fill() and implement buildIndex() Signed-off-by: kevindiu --- .../core/ngt/handler/grpc/handler_test.go | 129 ++++++++++-------- 1 file changed, 70 insertions(+), 59 deletions(-) diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 4119e678ba..0ca7a2c655 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -45,6 +45,51 @@ import ( "github.com/vdaas/vald/pkg/agent/core/ngt/service" ) +func buildIndex(ctx context.Context, t request.ObjectType, dist vector.Distribution, num int, insertCfg *payload.Insert_Config, + ngtCfg *config.NGT, ngtOpts []service.Option, overwriteIDs []string, overwriteVectors [][]float32) (Server, error) { + + eg, ctx := errgroup.New(ctx) + ngt, err := service.New(ngtCfg, append(ngtOpts, service.WithErrGroup(eg), service.WithEnableInMemoryMode(true))...) + if err != nil { + return nil, err + } + + s, err := New(WithErrGroup(eg), WithNGT(ngt)) + if err != nil { + return nil, err + } + + if num > 0 { + // gen insert request + reqs, err := request.GenMultiInsertReq(t, dist, num, ngtCfg.Dimension, insertCfg) + if err != nil { + return nil, err + } + + // overwrite ID if needed + for i, id := range overwriteIDs { + reqs.Requests[i].Vector.Id = id + } + + // overwrite Vectors if needed + for i, v := range overwriteVectors { + reqs.Requests[i].Vector.Vector = v + } + + // insert and create index + if _, err := s.MultiInsert(ctx, reqs); err != nil { + return nil, err + } + if _, err := s.CreateIndex(ctx, &payload.Control_CreateIndexRequest{ + PoolSize: 100, + }); err != nil { + return nil, err + } + } + + return s, nil +} + func TestNew(t *testing.T) { t.Parallel() type args struct { @@ -366,6 +411,7 @@ func Test_server_Exists(t *testing.T) { const ( insertNum = 1000 + dim = 128 ) jpStr := "こんにちは" @@ -379,7 +425,7 @@ func Test_server_Exists(t *testing.T) { } defaultNgtConfig := &config.NGT{ - Dimension: 128, + Dimension: dim, DistanceType: ngt.L2.String(), ObjectType: ngt.Float.String(), CreationEdgeSize: 60, @@ -392,38 +438,11 @@ func Test_server_Exists(t *testing.T) { DeleteBufferPoolSize: 1000, }, } + defaultInsertConfig := &payload.Insert_Config{ + SkipStrictExistCheck: true, + } defaultBeforeFunc := func(a args) (Server, error) { - eg, ctx := errgroup.New(a.ctx) - ngt, err := service.New(defaultNgtConfig, service.WithErrGroup(eg), service.WithEnableInMemoryMode(true)) - if err != nil { - return nil, err - } - - s, err := New(WithErrGroup(eg), WithNGT(ngt)) - if err != nil { - return nil, err - } - - reqs := make([]*payload.Insert_Request, insertNum) - for i, v := range vector.GaussianDistributedFloat32VectorGenerator(insertNum, defaultNgtConfig.Dimension) { - reqs[i] = &payload.Insert_Request{ - Vector: &payload.Object_Vector{ - Id: strconv.Itoa(i), - Vector: v, - }, - Config: &payload.Insert_Config{ - SkipStrictExistCheck: true, - }, - } - } - reqs[0].Vector.Id = a.indexId - if _, err := s.MultiInsert(ctx, &payload.Insert_MultiRequest{Requests: reqs}); err != nil { - return nil, err - } - if _, err := s.CreateIndex(ctx, &payload.Control_CreateIndexRequest{PoolSize: 100}); err != nil { - return nil, err - } - return s, nil + return buildIndex(a.ctx, request.Float, vector.Gaussian, insertNum, defaultInsertConfig, defaultNgtConfig, nil, []string{a.indexId}, nil) } /* @@ -689,14 +708,6 @@ func convertVectorsUint8ToFloat32(vectors [][]uint8) (ret [][]float32) { return } -func fill(f float32, dim int) (v []float32) { - v = make([]float32, dim) - for i := range v { - v[i] = f - } - return -} - func Test_server_Search(t *testing.T) { t.Parallel() @@ -937,7 +948,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: fill(float32(uint8(0)), defaultDimensionSize), + Vector: vector.GenSameValueVec(defaultDimensionSize, float32(uint8(0))), Config: defaultSearch_Config, }, }, @@ -957,7 +968,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: fill(+0.0, defaultDimensionSize), + Vector: vector.GenSameValueVec(defaultDimensionSize, +0.0), Config: defaultSearch_Config, }, }, @@ -975,7 +986,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: fill(float32(math.Copysign(0, -1.0)), defaultDimensionSize), + Vector: vector.GenSameValueVec(defaultDimensionSize, float32(math.Copysign(0, -1.0))), Config: defaultSearch_Config, }, }, @@ -993,7 +1004,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: fill(float32(math.MaxUint8), defaultDimensionSize), + Vector: vector.GenSameValueVec(defaultDimensionSize, float32(math.MaxUint8)), Config: defaultSearch_Config, }, }, @@ -1013,7 +1024,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: fill(math.MaxFloat32, defaultDimensionSize), + Vector: vector.GenSameValueVec(defaultDimensionSize, math.MaxFloat32), Config: defaultSearch_Config, }, }, @@ -1032,7 +1043,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: fill(-math.MaxFloat32, defaultDimensionSize), + Vector: vector.GenSameValueVec(defaultDimensionSize, -math.MaxFloat32), Config: defaultSearch_Config, }, }, @@ -1051,7 +1062,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: fill(float32(math.NaN()), defaultDimensionSize), + Vector: vector.GenSameValueVec(defaultDimensionSize, float32(math.NaN())), Config: defaultSearch_Config, }, }, @@ -1070,7 +1081,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: fill(float32(math.Inf(+1.0)), defaultDimensionSize), + Vector: vector.GenSameValueVec(defaultDimensionSize, float32(math.Inf(+1.0))), Config: defaultSearch_Config, }, }, @@ -1089,7 +1100,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: fill(float32(math.Inf(-1.0)), defaultDimensionSize), + Vector: vector.GenSameValueVec(defaultDimensionSize, float32(math.Inf(-1.0))), Config: defaultSearch_Config, }, }, @@ -7368,7 +7379,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: fill(float32(uint8(0)), dimension), + Vector: vector.GenSameValueVec(dimension, float32(uint8(0))), }, Config: defaultUpdateConfig, }, @@ -7385,7 +7396,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: fill(0, dimension), + Vector: vector.GenSameValueVec(dimension, 0), }, Config: defaultUpdateConfig, }, @@ -7402,7 +7413,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: fill(float32(math.Copysign(0, -1.0)), dimension), + Vector: vector.GenSameValueVec(dimension, float32(math.Copysign(0, -1.0))), }, Config: defaultUpdateConfig, }, @@ -7419,7 +7430,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: fill(float32(uint8(0)), dimension), + Vector: vector.GenSameValueVec(dimension, float32(uint8(0))), }, Config: defaultUpdateConfig, }, @@ -7436,7 +7447,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: fill(-math.MaxFloat32, dimension), + Vector: vector.GenSameValueVec(dimension, -math.MaxFloat32), }, Config: defaultUpdateConfig, }, @@ -7453,7 +7464,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: fill(float32(math.MaxUint8), dimension), + Vector: vector.GenSameValueVec(dimension, float32(math.MaxUint8)), }, Config: defaultUpdateConfig, }, @@ -7470,7 +7481,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: fill(math.MaxFloat32, dimension), + Vector: vector.GenSameValueVec(dimension, math.MaxFloat32), }, Config: defaultUpdateConfig, }, @@ -7487,7 +7498,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: fill(float32(math.NaN()), dimension), + Vector: vector.GenSameValueVec(dimension, float32(math.NaN())), }, Config: defaultUpdateConfig, }, @@ -7504,7 +7515,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: fill(float32(math.Inf(1.0)), dimension), + Vector: vector.GenSameValueVec(dimension, float32(math.Inf(1.0))), }, Config: defaultUpdateConfig, }, @@ -7521,7 +7532,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: fill(float32(math.Inf(-1.0)), dimension), + Vector: vector.GenSameValueVec(dimension, float32(math.Inf(-1.0))), }, Config: defaultUpdateConfig, }, From 90be937b5b78cb8b80d1bdf96e1c1b31f3ff7617 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Tue, 31 May 2022 15:56:59 +0900 Subject: [PATCH 09/29] fix comment Signed-off-by: kevindiu --- internal/test/data/request/insert.go | 2 +- internal/test/data/vector/gen.go | 2 +- .../core/ngt/handler/grpc/handler_test.go | 90 +++++++++---------- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/internal/test/data/request/insert.go b/internal/test/data/request/insert.go index cb2a657d60..93dd068b64 100644 --- a/internal/test/data/request/insert.go +++ b/internal/test/data/request/insert.go @@ -21,7 +21,7 @@ func GenMultiInsertReq(t ObjectType, dist vector.Distribution, num int, dim int, case Float: vecs, err = vector.GenF32Vec(dist, num, dim) case Uint8: - vecs, err = vector.GenIntVec(dist, num, dim) + vecs, err = vector.GenUint8Vec(dist, num, dim) } if err != nil { return nil, err diff --git a/internal/test/data/vector/gen.go b/internal/test/data/vector/gen.go index f8d28e5d05..398abad37a 100644 --- a/internal/test/data/vector/gen.go +++ b/internal/test/data/vector/gen.go @@ -140,7 +140,7 @@ func GenF32Vec(dist Distribution, num int, dim int) ([][]float32, error) { return generator(num, dim), nil } -func GenIntVec(dist Distribution, num int, dim int) ([][]float32, error) { +func GenUint8Vec(dist Distribution, num int, dim int) ([][]float32, error) { generator, err := Uint8VectorGenerator(dist) if err != nil { return nil, err diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 0ca7a2c655..f9f2464404 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -414,12 +414,12 @@ func Test_server_Exists(t *testing.T) { dim = 128 ) - jpStr := "こんにちは" - eucjpStr, err := conv.Utf8ToEucjp(jpStr) + utf8Str := "こんにちは" + eucjpStr, err := conv.Utf8ToEucjp(utf8Str) if err != nil { t.Error(err) } - sjisStr, err := conv.Utf8ToSjis(jpStr) + sjisStr, err := conv.Utf8ToSjis(utf8Str) if err != nil { t.Error(err) } @@ -561,8 +561,8 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.1: success exists with utf-8 ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, - searchId: jpStr, + indexId: utf8Str, + searchId: utf8Str, }, want: want{}, }, @@ -571,7 +571,7 @@ func Test_server_Exists(t *testing.T) { args: args{ ctx: ctx, indexId: sjisStr, - searchId: jpStr, + searchId: utf8Str, }, want: want{ code: codes.NotFound, @@ -582,7 +582,7 @@ func Test_server_Exists(t *testing.T) { args: args{ ctx: ctx, indexId: eucjpStr, - searchId: jpStr, + searchId: utf8Str, }, want: want{ code: codes.NotFound, @@ -592,7 +592,7 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.4: fail exists with s-jis ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, + indexId: utf8Str, searchId: sjisStr, }, want: want{ @@ -623,7 +623,7 @@ func Test_server_Exists(t *testing.T) { name: "Boundary Value Testing case 3.7: fail exists with euc-jp ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, + indexId: utf8Str, searchId: eucjpStr, }, want: want{ @@ -1572,12 +1572,12 @@ func Test_server_SearchByID(t *testing.T) { insertNum = 1000 ) - jpStr := "こんにちは" - eucjpStr, err := conv.Utf8ToEucjp(jpStr) + utf8Str := "こんにちは" + eucjpStr, err := conv.Utf8ToEucjp(utf8Str) if err != nil { t.Error(err) } - sjisStr, err := conv.Utf8ToSjis(jpStr) + sjisStr, err := conv.Utf8ToSjis(utf8Str) if err != nil { t.Error(err) } @@ -1768,8 +1768,8 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.1: success search with utf-8 ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, - searchId: jpStr, + indexId: utf8Str, + searchId: utf8Str, }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -1780,7 +1780,7 @@ func Test_server_SearchByID(t *testing.T) { args: args{ ctx: ctx, indexId: sjisStr, - searchId: jpStr, + searchId: utf8Str, }, want: want{ resultSize: 0, @@ -1792,7 +1792,7 @@ func Test_server_SearchByID(t *testing.T) { args: args{ ctx: ctx, indexId: eucjpStr, - searchId: jpStr, + searchId: utf8Str, }, want: want{ resultSize: 0, @@ -1803,7 +1803,7 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.4: fail search with s-jis ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, + indexId: utf8Str, searchId: sjisStr, }, want: want{ @@ -1838,7 +1838,7 @@ func Test_server_SearchByID(t *testing.T) { name: "Boundary Value Testing case 3.7: fail search with euc-jp ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, + indexId: utf8Str, searchId: eucjpStr, }, want: want{ @@ -2600,7 +2600,7 @@ func Test_server_Insert(t *testing.T) { }(), func() test { invalidDim := intVecDim + 1 - ivec, err := vector.GenIntVec(vector.Gaussian, 1, invalidDim) + ivec, err := vector.GenUint8Vec(vector.Gaussian, 1, invalidDim) if err != nil { t.Error(err) } @@ -2715,7 +2715,7 @@ func Test_server_Insert(t *testing.T) { } }(), func() test { - ivec, err := vector.GenIntVec(vector.Gaussian, 1, intVecDim) + ivec, err := vector.GenUint8Vec(vector.Gaussian, 1, intVecDim) if err != nil { t.Error(err) } @@ -2799,7 +2799,7 @@ func Test_server_Insert(t *testing.T) { } }(), func() test { - ivec, err := vector.GenIntVec(vector.Uniform, 1, intVecDim) + ivec, err := vector.GenUint8Vec(vector.Uniform, 1, intVecDim) if err != nil { t.Error(err) } @@ -3461,7 +3461,7 @@ func Test_server_Insert(t *testing.T) { // Decision Table Testing func() test { - bVecs, err := vector.GenIntVec(vector.Gaussian, 1, intVecDim) // used in beforeFunc + bVecs, err := vector.GenUint8Vec(vector.Gaussian, 1, intVecDim) // used in beforeFunc if err != nil { t.Error(err) } @@ -3605,7 +3605,7 @@ func Test_server_Insert(t *testing.T) { } }(), func() test { - bVec, err := vector.GenIntVec(vector.Gaussian, 1, intVecDim) // use in beforeFunc + bVec, err := vector.GenUint8Vec(vector.Gaussian, 1, intVecDim) // use in beforeFunc if err != nil { t.Error(err) } @@ -4317,7 +4317,7 @@ func Test_server_MultiInsert(t *testing.T) { t.Error(err) } - invalidVecs, err := vector.GenIntVec(vector.Gaussian, 1, intVecDim+1) + invalidVecs, err := vector.GenUint8Vec(vector.Gaussian, 1, intVecDim+1) if err != nil { t.Error(err) } @@ -4421,7 +4421,7 @@ func Test_server_MultiInsert(t *testing.T) { } invalidCnt := len(req.Requests) / 2 - invalidVec, err := vector.GenIntVec(vector.Gaussian, invalidCnt, intVecDim+1) + invalidVec, err := vector.GenUint8Vec(vector.Gaussian, invalidCnt, intVecDim+1) if err != nil { t.Error(err) } @@ -6889,12 +6889,12 @@ func Test_server_Update(t *testing.T) { dimension = 128 ) - jpStr := "こんにちは" - eucjpStr, err := conv.Utf8ToEucjp(jpStr) + utf8Str := "こんにちは" + eucjpStr, err := conv.Utf8ToEucjp(utf8Str) if err != nil { t.Error(err) } - sjisStr, err := conv.Utf8ToSjis(jpStr) + sjisStr, err := conv.Utf8ToSjis(utf8Str) if err != nil { t.Error(err) } @@ -7205,17 +7205,17 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.1: success update with utf-8 ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, + indexId: utf8Str, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: jpStr, + Id: utf8Str, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, }, }, want: want{ - wantUuid: jpStr, + wantUuid: utf8Str, }, }, { @@ -7225,7 +7225,7 @@ func Test_server_Update(t *testing.T) { indexId: sjisStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: jpStr, + Id: utf8Str, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7242,7 +7242,7 @@ func Test_server_Update(t *testing.T) { indexId: eucjpStr, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: jpStr, + Id: utf8Str, Vector: vector.GaussianDistributedFloat32VectorGenerator(1, dimension)[0], }, Config: defaultUpdateConfig, @@ -7256,7 +7256,7 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.4: fail update with s-jis ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, + indexId: utf8Str, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: sjisStr, @@ -7307,7 +7307,7 @@ func Test_server_Update(t *testing.T) { name: "Boundary Value Testing case 3.7: fail update with euc-jp ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, + indexId: utf8Str, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: eucjpStr, @@ -8311,12 +8311,12 @@ func Test_server_Remove(t *testing.T) { insertNum = 1000 ) - jpStr := "こんにちは" - eucjpStr, err := conv.Utf8ToEucjp(jpStr) + utf8Str := "こんにちは" + eucjpStr, err := conv.Utf8ToEucjp(utf8Str) if err != nil { t.Error(err) } - sjisStr, err := conv.Utf8ToSjis(jpStr) + sjisStr, err := conv.Utf8ToSjis(utf8Str) if err != nil { t.Error(err) } @@ -8499,11 +8499,11 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.1: success exists with utf-8 ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, - removeId: jpStr, + indexId: utf8Str, + removeId: utf8Str, }, want: want{ - wantUuid: jpStr, + wantUuid: utf8Str, }, }, { @@ -8511,7 +8511,7 @@ func Test_server_Remove(t *testing.T) { args: args{ ctx: ctx, indexId: sjisStr, - removeId: jpStr, + removeId: utf8Str, }, want: want{ code: codes.NotFound, @@ -8522,7 +8522,7 @@ func Test_server_Remove(t *testing.T) { args: args{ ctx: ctx, indexId: eucjpStr, - removeId: jpStr, + removeId: utf8Str, }, want: want{ code: codes.NotFound, @@ -8532,7 +8532,7 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.4: fail exists with s-jis ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, + indexId: utf8Str, removeId: sjisStr, }, want: want{ @@ -8565,7 +8565,7 @@ func Test_server_Remove(t *testing.T) { name: "Boundary Value Testing case 3.7: fail exists with euc-jp ID from utf-8 index", args: args{ ctx: ctx, - indexId: jpStr, + indexId: utf8Str, removeId: eucjpStr, }, want: want{ From 0796dc5105316bcba2d7b7c3d076f5f051ae6218 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Tue, 31 May 2022 16:37:07 +0900 Subject: [PATCH 10/29] refactor convertXXX Signed-off-by: kevindiu --- internal/test/data/vector/gen.go | 29 +++++---- .../core/ngt/handler/grpc/handler_test.go | 64 +++++++------------ 2 files changed, 41 insertions(+), 52 deletions(-) diff --git a/internal/test/data/vector/gen.go b/internal/test/data/vector/gen.go index 398abad37a..5bdd78445b 100644 --- a/internal/test/data/vector/gen.go +++ b/internal/test/data/vector/gen.go @@ -145,18 +145,7 @@ func GenUint8Vec(dist Distribution, num int, dim int) ([][]float32, error) { if err != nil { return nil, err } - - ivecs := generator(num, dim) - result := make([][]float32, num) - - for j, ivec := range ivecs { - vec := make([]float32, dim) - for i := 0; i < dim; i++ { - vec[i] = float32(ivec[i]) - } - result[j] = vec - } - return result, nil + return ConvertVectorsUint8ToFloat32(generator(num, dim)), nil } func GenSameValueVec(size int, val float32) []float32 { @@ -166,3 +155,19 @@ func GenSameValueVec(size int, val float32) []float32 { } return v } + +func ConvertVectorUint8ToFloat32(vector []uint8) (ret []float32) { + ret = make([]float32, len(vector)) + for i, e := range vector { + ret[i] = float32(e) + } + return +} + +func ConvertVectorsUint8ToFloat32(vectors [][]uint8) (ret [][]float32) { + ret = make([][]float32, 0, len(vectors)) + for _, v := range vectors { + ret = append(ret, ConvertVectorUint8ToFloat32(v)) + } + return +} diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index f9f2464404..fcaacec094 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -692,22 +692,6 @@ func Test_server_Exists(t *testing.T) { } } -func convertVectorUint8ToFloat32(vector []uint8) (ret []float32) { - ret = make([]float32, len(vector)) - for i, e := range vector { - ret[i] = float32(e) - } - return -} - -func convertVectorsUint8ToFloat32(vectors [][]uint8) (ret [][]float32) { - ret = make([][]float32, 0, len(vectors)) - for _, v := range vectors { - ret = append(ret, convertVectorUint8ToFloat32(v)) - } - return -} - func Test_server_Search(t *testing.T) { t.Parallel() @@ -868,13 +852,13 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), Config: defaultSearch_Config, }, }, fields: fields{ gen: func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) }, ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, @@ -906,13 +890,13 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize+1)[0]), + Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize+1)[0]), Config: defaultSearch_Config, }, }, fields: fields{ gen: func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) }, ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, @@ -954,7 +938,7 @@ func Test_server_Search(t *testing.T) { }, fields: fields{ gen: func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) }, ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, @@ -1010,7 +994,7 @@ func Test_server_Search(t *testing.T) { }, fields: fields{ gen: func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) }, ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, @@ -1125,7 +1109,7 @@ func Test_server_Search(t *testing.T) { }, fields: fields{ gen: func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) }, ngtCfg: ngtConfig(defaultDimensionSize, "uint8"), }, @@ -1159,13 +1143,13 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, math.MaxInt32>>7)[0]), + Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, math.MaxInt32>>7)[0]), Config: defaultSearch_Config, }, }, fields: fields{ gen: func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) }, ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, @@ -1205,7 +1189,7 @@ func Test_server_Search(t *testing.T) { }, fields: fields{ gen: func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) }, ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, @@ -1241,13 +1225,13 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 5, req: &payload.Search_Request{ - Vector: convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), Config: defaultSearch_Config, }, }, fields: fields{ gen: func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) }, ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, @@ -1279,13 +1263,13 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 10, req: &payload.Search_Request{ - Vector: convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), Config: defaultSearch_Config, }, }, fields: fields{ gen: func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) }, ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, @@ -1317,13 +1301,13 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 20, req: &payload.Search_Request{ - Vector: convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), Config: defaultSearch_Config, }, }, fields: fields{ gen: func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) }, ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, @@ -1355,13 +1339,13 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 5, req: &payload.Search_Request{ - Vector: convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), Config: defaultSearch_Config, }, }, fields: fields{ gen: func(n, dim int) [][]float32 { - v := convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dim)[0]) + v := vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dim)[0]) vectors := make([][]float32, n) for i := range vectors { vectors[i] = v @@ -1405,13 +1389,13 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 10, req: &payload.Search_Request{ - Vector: convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), Config: defaultSearch_Config, }, }, fields: fields{ gen: func(n, dim int) [][]float32 { - v := convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dim)[0]) + v := vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dim)[0]) vectors := make([][]float32, n) for i := range vectors { vectors[i] = v @@ -1455,13 +1439,13 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 20, req: &payload.Search_Request{ - Vector: convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), Config: defaultSearch_Config, }, }, fields: fields{ gen: func(n, dim int) [][]float32 { - v := convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dim)[0]) + v := vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dim)[0]) vectors := make([][]float32, n) for i := range vectors { vectors[i] = v @@ -6923,7 +6907,7 @@ func Test_server_Update(t *testing.T) { gen = vector.GaussianDistributedFloat32VectorGenerator case ngt.Uint8.String(): gen = func(n, dim int) [][]float32 { - return convertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) + return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) } } @@ -7054,7 +7038,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: convertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dimension+1)[0]), + Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dimension+1)[0]), }, Config: defaultUpdateConfig, }, From b5759bd18ac4d7990fe1b02b623f496239badf8d Mon Sep 17 00:00:00 2001 From: kevindiu Date: Tue, 31 May 2022 17:09:04 +0900 Subject: [PATCH 11/29] refactor search test func Signed-off-by: kevindiu --- .../core/ngt/handler/grpc/handler_test.go | 290 ++++++++---------- 1 file changed, 126 insertions(+), 164 deletions(-) diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index fcaacec094..b37c343866 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -704,9 +704,9 @@ func Test_server_Search(t *testing.T) { req *payload.Search_Request } type fields struct { - gen func(int, int) [][]float32 - - opts []Option + objectType request.ObjectType + distribution vector.Distribution + overwriteVec [][]float32 ngtCfg *config.NGT ngtOpts []service.Option @@ -729,44 +729,11 @@ func Test_server_Search(t *testing.T) { defaultDimensionSize = 32 ) + defaultInsertConfig := &payload.Insert_Config{ + SkipStrictExistCheck: true, + } defaultBeforeFunc := func(f fields, a args) (Server, error) { - eg, ctx := errgroup.New(a.ctx) - if f.ngtOpts == nil { - f.ngtOpts = []service.Option{} - } - f.ngtOpts = append(f.ngtOpts, service.WithErrGroup(eg), service.WithEnableInMemoryMode(true)) - ngt, err := service.New(f.ngtCfg, f.ngtOpts...) - if err != nil { - return nil, err - } - if f.opts == nil { - f.opts = []Option{} - } - f.opts = append(f.opts, WithErrGroup(eg), WithNGT(ngt)) - s, err := New(f.opts...) - if err != nil { - return nil, err - } - - reqs := make([]*payload.Insert_Request, a.insertNum) - for i, v := range f.gen(a.insertNum, f.ngtCfg.Dimension) { - reqs[i] = &payload.Insert_Request{ - Vector: &payload.Object_Vector{ - Id: strconv.Itoa(i), - Vector: v, - }, - Config: &payload.Insert_Config{ - SkipStrictExistCheck: true, - }, - } - } - if _, err := s.MultiInsert(ctx, &payload.Insert_MultiRequest{Requests: reqs}); err != nil { - return nil, err - } - if _, err := s.CreateIndex(ctx, &payload.Control_CreateIndexRequest{PoolSize: 100}); err != nil { - return nil, err - } - return s, nil + return buildIndex(a.ctx, f.objectType, f.distribution, a.insertNum, defaultInsertConfig, f.ngtCfg, f.ngtOpts, nil, f.overwriteVec) } defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error { if err != nil { @@ -806,6 +773,26 @@ func Test_server_Search(t *testing.T) { Epsilon: 0.1, Timeout: 1000000000, } + genSameVecs := func(ot request.ObjectType, n int, dim int) [][]float32 { + var vecs [][]float32 + var err error + switch ot { + case request.Float: + vecs, err = vector.GenF32Vec(vector.Gaussian, 1, dim) + case request.Uint8: + vecs, err = vector.GenUint8Vec(vector.Gaussian, 1, dim) + } + if err != nil { + t.Error(err) + } + + res := make([][]float32, n) + for i := 0; i < n; i++ { + res[i] = vecs[0] + } + + return res + } /* Search test cases: @@ -857,10 +844,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -877,8 +863,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -895,10 +882,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: 0, @@ -916,8 +902,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 0, @@ -937,10 +924,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -957,8 +943,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -975,8 +962,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -993,10 +981,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: int(defaultSearch_Config.GetNum()), @@ -1013,8 +1000,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 0, @@ -1032,8 +1020,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 0, @@ -1051,8 +1040,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 0, @@ -1070,8 +1060,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 0, @@ -1089,8 +1080,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 0, @@ -1108,10 +1100,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - }, - ngtCfg: ngtConfig(defaultDimensionSize, "uint8"), + distribution: vector.Gaussian, + objectType: request.Uint8, + ngtCfg: ngtConfig(defaultDimensionSize, "uint8"), }, want: want{ resultSize: 0, @@ -1129,8 +1120,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 0, @@ -1148,10 +1140,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: 0, @@ -1169,8 +1160,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 0, @@ -1188,10 +1180,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: 0, @@ -1209,8 +1200,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 0, @@ -1230,10 +1222,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: 5, @@ -1250,8 +1241,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 5, @@ -1268,10 +1260,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: 10, @@ -1288,8 +1279,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 10, @@ -1306,10 +1298,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: 10, @@ -1326,8 +1317,9 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: vector.GaussianDistributedFloat32VectorGenerator, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Float, + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 10, @@ -1344,15 +1336,10 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - v := vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dim)[0]) - vectors := make([][]float32, n) - for i := range vectors { - vectors[i] = v - } - return vectors - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + overwriteVec: genSameVecs(request.Uint8, 5, defaultDimensionSize), + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: 5, @@ -1369,15 +1356,10 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - v := vector.GaussianDistributedFloat32VectorGenerator(1, dim)[0] - vectors := make([][]float32, n) - for i := range vectors { - vectors[i] = v - } - return vectors - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + overwriteVec: genSameVecs(request.Float, 5, defaultDimensionSize), + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 5, @@ -1394,15 +1376,10 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - v := vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dim)[0]) - vectors := make([][]float32, n) - for i := range vectors { - vectors[i] = v - } - return vectors - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + overwriteVec: genSameVecs(request.Uint8, 10, defaultDimensionSize), + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: 10, @@ -1419,15 +1396,10 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - v := vector.GaussianDistributedFloat32VectorGenerator(1, dim)[0] - vectors := make([][]float32, n) - for i := range vectors { - vectors[i] = v - } - return vectors - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + overwriteVec: genSameVecs(request.Float, 10, defaultDimensionSize), + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 10, @@ -1444,15 +1416,10 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - v := vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dim)[0]) - vectors := make([][]float32, n) - for i := range vectors { - vectors[i] = v - } - return vectors - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + overwriteVec: genSameVecs(request.Uint8, 20, defaultDimensionSize), + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Uint8.String()), }, want: want{ resultSize: 10, @@ -1469,15 +1436,10 @@ func Test_server_Search(t *testing.T) { }, }, fields: fields{ - gen: func(n, dim int) [][]float32 { - v := vector.GaussianDistributedFloat32VectorGenerator(1, dim)[0] - vectors := make([][]float32, n) - for i := range vectors { - vectors[i] = v - } - return vectors - }, - ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), + distribution: vector.Gaussian, + objectType: request.Uint8, + overwriteVec: genSameVecs(request.Float, 20, defaultDimensionSize), + ngtCfg: ngtConfig(defaultDimensionSize, ngt.Float.String()), }, want: want{ resultSize: 10, From 6964a0e1977cfacb3504ece10d168e3e7f00c641 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 1 Jun 2022 10:27:29 +0900 Subject: [PATCH 12/29] refactor beforeFunc test func to use buildIndex Signed-off-by: kevindiu --- .../core/ngt/handler/grpc/handler_test.go | 129 +++--------------- 1 file changed, 20 insertions(+), 109 deletions(-) diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index b37c343866..c55755eeb8 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -22,7 +22,6 @@ import ( "fmt" "math" "reflect" - "strconv" "strings" "testing" @@ -1542,38 +1541,11 @@ func Test_server_SearchByID(t *testing.T) { DeleteBufferPoolSize: 1000, }, } + defaultInsertConfig := &payload.Insert_Config{ + SkipStrictExistCheck: true, + } defaultBeforeFunc := func(a args) (Server, error) { - eg, ctx := errgroup.New(a.ctx) - ngt, err := service.New(defaultNgtConfig, service.WithErrGroup(eg), service.WithEnableInMemoryMode(true)) - if err != nil { - return nil, err - } - - s, err := New(WithErrGroup(eg), WithNGT(ngt)) - if err != nil { - return nil, err - } - - reqs := make([]*payload.Insert_Request, insertNum) - for i, v := range vector.GaussianDistributedFloat32VectorGenerator(insertNum, defaultNgtConfig.Dimension) { - reqs[i] = &payload.Insert_Request{ - Vector: &payload.Object_Vector{ - Id: strconv.Itoa(i), - Vector: v, - }, - Config: &payload.Insert_Config{ - SkipStrictExistCheck: true, - }, - } - } - reqs[0].Vector.Id = a.indexId - if _, err := s.MultiInsert(ctx, &payload.Insert_MultiRequest{Requests: reqs}); err != nil { - return nil, err - } - if _, err := s.CreateIndex(ctx, &payload.Control_CreateIndexRequest{PoolSize: 100}); err != nil { - return nil, err - } - return s, nil + return buildIndex(a.ctx, request.Float, vector.Gaussian, insertNum, defaultInsertConfig, defaultNgtConfig, nil, []string{a.indexId}, nil) } defaultSearch_Config := &payload.Search_Config{ Num: 10, @@ -6848,6 +6820,9 @@ func Test_server_Update(t *testing.T) { defaultUpdateConfig := &payload.Update_Config{ SkipStrictExistCheck: true, } + defaultInsertConfig := &payload.Insert_Config{ + SkipStrictExistCheck: true, + } beforeFunc := func(objectType string) func(args) (Server, error) { cfg := &config.NGT{ Dimension: dimension, @@ -6863,52 +6838,15 @@ func Test_server_Update(t *testing.T) { DeleteBufferPoolSize: 1000, }, } - var gen func(int, int) [][]float32 - switch objectType { - case ngt.Float.String(): - gen = vector.GaussianDistributedFloat32VectorGenerator - case ngt.Uint8.String(): - gen = func(n, dim int) [][]float32 { - return vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(n, dim)) - } - } return func(a args) (Server, error) { - eg, ctx := errgroup.New(a.ctx) - ngt, err := service.New(cfg, service.WithErrGroup(eg), service.WithEnableInMemoryMode(true)) - if err != nil { - return nil, err - } - - s, err := New(WithErrGroup(eg), WithNGT(ngt)) - if err != nil { - return nil, err - } - - // TODO: use request.GenMultiInsertReq() - reqs := make([]*payload.Insert_Request, insertNum) - for i, v := range gen(insertNum, cfg.Dimension) { - reqs[i] = &payload.Insert_Request{ - Vector: &payload.Object_Vector{ - Id: strconv.Itoa(i), - Vector: v, - }, - Config: &payload.Insert_Config{ - SkipStrictExistCheck: true, - }, - } - } - reqs[0].Vector.Id = a.indexId + var overwriteVec [][]float32 if a.indexVector != nil { - reqs[0].Vector.Vector = a.indexVector - } - if _, err := s.MultiInsert(ctx, &payload.Insert_MultiRequest{Requests: reqs}); err != nil { - return nil, err - } - if _, err := s.CreateIndex(ctx, &payload.Control_CreateIndexRequest{PoolSize: 100}); err != nil { - return nil, err + overwriteVec = [][]float32{ + a.indexVector, + } } - return s, nil + return buildIndex(a.ctx, request.Float, vector.Gaussian, insertNum, defaultInsertConfig, cfg, nil, []string{a.indexId}, overwriteVec) } } @@ -7570,7 +7508,7 @@ func Test_server_Update(t *testing.T) { indexVector: vector, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: "1", + Id: "uuid-2", // the first uuid is overwritten, so use the second one Vector: vector, }, Config: defaultUpdateConfig, @@ -7578,7 +7516,7 @@ func Test_server_Update(t *testing.T) { } }(), want: want{ - wantUuid: "1", + wantUuid: "uuid-2", }, }, { @@ -7633,7 +7571,7 @@ func Test_server_Update(t *testing.T) { indexVector: vector, req: &payload.Update_Request{ Vector: &payload.Object_Vector{ - Id: "1", + Id: "uuid-2", // the first uuid is overwritten, so use the second one Vector: vector, }, Config: &payload.Update_Config{ @@ -7643,7 +7581,7 @@ func Test_server_Update(t *testing.T) { } }(), want: want{ - wantUuid: "1", + wantUuid: "uuid-2", }, }, } @@ -8281,38 +8219,11 @@ func Test_server_Remove(t *testing.T) { DeleteBufferPoolSize: 1000, }, } + defaultInsertConfig := &payload.Insert_Config{ + SkipStrictExistCheck: true, + } defaultBeforeFunc := func(a args) (Server, error) { - eg, ctx := errgroup.New(a.ctx) - ngt, err := service.New(defaultNgtConfig, service.WithErrGroup(eg), service.WithEnableInMemoryMode(true)) - if err != nil { - return nil, err - } - - s, err := New(WithErrGroup(eg), WithNGT(ngt)) - if err != nil { - return nil, err - } - - reqs := make([]*payload.Insert_Request, insertNum) - for i, v := range vector.GaussianDistributedFloat32VectorGenerator(insertNum, defaultNgtConfig.Dimension) { - reqs[i] = &payload.Insert_Request{ - Vector: &payload.Object_Vector{ - Id: strconv.Itoa(i), - Vector: v, - }, - Config: &payload.Insert_Config{ - SkipStrictExistCheck: true, - }, - } - } - reqs[0].Vector.Id = a.indexId - if _, err := s.MultiInsert(ctx, &payload.Insert_MultiRequest{Requests: reqs}); err != nil { - return nil, err - } - if _, err := s.CreateIndex(ctx, &payload.Control_CreateIndexRequest{PoolSize: 100}); err != nil { - return nil, err - } - return s, nil + return buildIndex(a.ctx, request.Float, vector.Gaussian, insertNum, defaultInsertConfig, defaultNgtConfig, nil, []string{a.indexId}, nil) } /* From 9cd164c816e0b24614f6a9890882b4b540cda443 Mon Sep 17 00:00:00 2001 From: Vdaas CI Date: Wed, 1 Jun 2022 01:41:36 +0000 Subject: [PATCH 13/29] :robot: Update license headers / Format go codes and yaml files Signed-off-by: Vdaas CI --- internal/net/control/control_darwin.go | 1 - internal/net/control/control_other.go | 1 - internal/net/control/control_unix.go | 1 - internal/net/control/control_windows.go | 1 - internal/runner/runner_race_test.go | 1 - internal/singleflight/singleflight_test.go | 1 - internal/test/data/request/insert.go | 15 +++++++++++++++ internal/test/data/request/object.go | 15 +++++++++++++++ internal/timeutil/location/set_test.go | 1 - pkg/agent/core/ngt/handler/grpc/handler_test.go | 4 ++-- tests/e2e/crud/crud_test.go | 1 - tests/e2e/hdf5/hdf5.go | 1 - tests/e2e/kubernetes/client/client.go | 1 - tests/e2e/kubernetes/portforward/portforward.go | 1 - tests/e2e/multiapis/multiapis_test.go | 1 - tests/e2e/operation/doc.go | 1 - tests/e2e/operation/multi.go | 1 - tests/e2e/operation/operation.go | 1 - tests/e2e/operation/stream.go | 1 - tests/e2e/performance/max_vector_dim_test.go | 1 - tests/e2e/sidecar/sidecar_test.go | 1 - 21 files changed, 32 insertions(+), 20 deletions(-) diff --git a/internal/net/control/control_darwin.go b/internal/net/control/control_darwin.go index 67ce3fbfcf..e29f4550e7 100644 --- a/internal/net/control/control_darwin.go +++ b/internal/net/control/control_darwin.go @@ -1,4 +1,3 @@ -//go:build darwin && !linux && !windows && !wasm && !js // +build darwin,!linux,!windows,!wasm,!js // diff --git a/internal/net/control/control_other.go b/internal/net/control/control_other.go index 05306d5c19..117b492c37 100644 --- a/internal/net/control/control_other.go +++ b/internal/net/control/control_other.go @@ -1,4 +1,3 @@ -//go:build wasm && js && !windows && !linux && !darwin // +build wasm,js,!windows,!linux,!darwin // diff --git a/internal/net/control/control_unix.go b/internal/net/control/control_unix.go index 39ed43ba1a..ed22e62b2c 100644 --- a/internal/net/control/control_unix.go +++ b/internal/net/control/control_unix.go @@ -1,4 +1,3 @@ -//go:build linux && !windows && !wasm && !js && !darwin // +build linux,!windows,!wasm,!js,!darwin // diff --git a/internal/net/control/control_windows.go b/internal/net/control/control_windows.go index cd20c216f3..8d66834622 100644 --- a/internal/net/control/control_windows.go +++ b/internal/net/control/control_windows.go @@ -1,4 +1,3 @@ -//go:build windows // +build windows // diff --git a/internal/runner/runner_race_test.go b/internal/runner/runner_race_test.go index 89a0d379df..a6deaaf31a 100644 --- a/internal/runner/runner_race_test.go +++ b/internal/runner/runner_race_test.go @@ -1,4 +1,3 @@ -//go:build !race // +build !race // diff --git a/internal/singleflight/singleflight_test.go b/internal/singleflight/singleflight_test.go index b5b8c9e571..20954be6d7 100644 --- a/internal/singleflight/singleflight_test.go +++ b/internal/singleflight/singleflight_test.go @@ -1,4 +1,3 @@ -//go:build !race // +build !race // diff --git a/internal/test/data/request/insert.go b/internal/test/data/request/insert.go index 93dd068b64..884fbac31e 100644 --- a/internal/test/data/request/insert.go +++ b/internal/test/data/request/insert.go @@ -1,3 +1,18 @@ +// +// Copyright (C) 2019-2022 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package request import ( diff --git a/internal/test/data/request/object.go b/internal/test/data/request/object.go index a471cb9b74..4e4fdeeee2 100644 --- a/internal/test/data/request/object.go +++ b/internal/test/data/request/object.go @@ -1,3 +1,18 @@ +// +// Copyright (C) 2019-2022 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package request import ( diff --git a/internal/timeutil/location/set_test.go b/internal/timeutil/location/set_test.go index 82945e1ac8..18027abb50 100644 --- a/internal/timeutil/location/set_test.go +++ b/internal/timeutil/location/set_test.go @@ -1,4 +1,3 @@ -//go:build !race // +build !race // diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index c55755eeb8..445aa23051 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -45,8 +45,8 @@ import ( ) func buildIndex(ctx context.Context, t request.ObjectType, dist vector.Distribution, num int, insertCfg *payload.Insert_Config, - ngtCfg *config.NGT, ngtOpts []service.Option, overwriteIDs []string, overwriteVectors [][]float32) (Server, error) { - + ngtCfg *config.NGT, ngtOpts []service.Option, overwriteIDs []string, overwriteVectors [][]float32, +) (Server, error) { eg, ctx := errgroup.New(ctx) ngt, err := service.New(ngtCfg, append(ngtOpts, service.WithErrGroup(eg), service.WithEnableInMemoryMode(true))...) if err != nil { diff --git a/tests/e2e/crud/crud_test.go b/tests/e2e/crud/crud_test.go index 3ec1deed98..3a48d04377 100644 --- a/tests/e2e/crud/crud_test.go +++ b/tests/e2e/crud/crud_test.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // diff --git a/tests/e2e/hdf5/hdf5.go b/tests/e2e/hdf5/hdf5.go index 5c9083cf0e..c8ebd2e88c 100644 --- a/tests/e2e/hdf5/hdf5.go +++ b/tests/e2e/hdf5/hdf5.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // diff --git a/tests/e2e/kubernetes/client/client.go b/tests/e2e/kubernetes/client/client.go index 5dd261b415..2f696f79c3 100644 --- a/tests/e2e/kubernetes/client/client.go +++ b/tests/e2e/kubernetes/client/client.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // diff --git a/tests/e2e/kubernetes/portforward/portforward.go b/tests/e2e/kubernetes/portforward/portforward.go index 622d9ff0a1..270cc8d293 100644 --- a/tests/e2e/kubernetes/portforward/portforward.go +++ b/tests/e2e/kubernetes/portforward/portforward.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // diff --git a/tests/e2e/multiapis/multiapis_test.go b/tests/e2e/multiapis/multiapis_test.go index 24bbbfa1f1..ec963f34f3 100644 --- a/tests/e2e/multiapis/multiapis_test.go +++ b/tests/e2e/multiapis/multiapis_test.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // diff --git a/tests/e2e/operation/doc.go b/tests/e2e/operation/doc.go index edc3e17c2d..c09ee5a55b 100644 --- a/tests/e2e/operation/doc.go +++ b/tests/e2e/operation/doc.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // diff --git a/tests/e2e/operation/multi.go b/tests/e2e/operation/multi.go index 1f263760ba..cf15592a85 100644 --- a/tests/e2e/operation/multi.go +++ b/tests/e2e/operation/multi.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // diff --git a/tests/e2e/operation/operation.go b/tests/e2e/operation/operation.go index c5c5227642..17c07be050 100644 --- a/tests/e2e/operation/operation.go +++ b/tests/e2e/operation/operation.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // diff --git a/tests/e2e/operation/stream.go b/tests/e2e/operation/stream.go index 19069c4af5..b8ad83e78a 100644 --- a/tests/e2e/operation/stream.go +++ b/tests/e2e/operation/stream.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // diff --git a/tests/e2e/performance/max_vector_dim_test.go b/tests/e2e/performance/max_vector_dim_test.go index b42a993f4b..0836190b5f 100644 --- a/tests/e2e/performance/max_vector_dim_test.go +++ b/tests/e2e/performance/max_vector_dim_test.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // diff --git a/tests/e2e/sidecar/sidecar_test.go b/tests/e2e/sidecar/sidecar_test.go index 230ea13f83..3c57cb7ce4 100644 --- a/tests/e2e/sidecar/sidecar_test.go +++ b/tests/e2e/sidecar/sidecar_test.go @@ -1,4 +1,3 @@ -//go:build e2e // +build e2e // From 703741533f889cce3d21a14e71181afaec1fe0d8 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 1 Jun 2022 11:37:28 +0900 Subject: [PATCH 14/29] Revert ":robot: Update license headers / Format go codes and yaml files" This reverts commit a8b44c37f418aac174690a7b08a9d00fbb1f3ad9. --- internal/net/control/control_darwin.go | 1 + internal/net/control/control_other.go | 1 + internal/net/control/control_unix.go | 1 + internal/net/control/control_windows.go | 1 + internal/runner/runner_race_test.go | 1 + internal/singleflight/singleflight_test.go | 1 + internal/test/data/request/insert.go | 15 --------------- internal/test/data/request/object.go | 15 --------------- internal/timeutil/location/set_test.go | 1 + pkg/agent/core/ngt/handler/grpc/handler_test.go | 4 ++-- tests/e2e/crud/crud_test.go | 1 + tests/e2e/hdf5/hdf5.go | 1 + tests/e2e/kubernetes/client/client.go | 1 + tests/e2e/kubernetes/portforward/portforward.go | 1 + tests/e2e/multiapis/multiapis_test.go | 1 + tests/e2e/operation/doc.go | 1 + tests/e2e/operation/multi.go | 1 + tests/e2e/operation/operation.go | 1 + tests/e2e/operation/stream.go | 1 + tests/e2e/performance/max_vector_dim_test.go | 1 + tests/e2e/sidecar/sidecar_test.go | 1 + 21 files changed, 20 insertions(+), 32 deletions(-) diff --git a/internal/net/control/control_darwin.go b/internal/net/control/control_darwin.go index e29f4550e7..67ce3fbfcf 100644 --- a/internal/net/control/control_darwin.go +++ b/internal/net/control/control_darwin.go @@ -1,3 +1,4 @@ +//go:build darwin && !linux && !windows && !wasm && !js // +build darwin,!linux,!windows,!wasm,!js // diff --git a/internal/net/control/control_other.go b/internal/net/control/control_other.go index 117b492c37..05306d5c19 100644 --- a/internal/net/control/control_other.go +++ b/internal/net/control/control_other.go @@ -1,3 +1,4 @@ +//go:build wasm && js && !windows && !linux && !darwin // +build wasm,js,!windows,!linux,!darwin // diff --git a/internal/net/control/control_unix.go b/internal/net/control/control_unix.go index ed22e62b2c..39ed43ba1a 100644 --- a/internal/net/control/control_unix.go +++ b/internal/net/control/control_unix.go @@ -1,3 +1,4 @@ +//go:build linux && !windows && !wasm && !js && !darwin // +build linux,!windows,!wasm,!js,!darwin // diff --git a/internal/net/control/control_windows.go b/internal/net/control/control_windows.go index 8d66834622..cd20c216f3 100644 --- a/internal/net/control/control_windows.go +++ b/internal/net/control/control_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows // diff --git a/internal/runner/runner_race_test.go b/internal/runner/runner_race_test.go index a6deaaf31a..89a0d379df 100644 --- a/internal/runner/runner_race_test.go +++ b/internal/runner/runner_race_test.go @@ -1,3 +1,4 @@ +//go:build !race // +build !race // diff --git a/internal/singleflight/singleflight_test.go b/internal/singleflight/singleflight_test.go index 20954be6d7..b5b8c9e571 100644 --- a/internal/singleflight/singleflight_test.go +++ b/internal/singleflight/singleflight_test.go @@ -1,3 +1,4 @@ +//go:build !race // +build !race // diff --git a/internal/test/data/request/insert.go b/internal/test/data/request/insert.go index 884fbac31e..93dd068b64 100644 --- a/internal/test/data/request/insert.go +++ b/internal/test/data/request/insert.go @@ -1,18 +1,3 @@ -// -// Copyright (C) 2019-2022 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// package request import ( diff --git a/internal/test/data/request/object.go b/internal/test/data/request/object.go index 4e4fdeeee2..a471cb9b74 100644 --- a/internal/test/data/request/object.go +++ b/internal/test/data/request/object.go @@ -1,18 +1,3 @@ -// -// Copyright (C) 2019-2022 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// package request import ( diff --git a/internal/timeutil/location/set_test.go b/internal/timeutil/location/set_test.go index 18027abb50..82945e1ac8 100644 --- a/internal/timeutil/location/set_test.go +++ b/internal/timeutil/location/set_test.go @@ -1,3 +1,4 @@ +//go:build !race // +build !race // diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 445aa23051..c55755eeb8 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -45,8 +45,8 @@ import ( ) func buildIndex(ctx context.Context, t request.ObjectType, dist vector.Distribution, num int, insertCfg *payload.Insert_Config, - ngtCfg *config.NGT, ngtOpts []service.Option, overwriteIDs []string, overwriteVectors [][]float32, -) (Server, error) { + ngtCfg *config.NGT, ngtOpts []service.Option, overwriteIDs []string, overwriteVectors [][]float32) (Server, error) { + eg, ctx := errgroup.New(ctx) ngt, err := service.New(ngtCfg, append(ngtOpts, service.WithErrGroup(eg), service.WithEnableInMemoryMode(true))...) if err != nil { diff --git a/tests/e2e/crud/crud_test.go b/tests/e2e/crud/crud_test.go index 3a48d04377..3ec1deed98 100644 --- a/tests/e2e/crud/crud_test.go +++ b/tests/e2e/crud/crud_test.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // diff --git a/tests/e2e/hdf5/hdf5.go b/tests/e2e/hdf5/hdf5.go index c8ebd2e88c..5c9083cf0e 100644 --- a/tests/e2e/hdf5/hdf5.go +++ b/tests/e2e/hdf5/hdf5.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // diff --git a/tests/e2e/kubernetes/client/client.go b/tests/e2e/kubernetes/client/client.go index 2f696f79c3..5dd261b415 100644 --- a/tests/e2e/kubernetes/client/client.go +++ b/tests/e2e/kubernetes/client/client.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // diff --git a/tests/e2e/kubernetes/portforward/portforward.go b/tests/e2e/kubernetes/portforward/portforward.go index 270cc8d293..622d9ff0a1 100644 --- a/tests/e2e/kubernetes/portforward/portforward.go +++ b/tests/e2e/kubernetes/portforward/portforward.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // diff --git a/tests/e2e/multiapis/multiapis_test.go b/tests/e2e/multiapis/multiapis_test.go index ec963f34f3..24bbbfa1f1 100644 --- a/tests/e2e/multiapis/multiapis_test.go +++ b/tests/e2e/multiapis/multiapis_test.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // diff --git a/tests/e2e/operation/doc.go b/tests/e2e/operation/doc.go index c09ee5a55b..edc3e17c2d 100644 --- a/tests/e2e/operation/doc.go +++ b/tests/e2e/operation/doc.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // diff --git a/tests/e2e/operation/multi.go b/tests/e2e/operation/multi.go index cf15592a85..1f263760ba 100644 --- a/tests/e2e/operation/multi.go +++ b/tests/e2e/operation/multi.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // diff --git a/tests/e2e/operation/operation.go b/tests/e2e/operation/operation.go index 17c07be050..c5c5227642 100644 --- a/tests/e2e/operation/operation.go +++ b/tests/e2e/operation/operation.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // diff --git a/tests/e2e/operation/stream.go b/tests/e2e/operation/stream.go index b8ad83e78a..19069c4af5 100644 --- a/tests/e2e/operation/stream.go +++ b/tests/e2e/operation/stream.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // diff --git a/tests/e2e/performance/max_vector_dim_test.go b/tests/e2e/performance/max_vector_dim_test.go index 0836190b5f..b42a993f4b 100644 --- a/tests/e2e/performance/max_vector_dim_test.go +++ b/tests/e2e/performance/max_vector_dim_test.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // diff --git a/tests/e2e/sidecar/sidecar_test.go b/tests/e2e/sidecar/sidecar_test.go index 3c57cb7ce4..230ea13f83 100644 --- a/tests/e2e/sidecar/sidecar_test.go +++ b/tests/e2e/sidecar/sidecar_test.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e // From 15ea532137ecae182da0c3d27b635c3c1cc31fde Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 1 Jun 2022 11:40:24 +0900 Subject: [PATCH 15/29] format code Signed-off-by: kevindiu --- internal/test/data/request/doc.go | 18 ++++++++++++++++++ internal/test/data/request/insert.go | 15 +++++++++++++++ internal/test/data/request/object.go | 15 +++++++++++++++ .../core/ngt/handler/grpc/handler_test.go | 4 ++-- 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 internal/test/data/request/doc.go diff --git a/internal/test/data/request/doc.go b/internal/test/data/request/doc.go new file mode 100644 index 0000000000..ab74b7f8d9 --- /dev/null +++ b/internal/test/data/request/doc.go @@ -0,0 +1,18 @@ +// +// Copyright (C) 2019-2022 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package request provides functions to generate proto request for testing +package request diff --git a/internal/test/data/request/insert.go b/internal/test/data/request/insert.go index 93dd068b64..884fbac31e 100644 --- a/internal/test/data/request/insert.go +++ b/internal/test/data/request/insert.go @@ -1,3 +1,18 @@ +// +// Copyright (C) 2019-2022 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package request import ( diff --git a/internal/test/data/request/object.go b/internal/test/data/request/object.go index a471cb9b74..4e4fdeeee2 100644 --- a/internal/test/data/request/object.go +++ b/internal/test/data/request/object.go @@ -1,3 +1,18 @@ +// +// Copyright (C) 2019-2022 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package request import ( diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index c55755eeb8..445aa23051 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -45,8 +45,8 @@ import ( ) func buildIndex(ctx context.Context, t request.ObjectType, dist vector.Distribution, num int, insertCfg *payload.Insert_Config, - ngtCfg *config.NGT, ngtOpts []service.Option, overwriteIDs []string, overwriteVectors [][]float32) (Server, error) { - + ngtCfg *config.NGT, ngtOpts []service.Option, overwriteIDs []string, overwriteVectors [][]float32, +) (Server, error) { eg, ctx := errgroup.New(ctx) ngt, err := service.New(ngtCfg, append(ngtOpts, service.WithErrGroup(eg), service.WithEnableInMemoryMode(true))...) if err != nil { From 1343782986218c099c30723fb00079d1695bca78 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 1 Jun 2022 14:11:44 +0900 Subject: [PATCH 16/29] fix comment Signed-off-by: kevindiu --- internal/test/data/request/object.go | 4 ++-- internal/test/data/vector/gen.go | 4 ++-- .../core/ngt/handler/grpc/handler_test.go | 20 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/test/data/request/object.go b/internal/test/data/request/object.go index 4e4fdeeee2..d1006fc185 100644 --- a/internal/test/data/request/object.go +++ b/internal/test/data/request/object.go @@ -21,7 +21,7 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/payload" ) -func GenObjectLocations(num int, name string, ip string) *payload.Object_Locations { +func GenObjectLocations(num int, name string, ipAddr string) *payload.Object_Locations { result := &payload.Object_Locations{ Locations: make([]*payload.Object_Location, num), } @@ -30,7 +30,7 @@ func GenObjectLocations(num int, name string, ip string) *payload.Object_Locatio result.Locations[i] = &payload.Object_Location{ Name: name, Uuid: "uuid-" + strconv.Itoa(i+1), - Ips: []string{ip}, + Ips: []string{ipAddr}, } } return result diff --git a/internal/test/data/vector/gen.go b/internal/test/data/vector/gen.go index 5bdd78445b..469c46e787 100644 --- a/internal/test/data/vector/gen.go +++ b/internal/test/data/vector/gen.go @@ -156,7 +156,7 @@ func GenSameValueVec(size int, val float32) []float32 { return v } -func ConvertVectorUint8ToFloat32(vector []uint8) (ret []float32) { +func convertVectorUint8ToFloat32(vector []uint8) (ret []float32) { ret = make([]float32, len(vector)) for i, e := range vector { ret[i] = float32(e) @@ -167,7 +167,7 @@ func ConvertVectorUint8ToFloat32(vector []uint8) (ret []float32) { func ConvertVectorsUint8ToFloat32(vectors [][]uint8) (ret [][]float32) { ret = make([][]float32, 0, len(vectors)) for _, v := range vectors { - ret = append(ret, ConvertVectorUint8ToFloat32(v)) + ret = append(ret, convertVectorUint8ToFloat32(v)) } return } diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 445aa23051..0e5625baaa 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -838,7 +838,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize))[0], Config: defaultSearch_Config, }, }, @@ -876,7 +876,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize+1)[0]), + Vector: vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize+1))[0], Config: defaultSearch_Config, }, }, @@ -1134,7 +1134,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 1000, req: &payload.Search_Request{ - Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, math.MaxInt32>>7)[0]), + Vector: vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, math.MaxInt32>>7))[0], Config: defaultSearch_Config, }, }, @@ -1216,7 +1216,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 5, req: &payload.Search_Request{ - Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize))[0], Config: defaultSearch_Config, }, }, @@ -1254,7 +1254,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 10, req: &payload.Search_Request{ - Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize))[0], Config: defaultSearch_Config, }, }, @@ -1292,7 +1292,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 20, req: &payload.Search_Request{ - Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize))[0], Config: defaultSearch_Config, }, }, @@ -1330,7 +1330,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 5, req: &payload.Search_Request{ - Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize))[0], Config: defaultSearch_Config, }, }, @@ -1370,7 +1370,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 10, req: &payload.Search_Request{ - Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize))[0], Config: defaultSearch_Config, }, }, @@ -1410,7 +1410,7 @@ func Test_server_Search(t *testing.T) { ctx: ctx, insertNum: 20, req: &payload.Search_Request{ - Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize)[0]), + Vector: vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, defaultDimensionSize))[0], Config: defaultSearch_Config, }, }, @@ -6938,7 +6938,7 @@ func Test_server_Update(t *testing.T) { req: &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: "test", - Vector: vector.ConvertVectorUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dimension+1)[0]), + Vector: vector.ConvertVectorsUint8ToFloat32(vector.GaussianDistributedUint8VectorGenerator(1, dimension+1))[0], }, Config: defaultUpdateConfig, }, From b6a9057261a0cf9e37dd8f094b4ce57f6357ebf7 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 1 Jun 2022 15:50:34 +0900 Subject: [PATCH 17/29] implement test for helper functions Signed-off-by: kevindiu --- internal/conv/conv_test.go | 156 ++++++++ internal/test/data/request/insert_test.go | 347 +++++++++++++++++ internal/test/data/request/object_test.go | 142 +++++++ internal/test/data/vector/gen.go | 16 +- internal/test/data/vector/gen_test.go | 436 ++++++++++++++++++++++ 5 files changed, 1089 insertions(+), 8 deletions(-) create mode 100644 internal/test/data/request/insert_test.go create mode 100644 internal/test/data/request/object_test.go diff --git a/internal/conv/conv_test.go b/internal/conv/conv_test.go index 991b320768..39a744ed7e 100644 --- a/internal/conv/conv_test.go +++ b/internal/conv/conv_test.go @@ -236,3 +236,159 @@ func TestF32stos(t *testing.T) { }) } } + +func TestUtf8ToSjis(t *testing.T) { + type args struct { + s string + } + type want struct { + want string + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, string, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got string, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + s: "", + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + s: "", + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got, err := Utf8ToSjis(test.args.s) + if err := checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func TestUtf8ToEucjp(t *testing.T) { + type args struct { + s string + } + type want struct { + want string + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, string, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got string, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + s: "", + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + s: "", + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got, err := Utf8ToEucjp(test.args.s) + if err := checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} diff --git a/internal/test/data/request/insert_test.go b/internal/test/data/request/insert_test.go new file mode 100644 index 0000000000..1e82b285e0 --- /dev/null +++ b/internal/test/data/request/insert_test.go @@ -0,0 +1,347 @@ +// +// Copyright (C) 2019-2022 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package request + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/test/comparator" + "github.com/vdaas/vald/internal/test/data/vector" + "github.com/vdaas/vald/internal/test/goleak" +) + +var ( + defaultMultiInsertReqComparators = []cmp.Option{ + comparator.IgnoreUnexported(payload.Insert_Request{}), + comparator.IgnoreUnexported(payload.Insert_MultiRequest{}), + comparator.IgnoreUnexported(payload.Object_Vector{}), + } +) + +func TestGenMultiInsertReq(t *testing.T) { + type args struct { + t ObjectType + dist vector.Distribution + num int + dim int + cfg *payload.Insert_Config + } + type want struct { + want *payload.Insert_MultiRequest + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, *payload.Insert_MultiRequest, error) error + beforeFunc func(args) + afterFunc func(args) + } + dim := 10 + comparators := append(defaultMultiInsertReqComparators, comparator.IgnoreFields(payload.Object_Vector{}, "Vector")) + + defaultCheckFunc := func(w want, got *payload.Insert_MultiRequest, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if diff := comparator.Diff(got, w.want, comparators...); diff != "" { + return errors.Errorf("diff: %v", diff) + } + if len(got.Requests[0].Vector.Vector) != dim { + return errors.New("vector length not match") + } + return nil + } + tests := []test{ + { + name: "success to generate 1 float request", + args: args{ + t: Float, + dist: vector.Gaussian, + num: 1, + dim: dim, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{ + { + Vector: &payload.Object_Vector{ + Id: "uuid-1", + }, + }, + }, + }, + }, + }, + { + name: "success to generate 1 uint8 request", + args: args{ + t: Uint8, + dist: vector.Gaussian, + num: 1, + dim: dim, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{ + { + Vector: &payload.Object_Vector{ + Id: "uuid-1", + }, + }, + }, + }, + }, + }, + { + name: "success to generate 5 float request", + args: args{ + t: Float, + dist: vector.Gaussian, + num: 5, + dim: dim, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{ + { + Vector: &payload.Object_Vector{ + Id: "uuid-1", + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-2", + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-3", + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-4", + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-5", + }, + }, + }, + }, + }, + }, + { + name: "success to generate 5 uint8 request", + args: args{ + t: Uint8, + dist: vector.Gaussian, + num: 5, + dim: dim, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{ + { + Vector: &payload.Object_Vector{ + Id: "uuid-1", + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-2", + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-3", + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-4", + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-5", + }, + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got, err := GenMultiInsertReq(test.args.t, test.args.dist, test.args.num, test.args.dim, test.args.cfg) + if err := checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func TestGenSameVecMultiInsertReq(t *testing.T) { + type args struct { + num int + vec []float32 + cfg *payload.Insert_Config + } + type want struct { + want *payload.Insert_MultiRequest + } + type test struct { + name string + args args + want want + checkFunc func(want, *payload.Insert_MultiRequest) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got *payload.Insert_MultiRequest) error { + if diff := comparator.Diff(got, w.want, defaultMultiInsertReqComparators...); diff != "" { + return errors.Errorf("diff: %v", diff) + } + return nil + } + tests := []test{ + func() test { + vecs, err := vector.GenF32Vec(vector.Gaussian, 1, 10) + if err != nil { + t.Error(err) + } + + return test{ + name: "success to generate 1 same vector request", + args: args{ + num: 1, + vec: vecs[0], + cfg: nil, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{ + { + Vector: &payload.Object_Vector{ + Id: "uuid-1", + Vector: vecs[0], + }, + }, + }, + }, + }, + } + }(), + func() test { + vecs, err := vector.GenF32Vec(vector.Gaussian, 1, 10) + if err != nil { + t.Error(err) + } + + return test{ + name: "success to generate 5 same vector request", + args: args{ + num: 5, + vec: vecs[0], + cfg: nil, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{ + { + Vector: &payload.Object_Vector{ + Id: "uuid-1", + Vector: vecs[0], + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-2", + Vector: vecs[0], + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-3", + Vector: vecs[0], + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-4", + Vector: vecs[0], + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-5", + Vector: vecs[0], + }, + }, + }, + }, + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got := GenSameVecMultiInsertReq(test.args.num, test.args.vec, test.args.cfg) + if err := checkFunc(test.want, got); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} diff --git a/internal/test/data/request/object_test.go b/internal/test/data/request/object_test.go new file mode 100644 index 0000000000..64656863b8 --- /dev/null +++ b/internal/test/data/request/object_test.go @@ -0,0 +1,142 @@ +// +// Copyright (C) 2019-2022 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package request + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/test/comparator" + "github.com/vdaas/vald/internal/test/goleak" +) + +var ( + defaultOvjectLocationComparators = []cmp.Option{ + comparator.IgnoreUnexported(payload.Object_Locations{}), + comparator.IgnoreUnexported(payload.Object_Location{}), + } +) + +func TestGenObjectLocations(t *testing.T) { + type args struct { + num int + name string + ipAddr string + } + type want struct { + want *payload.Object_Locations + } + type test struct { + name string + args args + want want + checkFunc func(want, *payload.Object_Locations) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got *payload.Object_Locations) error { + if diff := comparator.Diff(got, w.want, defaultOvjectLocationComparators...); diff != "" { + return errors.Errorf("diff: %v", diff) + } + return nil + } + tests := []test{ + { + name: "success to generate 1 object location", + args: args{ + num: 1, + name: "vald-agent-01", + ipAddr: "127.0.0.1", + }, + want: want{ + want: &payload.Object_Locations{ + Locations: []*payload.Object_Location{ + { + Name: "vald-agent-01", + Uuid: "uuid-1", + Ips: []string{"127.0.0.1"}, + }, + }, + }, + }, + }, + { + name: "success to generate 5 object location", + args: args{ + num: 5, + name: "vald-agent-01", + ipAddr: "127.0.0.1", + }, + want: want{ + want: &payload.Object_Locations{ + Locations: []*payload.Object_Location{ + { + Name: "vald-agent-01", + Uuid: "uuid-1", + Ips: []string{"127.0.0.1"}, + }, + { + Name: "vald-agent-01", + Uuid: "uuid-2", + Ips: []string{"127.0.0.1"}, + }, + { + Name: "vald-agent-01", + Uuid: "uuid-3", + Ips: []string{"127.0.0.1"}, + }, + { + Name: "vald-agent-01", + Uuid: "uuid-4", + Ips: []string{"127.0.0.1"}, + }, + { + Name: "vald-agent-01", + Uuid: "uuid-5", + Ips: []string{"127.0.0.1"}, + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got := GenObjectLocations(test.args.num, test.args.name, test.args.ipAddr) + if err := checkFunc(test.want, got); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} diff --git a/internal/test/data/vector/gen.go b/internal/test/data/vector/gen.go index 469c46e787..b1c162e472 100644 --- a/internal/test/data/vector/gen.go +++ b/internal/test/data/vector/gen.go @@ -156,14 +156,6 @@ func GenSameValueVec(size int, val float32) []float32 { return v } -func convertVectorUint8ToFloat32(vector []uint8) (ret []float32) { - ret = make([]float32, len(vector)) - for i, e := range vector { - ret[i] = float32(e) - } - return -} - func ConvertVectorsUint8ToFloat32(vectors [][]uint8) (ret [][]float32) { ret = make([][]float32, 0, len(vectors)) for _, v := range vectors { @@ -171,3 +163,11 @@ func ConvertVectorsUint8ToFloat32(vectors [][]uint8) (ret [][]float32) { } return } + +func convertVectorUint8ToFloat32(vector []uint8) (ret []float32) { + ret = make([]float32, len(vector)) + for i, e := range vector { + ret[i] = float32(e) + } + return +} diff --git a/internal/test/data/vector/gen_test.go b/internal/test/data/vector/gen_test.go index 79f96e5bcd..1922b4681c 100644 --- a/internal/test/data/vector/gen_test.go +++ b/internal/test/data/vector/gen_test.go @@ -16,6 +16,7 @@ package vector import ( + "math" "reflect" "testing" @@ -752,3 +753,438 @@ func Test_gaussianDistributedUint8VectorGenerator(t *testing.T) { }) } } + +func TestGenF32Vec(t *testing.T) { + type args struct { + dist Distribution + num int + dim int + } + type want struct { + wantLen int + wantDim int + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, [][]float32, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got [][]float32, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if len(got) != w.wantLen { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twantLen: \"%#v\"", got, w.wantLen) + } + for _, vec := range got { + if len(vec) != w.wantDim { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twantDim: \"%#v\"", got, w.wantDim) + } + } + return nil + } + tests := []test{ + { + name: "return 1 generated float32 vector", + args: args{ + dist: Gaussian, + num: 1, + dim: 5, + }, + want: want{ + wantLen: 1, + wantDim: 5, + }, + }, + { + name: "return 5 generated float32 vector", + args: args{ + dist: Gaussian, + num: 5, + dim: 5, + }, + want: want{ + wantLen: 5, + wantDim: 5, + }, + }, + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got, err := GenF32Vec(test.args.dist, test.args.num, test.args.dim) + if err := checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func TestGenUint8Vec(t *testing.T) { + type args struct { + dist Distribution + num int + dim int + } + type want struct { + wantLen int + wantDim int + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, [][]float32, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got [][]float32, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if len(got) != w.wantLen { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twantLen: \"%#v\"", got, w.wantLen) + } + for _, vec := range got { + if len(vec) != w.wantDim { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twantDim: \"%#v\"", got, w.wantDim) + } + } + return nil + } + tests := []test{ + { + name: "return 1 generated float32 vector", + args: args{ + dist: Gaussian, + num: 1, + dim: 5, + }, + want: want{ + wantLen: 1, + wantDim: 5, + }, + }, + { + name: "return 5 generated float32 vector", + args: args{ + dist: Gaussian, + num: 5, + dim: 5, + }, + want: want{ + wantLen: 5, + wantDim: 5, + }, + }, + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got, err := GenUint8Vec(test.args.dist, test.args.num, test.args.dim) + if err := checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func TestGenSameValueVec(t *testing.T) { + type args struct { + size int + val float32 + } + type want struct { + want []float32 + } + type test struct { + name string + args args + want want + checkFunc func(want, []float32) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got []float32) error { + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + return nil + } + tests := []test{ + func() test { + val := float32(1) + return test{ + name: "return same value vector with size 1", + args: args{ + size: 1, + val: val, + }, + want: want{ + want: []float32{ + val, + }, + }, + checkFunc: defaultCheckFunc, + } + }(), + func() test { + val := float32(1) + return test{ + name: "return same value vector with size 5", + args: args{ + size: 5, + val: val, + }, + want: want{ + want: []float32{ + val, val, val, val, val, + }, + }, + checkFunc: defaultCheckFunc, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got := GenSameValueVec(test.args.size, test.args.val) + if err := checkFunc(test.want, got); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func TestConvertVectorsUint8ToFloat32(t *testing.T) { + type args struct { + vectors [][]uint8 + } + type want struct { + wantRet [][]float32 + } + type test struct { + name string + args args + want want + checkFunc func(want, [][]float32) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, gotRet [][]float32) error { + if !reflect.DeepEqual(gotRet, w.wantRet) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRet, w.wantRet) + } + return nil + } + tests := []test{ + { + name: "return 1 float32 vector from uint8 vector", + args: args{ + vectors: [][]uint8{ + { + 1, 2, 3, + }, + }, + }, + want: want{ + wantRet: [][]float32{ + { + 1, 2, 3, + }, + }, + }, + }, + { + name: "return 3 float32 vector from uint8 vector", + args: args{ + vectors: [][]uint8{ + { + 1, 2, 3, + }, + { + 4, 5, 6, + }, + { + 7, 8, 9, + }, + }, + }, + want: want{ + wantRet: [][]float32{ + { + 1, 2, 3, + }, + { + 4, 5, 6, + }, + { + 7, 8, 9, + }, + }, + }, + }, + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + gotRet := ConvertVectorsUint8ToFloat32(test.args.vectors) + if err := checkFunc(test.want, gotRet); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func Test_convertVectorUint8ToFloat32(t *testing.T) { + type args struct { + vector []uint8 + } + type want struct { + wantRet []float32 + } + type test struct { + name string + args args + want want + checkFunc func(want, []float32) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, gotRet []float32) error { + if !reflect.DeepEqual(gotRet, w.wantRet) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRet, w.wantRet) + } + return nil + } + tests := []test{ + { + name: "return float32 vector from uint8 vector", + args: args{ + vector: []uint8{ + 1, 2, 3, + }, + }, + want: want{ + wantRet: []float32{ + 1, 2, 3, + }, + }, + }, + { + name: "return float32 vector from uint8 vector with min value", + args: args{ + vector: []uint8{ + 0, 0, 0, + }, + }, + want: want{ + wantRet: []float32{ + 0, 0, 0, + }, + }, + }, + { + name: "return float32 vector from uint8 vector with max value", + args: args{ + vector: []uint8{ + math.MaxUint8, math.MaxUint8, math.MaxUint8, + }, + }, + want: want{ + wantRet: []float32{ + math.MaxUint8, math.MaxUint8, math.MaxUint8, + }, + }, + }, + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + gotRet := convertVectorUint8ToFloat32(test.args.vector) + if err := checkFunc(test.want, gotRet); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} From 61d76d549ca326c80b547b2c3710de0f58ba4307 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 1 Jun 2022 15:57:55 +0900 Subject: [PATCH 18/29] add comments to helper functions Signed-off-by: kevindiu --- internal/conv/conv.go | 2 ++ internal/test/data/request/object.go | 1 + internal/test/data/vector/gen.go | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/internal/conv/conv.go b/internal/conv/conv.go index 1d3f1219f4..6efbce4ae4 100644 --- a/internal/conv/conv.go +++ b/internal/conv/conv.go @@ -54,6 +54,7 @@ func F32stos(fs []float32) (s string) { return Btoa(buf) } +// Utf8ToSjis converts a UTF8 string to sjis string. func Utf8ToSjis(s string) (string, error) { b, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())) if err != nil { @@ -62,6 +63,7 @@ func Utf8ToSjis(s string) (string, error) { return string(b), nil } +// Utf8ToEucjp converts a UTF8 string to eucjp string. func Utf8ToEucjp(s string) (string, error) { b, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.EUCJP.NewEncoder())) if err != nil { diff --git a/internal/test/data/request/object.go b/internal/test/data/request/object.go index d1006fc185..75ca15185b 100644 --- a/internal/test/data/request/object.go +++ b/internal/test/data/request/object.go @@ -21,6 +21,7 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/payload" ) +// GenObjectLocations generate ObjectLocations payload with multiple name and ip with generated uuid. func GenObjectLocations(num int, name string, ipAddr string) *payload.Object_Locations { result := &payload.Object_Locations{ Locations: make([]*payload.Object_Location, num), diff --git a/internal/test/data/vector/gen.go b/internal/test/data/vector/gen.go index b1c162e472..2c3e0b3dfa 100644 --- a/internal/test/data/vector/gen.go +++ b/internal/test/data/vector/gen.go @@ -132,6 +132,7 @@ func gaussianDistributedUint8VectorGenerator(n, dim int, mean, sigma float64) [] }) } +// GenF32Vec returns multiple float32 vectors. func GenF32Vec(dist Distribution, num int, dim int) ([][]float32, error) { generator, err := Float32VectorGenerator(dist) if err != nil { @@ -140,6 +141,7 @@ func GenF32Vec(dist Distribution, num int, dim int) ([][]float32, error) { return generator(num, dim), nil } +// GenUint8Vec returns multiple uint8 vectors. func GenUint8Vec(dist Distribution, num int, dim int) ([][]float32, error) { generator, err := Uint8VectorGenerator(dist) if err != nil { @@ -148,6 +150,7 @@ func GenUint8Vec(dist Distribution, num int, dim int) ([][]float32, error) { return ConvertVectorsUint8ToFloat32(generator(num, dim)), nil } +// GenSameValueVec returns a float32 vector filled with value. func GenSameValueVec(size int, val float32) []float32 { v := make([]float32, size) for i := 0; i < size; i++ { @@ -156,6 +159,7 @@ func GenSameValueVec(size int, val float32) []float32 { return v } +// ConvertVectorsUint8ToFloat32 converts uint8 vectors and return float32 vectors func ConvertVectorsUint8ToFloat32(vectors [][]uint8) (ret [][]float32) { ret = make([][]float32, 0, len(vectors)) for _, v := range vectors { From 928b4c87314e6c90141e03fecdb4aa9902243b0b Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 1 Jun 2022 16:03:27 +0900 Subject: [PATCH 19/29] implement conv test Signed-off-by: kevindiu --- internal/conv/conv_test.go | 68 ++++++++++---------------------------- 1 file changed, 18 insertions(+), 50 deletions(-) diff --git a/internal/conv/conv_test.go b/internal/conv/conv_test.go index 39a744ed7e..6168c2a6d5 100644 --- a/internal/conv/conv_test.go +++ b/internal/conv/conv_test.go @@ -263,31 +263,15 @@ func TestUtf8ToSjis(t *testing.T) { return nil } tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - s: "", - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - s: "", - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ + { + name: "return sjis string from UTF8 string", + args: args{ + s: "こんにちは", + }, + want: want{ + want: "\x82\xb1\x82\xf1\x82ɂ\xbf\x82\xcd", + }, + }, } for _, tc := range tests { @@ -341,31 +325,15 @@ func TestUtf8ToEucjp(t *testing.T) { return nil } tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - s: "", - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - s: "", - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ + { + name: "return eucjp string from UTF8 string", + args: args{ + s: "こんにちは", + }, + want: want{ + want: "\xa4\xb3\xa4\xf3\xa4ˤ\xc1\xa4\xcf", + }, + }, } for _, tc := range tests { From 9eb32fc542480e50b93c6faf859c667fc598d0d5 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 1 Jun 2022 16:43:55 +0900 Subject: [PATCH 20/29] fix golangci warning Signed-off-by: kevindiu --- internal/conv/conv_test.go | 2 -- internal/test/data/request/insert_test.go | 2 -- internal/test/data/request/object_test.go | 1 - internal/test/data/vector/gen_test.go | 3 --- 4 files changed, 8 deletions(-) diff --git a/internal/conv/conv_test.go b/internal/conv/conv_test.go index 6168c2a6d5..c91d4904fb 100644 --- a/internal/conv/conv_test.go +++ b/internal/conv/conv_test.go @@ -294,7 +294,6 @@ func TestUtf8ToSjis(t *testing.T) { if err := checkFunc(test.want, got, err); err != nil { tt.Errorf("error = %v", err) } - }) } } @@ -356,7 +355,6 @@ func TestUtf8ToEucjp(t *testing.T) { if err := checkFunc(test.want, got, err); err != nil { tt.Errorf("error = %v", err) } - }) } } diff --git a/internal/test/data/request/insert_test.go b/internal/test/data/request/insert_test.go index 1e82b285e0..3ded576b11 100644 --- a/internal/test/data/request/insert_test.go +++ b/internal/test/data/request/insert_test.go @@ -212,7 +212,6 @@ func TestGenMultiInsertReq(t *testing.T) { if err := checkFunc(test.want, got, err); err != nil { tt.Errorf("error = %v", err) } - }) } } @@ -341,7 +340,6 @@ func TestGenSameVecMultiInsertReq(t *testing.T) { if err := checkFunc(test.want, got); err != nil { tt.Errorf("error = %v", err) } - }) } } diff --git a/internal/test/data/request/object_test.go b/internal/test/data/request/object_test.go index 64656863b8..53a58156c9 100644 --- a/internal/test/data/request/object_test.go +++ b/internal/test/data/request/object_test.go @@ -136,7 +136,6 @@ func TestGenObjectLocations(t *testing.T) { if err := checkFunc(test.want, got); err != nil { tt.Errorf("error = %v", err) } - }) } } diff --git a/internal/test/data/vector/gen_test.go b/internal/test/data/vector/gen_test.go index 1922b4681c..637ae1199d 100644 --- a/internal/test/data/vector/gen_test.go +++ b/internal/test/data/vector/gen_test.go @@ -834,7 +834,6 @@ func TestGenF32Vec(t *testing.T) { if err := checkFunc(test.want, got, err); err != nil { tt.Errorf("error = %v", err) } - }) } } @@ -919,7 +918,6 @@ func TestGenUint8Vec(t *testing.T) { if err := checkFunc(test.want, got, err); err != nil { tt.Errorf("error = %v", err) } - }) } } @@ -1001,7 +999,6 @@ func TestGenSameValueVec(t *testing.T) { if err := checkFunc(test.want, got); err != nil { tt.Errorf("error = %v", err) } - }) } } From c203493126d087dbf7b9d86ea29c26421ef4716a Mon Sep 17 00:00:00 2001 From: kevindiu Date: Wed, 1 Jun 2022 17:46:55 +0900 Subject: [PATCH 21/29] fix comment Signed-off-by: kevindiu --- internal/conv/conv_test.go | 18 +++++ internal/test/data/request/insert_test.go | 12 ++- internal/test/data/vector/gen.go | 16 ++-- internal/test/data/vector/gen_test.go | 90 ----------------------- 4 files changed, 29 insertions(+), 107 deletions(-) diff --git a/internal/conv/conv_test.go b/internal/conv/conv_test.go index c91d4904fb..4495cdeecb 100644 --- a/internal/conv/conv_test.go +++ b/internal/conv/conv_test.go @@ -272,6 +272,15 @@ func TestUtf8ToSjis(t *testing.T) { want: "\x82\xb1\x82\xf1\x82ɂ\xbf\x82\xcd", }, }, + { + name: "return empty string when the UTF8 string is empty", + args: args{ + s: "", + }, + want: want{ + want: "", + }, + }, } for _, tc := range tests { @@ -333,6 +342,15 @@ func TestUtf8ToEucjp(t *testing.T) { want: "\xa4\xb3\xa4\xf3\xa4ˤ\xc1\xa4\xcf", }, }, + { + name: "return empty string when the UTF8 string is empty", + args: args{ + s: "", + }, + want: want{ + want: "", + }, + }, } for _, tc := range tests { diff --git a/internal/test/data/request/insert_test.go b/internal/test/data/request/insert_test.go index 3ded576b11..a1ec1f2c30 100644 --- a/internal/test/data/request/insert_test.go +++ b/internal/test/data/request/insert_test.go @@ -26,13 +26,11 @@ import ( "github.com/vdaas/vald/internal/test/goleak" ) -var ( - defaultMultiInsertReqComparators = []cmp.Option{ - comparator.IgnoreUnexported(payload.Insert_Request{}), - comparator.IgnoreUnexported(payload.Insert_MultiRequest{}), - comparator.IgnoreUnexported(payload.Object_Vector{}), - } -) +var defaultMultiInsertReqComparators = []cmp.Option{ + comparator.IgnoreUnexported(payload.Insert_Request{}), + comparator.IgnoreUnexported(payload.Insert_MultiRequest{}), + comparator.IgnoreUnexported(payload.Object_Vector{}), +} func TestGenMultiInsertReq(t *testing.T) { type args struct { diff --git a/internal/test/data/vector/gen.go b/internal/test/data/vector/gen.go index 2c3e0b3dfa..5401782610 100644 --- a/internal/test/data/vector/gen.go +++ b/internal/test/data/vector/gen.go @@ -162,16 +162,12 @@ func GenSameValueVec(size int, val float32) []float32 { // ConvertVectorsUint8ToFloat32 converts uint8 vectors and return float32 vectors func ConvertVectorsUint8ToFloat32(vectors [][]uint8) (ret [][]float32) { ret = make([][]float32, 0, len(vectors)) - for _, v := range vectors { - ret = append(ret, convertVectorUint8ToFloat32(v)) - } - return -} - -func convertVectorUint8ToFloat32(vector []uint8) (ret []float32) { - ret = make([]float32, len(vector)) - for i, e := range vector { - ret[i] = float32(e) + for _, vec := range vectors { + fvec := make([]float32, len(vec)) + for i, v := range vec { + fvec[i] = float32(v) + } + ret = append(ret, fvec) } return } diff --git a/internal/test/data/vector/gen_test.go b/internal/test/data/vector/gen_test.go index 637ae1199d..6177ee5985 100644 --- a/internal/test/data/vector/gen_test.go +++ b/internal/test/data/vector/gen_test.go @@ -16,7 +16,6 @@ package vector import ( - "math" "reflect" "testing" @@ -1093,95 +1092,6 @@ func TestConvertVectorsUint8ToFloat32(t *testing.T) { if err := checkFunc(test.want, gotRet); err != nil { tt.Errorf("error = %v", err) } - - }) - } -} - -func Test_convertVectorUint8ToFloat32(t *testing.T) { - type args struct { - vector []uint8 - } - type want struct { - wantRet []float32 - } - type test struct { - name string - args args - want want - checkFunc func(want, []float32) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want, gotRet []float32) error { - if !reflect.DeepEqual(gotRet, w.wantRet) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRet, w.wantRet) - } - return nil - } - tests := []test{ - { - name: "return float32 vector from uint8 vector", - args: args{ - vector: []uint8{ - 1, 2, 3, - }, - }, - want: want{ - wantRet: []float32{ - 1, 2, 3, - }, - }, - }, - { - name: "return float32 vector from uint8 vector with min value", - args: args{ - vector: []uint8{ - 0, 0, 0, - }, - }, - want: want{ - wantRet: []float32{ - 0, 0, 0, - }, - }, - }, - { - name: "return float32 vector from uint8 vector with max value", - args: args{ - vector: []uint8{ - math.MaxUint8, math.MaxUint8, math.MaxUint8, - }, - }, - want: want{ - wantRet: []float32{ - math.MaxUint8, math.MaxUint8, math.MaxUint8, - }, - }, - }, - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - checkFunc := test.checkFunc - if test.checkFunc == nil { - checkFunc = defaultCheckFunc - } - - gotRet := convertVectorUint8ToFloat32(test.args.vector) - if err := checkFunc(test.want, gotRet); err != nil { - tt.Errorf("error = %v", err) - } - }) } } From c85b54c431e907336fd8965ce6d59b7b480bf2e8 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Thu, 2 Jun 2022 10:01:25 +0900 Subject: [PATCH 22/29] fix golangci warning Signed-off-by: kevindiu --- internal/test/data/request/object_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/test/data/request/object_test.go b/internal/test/data/request/object_test.go index 53a58156c9..b5425de7e3 100644 --- a/internal/test/data/request/object_test.go +++ b/internal/test/data/request/object_test.go @@ -25,12 +25,10 @@ import ( "github.com/vdaas/vald/internal/test/goleak" ) -var ( - defaultOvjectLocationComparators = []cmp.Option{ - comparator.IgnoreUnexported(payload.Object_Locations{}), - comparator.IgnoreUnexported(payload.Object_Location{}), - } -) +var defaultObjectLocationComparators = []cmp.Option{ + comparator.IgnoreUnexported(payload.Object_Locations{}), + comparator.IgnoreUnexported(payload.Object_Location{}), +} func TestGenObjectLocations(t *testing.T) { type args struct { @@ -50,7 +48,7 @@ func TestGenObjectLocations(t *testing.T) { afterFunc func(args) } defaultCheckFunc := func(w want, got *payload.Object_Locations) error { - if diff := comparator.Diff(got, w.want, defaultOvjectLocationComparators...); diff != "" { + if diff := comparator.Diff(got, w.want, defaultObjectLocationComparators...); diff != "" { return errors.Errorf("diff: %v", diff) } return nil From 5ad9c5ab43c62665832eb9e8002ce5697f211104 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Thu, 2 Jun 2022 10:12:21 +0900 Subject: [PATCH 23/29] split encode() Signed-off-by: kevindiu --- internal/conv/conv.go | 13 +++--- internal/conv/conv_test.go | 82 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 6 deletions(-) diff --git a/internal/conv/conv.go b/internal/conv/conv.go index 6efbce4ae4..6d38e3a7ac 100644 --- a/internal/conv/conv.go +++ b/internal/conv/conv.go @@ -16,6 +16,7 @@ package conv import ( + "io" "io/ioutil" "reflect" "strings" @@ -56,16 +57,16 @@ func F32stos(fs []float32) (s string) { // Utf8ToSjis converts a UTF8 string to sjis string. func Utf8ToSjis(s string) (string, error) { - b, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())) - if err != nil { - return "", err - } - return string(b), nil + return encode(strings.NewReader(s), japanese.ShiftJIS.NewEncoder()) } // Utf8ToEucjp converts a UTF8 string to eucjp string. func Utf8ToEucjp(s string) (string, error) { - b, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(s), japanese.EUCJP.NewEncoder())) + return encode(strings.NewReader(s), japanese.EUCJP.NewEncoder()) +} + +func encode(r io.Reader, t transform.Transformer) (string, error) { + b, err := ioutil.ReadAll(transform.NewReader(r, t)) if err != nil { return "", err } diff --git a/internal/conv/conv_test.go b/internal/conv/conv_test.go index 4495cdeecb..15b8bb1fc7 100644 --- a/internal/conv/conv_test.go +++ b/internal/conv/conv_test.go @@ -16,11 +16,16 @@ package conv import ( + "io" "reflect" + "strings" "testing" + "testing/iotest" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/test/goleak" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/transform" ) var testData = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" @@ -376,3 +381,80 @@ func TestUtf8ToEucjp(t *testing.T) { }) } } + +func Test_encode(t *testing.T) { + type args struct { + r io.Reader + t transform.Transformer + } + type want struct { + want string + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, string, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got string, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + return nil + } + tests := []test{ + { + name: "success to encode string", + args: args{ + r: strings.NewReader("こんにちは"), + t: japanese.EUCJP.NewEncoder(), + }, + want: want{ + want: "\xa4\xb3\xa4\xf3\xa4ˤ\xc1\xa4\xcf", + }, + }, + func() test { + err := errors.New("invalid reader") + return test{ + name: "fail to encode string", + args: args{ + r: iotest.ErrReader(err), + t: japanese.EUCJP.NewEncoder(), + }, + want: want{ + err: err, + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got, err := encode(test.args.r, test.args.t) + if err := checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} From b6591a46a73179f2a3e5715b49ba264dd3a45ddf Mon Sep 17 00:00:00 2001 From: kevindiu Date: Thu, 2 Jun 2022 10:24:10 +0900 Subject: [PATCH 24/29] implement boundary value test for vector/gen Signed-off-by: kevindiu --- internal/test/data/vector/gen_test.go | 39 +++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/internal/test/data/vector/gen_test.go b/internal/test/data/vector/gen_test.go index 6177ee5985..cd29bc268f 100644 --- a/internal/test/data/vector/gen_test.go +++ b/internal/test/data/vector/gen_test.go @@ -16,6 +16,7 @@ package vector import ( + "math" "reflect" "testing" @@ -1025,7 +1026,7 @@ func TestConvertVectorsUint8ToFloat32(t *testing.T) { } tests := []test{ { - name: "return 1 float32 vector from uint8 vector", + name: "convert 1 float32 vector from uint8 vector", args: args{ vectors: [][]uint8{ { @@ -1042,7 +1043,7 @@ func TestConvertVectorsUint8ToFloat32(t *testing.T) { }, }, { - name: "return 3 float32 vector from uint8 vector", + name: "convert 3 float32 vector from uint8 vector", args: args{ vectors: [][]uint8{ { @@ -1070,6 +1071,40 @@ func TestConvertVectorsUint8ToFloat32(t *testing.T) { }, }, }, + { + name: "convert 0 value vector", + args: args{ + vectors: [][]uint8{ + { + 0, 0, 0, + }, + }, + }, + want: want{ + wantRet: [][]float32{ + { + 0, 0, 0, + }, + }, + }, + }, + { + name: "convert max value vector", + args: args{ + vectors: [][]uint8{ + { + math.MaxUint8, math.MaxUint8, math.MaxUint8, + }, + }, + }, + want: want{ + wantRet: [][]float32{ + { + math.MaxUint8, math.MaxUint8, math.MaxUint8, + }, + }, + }, + }, } for _, tc := range tests { From f60334a3a66d79515a9889480c867c5c79ad626b Mon Sep 17 00:00:00 2001 From: kevindiu Date: Thu, 2 Jun 2022 10:25:26 +0900 Subject: [PATCH 25/29] use io instead of ioutil package Signed-off-by: kevindiu --- internal/conv/conv.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/conv/conv.go b/internal/conv/conv.go index 6d38e3a7ac..26ce135a7a 100644 --- a/internal/conv/conv.go +++ b/internal/conv/conv.go @@ -17,7 +17,6 @@ package conv import ( "io" - "io/ioutil" "reflect" "strings" "unsafe" @@ -66,7 +65,7 @@ func Utf8ToEucjp(s string) (string, error) { } func encode(r io.Reader, t transform.Transformer) (string, error) { - b, err := ioutil.ReadAll(transform.NewReader(r, t)) + b, err := io.ReadAll(transform.NewReader(r, t)) if err != nil { return "", err } From dd4d4f7231e0032e823c652d6e1ab592fcae26e9 Mon Sep 17 00:00:00 2001 From: Kevin Diu Date: Thu, 2 Jun 2022 11:09:01 +0900 Subject: [PATCH 26/29] Update internal/conv/conv_test.go Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- internal/conv/conv_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/conv/conv_test.go b/internal/conv/conv_test.go index 15b8bb1fc7..2dd3a7476d 100644 --- a/internal/conv/conv_test.go +++ b/internal/conv/conv_test.go @@ -454,7 +454,6 @@ func Test_encode(t *testing.T) { if err := checkFunc(test.want, got, err); err != nil { tt.Errorf("error = %v", err) } - }) } } From be8dd0b93417f28f67d09310479e19dc8aaead66 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Thu, 2 Jun 2022 14:31:16 +0900 Subject: [PATCH 27/29] add boundary tests Signed-off-by: kevindiu --- internal/test/data/request/insert_test.go | 41 +++++++++++++++++++++++ internal/test/data/request/object_test.go | 13 +++++++ 2 files changed, 54 insertions(+) diff --git a/internal/test/data/request/insert_test.go b/internal/test/data/request/insert_test.go index a1ec1f2c30..26bc67acd3 100644 --- a/internal/test/data/request/insert_test.go +++ b/internal/test/data/request/insert_test.go @@ -188,6 +188,34 @@ func TestGenMultiInsertReq(t *testing.T) { }, }, }, + { + name: "success to generate 0 float request", + args: args{ + t: Float, + dist: vector.Gaussian, + num: 0, + dim: dim, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{}, + }, + }, + }, + { + name: "success to generate 0 uint8 request", + args: args{ + t: Uint8, + dist: vector.Gaussian, + num: 0, + dim: dim, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{}, + }, + }, + }, } for _, tc := range tests { @@ -316,6 +344,19 @@ func TestGenSameVecMultiInsertReq(t *testing.T) { }, } }(), + { + name: "success to generate 0 same vector request", + args: args{ + num: 0, + vec: []float32{1, 2, 3}, + cfg: nil, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{}, + }, + }, + }, } for _, tc := range tests { diff --git a/internal/test/data/request/object_test.go b/internal/test/data/request/object_test.go index b5425de7e3..60e8407849 100644 --- a/internal/test/data/request/object_test.go +++ b/internal/test/data/request/object_test.go @@ -112,6 +112,19 @@ func TestGenObjectLocations(t *testing.T) { }, }, }, + { + name: "success to generate 0 object location", + args: args{ + num: 0, + name: "vald-agent-01", + ipAddr: "127.0.0.1", + }, + want: want{ + want: &payload.Object_Locations{ + Locations: []*payload.Object_Location{}, + }, + }, + }, } for _, tc := range tests { From 082ac196df09fe210ae9d2c1deca8ef2ce2e3376 Mon Sep 17 00:00:00 2001 From: kevindiu Date: Thu, 2 Jun 2022 14:40:22 +0900 Subject: [PATCH 28/29] fix test error Signed-off-by: kevindiu --- internal/test/data/request/insert_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/test/data/request/insert_test.go b/internal/test/data/request/insert_test.go index 26bc67acd3..5be9525cc7 100644 --- a/internal/test/data/request/insert_test.go +++ b/internal/test/data/request/insert_test.go @@ -62,7 +62,7 @@ func TestGenMultiInsertReq(t *testing.T) { if diff := comparator.Diff(got, w.want, comparators...); diff != "" { return errors.Errorf("diff: %v", diff) } - if len(got.Requests[0].Vector.Vector) != dim { + if len(got.Requests) != 0 && len(got.Requests[0].Vector.Vector) != dim { return errors.New("vector length not match") } return nil From c477b8532b546a6b77296dd163c9fdbe8862a22f Mon Sep 17 00:00:00 2001 From: kevindiu Date: Thu, 2 Jun 2022 16:04:45 +0900 Subject: [PATCH 29/29] add more boundary tests Signed-off-by: kevindiu --- internal/test/data/request/insert_test.go | 112 ++++++++++++++++++- internal/test/data/request/object_test.go | 1 + internal/test/data/vector/gen_test.go | 129 +++++++++++++++++++++- 3 files changed, 236 insertions(+), 6 deletions(-) diff --git a/internal/test/data/request/insert_test.go b/internal/test/data/request/insert_test.go index 5be9525cc7..52342705c6 100644 --- a/internal/test/data/request/insert_test.go +++ b/internal/test/data/request/insert_test.go @@ -30,6 +30,7 @@ var defaultMultiInsertReqComparators = []cmp.Option{ comparator.IgnoreUnexported(payload.Insert_Request{}), comparator.IgnoreUnexported(payload.Insert_MultiRequest{}), comparator.IgnoreUnexported(payload.Object_Vector{}), + comparator.IgnoreUnexported(payload.Insert_Config{}), } func TestGenMultiInsertReq(t *testing.T) { @@ -216,6 +217,7 @@ func TestGenMultiInsertReq(t *testing.T) { }, }, }, + // max num and max dim test is ignored due to test timeout } for _, tc := range tests { @@ -271,13 +273,16 @@ func TestGenSameVecMultiInsertReq(t *testing.T) { if err != nil { t.Error(err) } + cfg := &payload.Insert_Config{ + SkipStrictExistCheck: true, + } return test{ name: "success to generate 1 same vector request", args: args{ num: 1, vec: vecs[0], - cfg: nil, + cfg: cfg, }, want: want{ want: &payload.Insert_MultiRequest{ @@ -287,6 +292,7 @@ func TestGenSameVecMultiInsertReq(t *testing.T) { Id: "uuid-1", Vector: vecs[0], }, + Config: cfg, }, }, }, @@ -298,13 +304,16 @@ func TestGenSameVecMultiInsertReq(t *testing.T) { if err != nil { t.Error(err) } + cfg := &payload.Insert_Config{ + SkipStrictExistCheck: true, + } return test{ name: "success to generate 5 same vector request", args: args{ num: 5, vec: vecs[0], - cfg: nil, + cfg: cfg, }, want: want{ want: &payload.Insert_MultiRequest{ @@ -314,30 +323,64 @@ func TestGenSameVecMultiInsertReq(t *testing.T) { Id: "uuid-1", Vector: vecs[0], }, + Config: cfg, }, { Vector: &payload.Object_Vector{ Id: "uuid-2", Vector: vecs[0], }, + Config: cfg, }, { Vector: &payload.Object_Vector{ Id: "uuid-3", Vector: vecs[0], }, + Config: cfg, }, { Vector: &payload.Object_Vector{ Id: "uuid-4", Vector: vecs[0], }, + Config: cfg, }, { Vector: &payload.Object_Vector{ Id: "uuid-5", Vector: vecs[0], }, + Config: cfg, + }, + }, + }, + }, + } + }(), + func() test { + vecs, err := vector.GenF32Vec(vector.Gaussian, 1, 10) + if err != nil { + t.Error(err) + } + var cfg *payload.Insert_Config + + return test{ + name: "success to generate 1 same vector request when cfg is nil", + args: args{ + num: 1, + vec: vecs[0], + cfg: cfg, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{ + { + Vector: &payload.Object_Vector{ + Id: "uuid-1", + Vector: vecs[0], + }, + Config: cfg, }, }, }, @@ -357,6 +400,71 @@ func TestGenSameVecMultiInsertReq(t *testing.T) { }, }, }, + { + name: "success to generate empty vector request", + args: args{ + num: 1, + vec: []float32{}, + cfg: nil, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{ + { + Vector: &payload.Object_Vector{ + Id: "uuid-1", + Vector: []float32{}, + }, + }, + }, + }, + }, + }, + { + name: "success to generate multiple empty vector request", + args: args{ + num: 5, + vec: []float32{}, + cfg: nil, + }, + want: want{ + want: &payload.Insert_MultiRequest{ + Requests: []*payload.Insert_Request{ + { + Vector: &payload.Object_Vector{ + Id: "uuid-1", + Vector: []float32{}, + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-2", + Vector: []float32{}, + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-3", + Vector: []float32{}, + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-4", + Vector: []float32{}, + }, + }, + { + Vector: &payload.Object_Vector{ + Id: "uuid-5", + Vector: []float32{}, + }, + }, + }, + }, + }, + }, + // max num test is ignored due to test timeout } for _, tc := range tests { diff --git a/internal/test/data/request/object_test.go b/internal/test/data/request/object_test.go index 60e8407849..5aeabfacbc 100644 --- a/internal/test/data/request/object_test.go +++ b/internal/test/data/request/object_test.go @@ -125,6 +125,7 @@ func TestGenObjectLocations(t *testing.T) { }, }, }, + // max num test is ignored due to test timeout } for _, tc := range tests { diff --git a/internal/test/data/vector/gen_test.go b/internal/test/data/vector/gen_test.go index cd29bc268f..5749678387 100644 --- a/internal/test/data/vector/gen_test.go +++ b/internal/test/data/vector/gen_test.go @@ -812,6 +812,30 @@ func TestGenF32Vec(t *testing.T) { wantDim: 5, }, }, + { + name: "return no generated float32 vector with num 0", + args: args{ + dist: Gaussian, + num: 0, + dim: 5, + }, + want: want{ + wantLen: 0, + }, + }, + { + name: "return generated float32 vector with dim 0", + args: args{ + dist: Gaussian, + num: 1, + dim: 0, + }, + want: want{ + wantLen: 1, + wantDim: 0, + }, + }, + // max dim and max num test is ignored due to test timeout } for _, tc := range tests { @@ -873,7 +897,7 @@ func TestGenUint8Vec(t *testing.T) { } tests := []test{ { - name: "return 1 generated float32 vector", + name: "return 1 generated uint8 vector", args: args{ dist: Gaussian, num: 1, @@ -885,7 +909,7 @@ func TestGenUint8Vec(t *testing.T) { }, }, { - name: "return 5 generated float32 vector", + name: "return 5 generated uint8 vector", args: args{ dist: Gaussian, num: 5, @@ -896,6 +920,30 @@ func TestGenUint8Vec(t *testing.T) { wantDim: 5, }, }, + { + name: "return no generated uint8 vector with num 0", + args: args{ + dist: Gaussian, + num: 0, + dim: 5, + }, + want: want{ + wantLen: 0, + }, + }, + { + name: "return generated uint8 vector with dim 0", + args: args{ + dist: Gaussian, + num: 1, + dim: 0, + }, + want: want{ + wantLen: 1, + wantDim: 0, + }, + }, + // max dim and max num test is ignored due to test timeout } for _, tc := range tests { @@ -958,7 +1006,6 @@ func TestGenSameValueVec(t *testing.T) { val, }, }, - checkFunc: defaultCheckFunc, } }(), func() test { @@ -974,9 +1021,52 @@ func TestGenSameValueVec(t *testing.T) { val, val, val, val, val, }, }, - checkFunc: defaultCheckFunc, } }(), + func() test { + val := float32(1) + return test{ + name: "return same value vector with size 0", + args: args{ + size: 0, + val: val, + }, + want: want{ + want: []float32{}, + }, + } + }(), + func() test { + val := float32(math.SmallestNonzeroFloat32) + return test{ + name: "return same value vector with min value", + args: args{ + size: 1, + val: val, + }, + want: want{ + want: []float32{ + val, + }, + }, + } + }(), + func() test { + val := float32(math.MaxFloat32) + return test{ + name: "return same value vector with max value", + args: args{ + size: 1, + val: val, + }, + want: want{ + want: []float32{ + val, + }, + }, + } + }(), + // max size test is ignored due to test timeout } for _, tc := range tests { @@ -1105,6 +1195,37 @@ func TestConvertVectorsUint8ToFloat32(t *testing.T) { }, }, }, + { + name: "return empty slice when vectors is empty", + args: args{ + vectors: [][]uint8{}, + }, + want: want{ + wantRet: [][]float32{}, + }, + }, + { + name: "return empty slice when vectors is nil", + args: args{ + vectors: nil, + }, + want: want{ + wantRet: [][]float32{}, + }, + }, + { + name: "return empty vector when vector is empty", + args: args{ + vectors: [][]uint8{ + {}, + }, + }, + want: want{ + wantRet: [][]float32{ + {}, + }, + }, + }, } for _, tc := range tests {