Refactor pkg test helper functions

internal/conv/conv.go

@@ -16,8 +16,13 @@
 package conv
 
 import (
+	"io"
 	"reflect"
+	"strings"
 	"unsafe"
+
+	"golang.org/x/text/encoding/japanese"
+	"golang.org/x/text/transform"
 )
 
 // Btoa converts from byte slice to string.
@@ -48,3 +53,21 @@ func F32stos(fs []float32) (s string) {
 	(*(*int)(unsafe.Pointer(uintptr(addr) + uintptr(16)))) = lf
 	return Btoa(buf)
 }
+
+// Utf8ToSjis converts a UTF8 string to sjis string.
+func Utf8ToSjis(s string) (string, error) {
+	return encode(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())
+}
+
+// Utf8ToEucjp converts a UTF8 string to eucjp string.
+func Utf8ToEucjp(s string) (string, error) {
+	return encode(strings.NewReader(s), japanese.EUCJP.NewEncoder())
+}
+
+func encode(r io.Reader, t transform.Transformer) (string, error) {
+	b, err := io.ReadAll(transform.NewReader(r, t))
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}

internal/conv/conv_test.go

@@ -16,11 +16,16 @@
 package conv
 
 import (
+	"io"
 	"reflect"
+	"strings"
 	"testing"
+	"testing/iotest"
 
 	"github.com/vdaas/vald/internal/errors"
 	"github.com/vdaas/vald/internal/test/goleak"
+	"golang.org/x/text/encoding/japanese"
+	"golang.org/x/text/transform"
 )
 
 var testData = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
@@ -236,3 +241,219 @@ func TestF32stos(t *testing.T) {
 		})
 	}
 }
+
+func TestUtf8ToSjis(t *testing.T) {
+	type args struct {
+		s string
+	}
+	type want struct {
+		want string
+		err  error
+	}
+	type test struct {
+		name       string
+		args       args
+		want       want
+		checkFunc  func(want, string, error) error
+		beforeFunc func(args)
+		afterFunc  func(args)
+	}
+	defaultCheckFunc := func(w want, got string, err error) error {
+		if !errors.Is(err, w.err) {
+			return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
+		}
+		if !reflect.DeepEqual(got, w.want) {
+			return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+		}
+		return nil
+	}
+	tests := []test{
+		{
+			name: "return sjis string from UTF8 string",
+			args: args{
+				s: "こんにちは",
+			},
+			want: want{
+				want: "\x82\xb1\x82\xf1\x82ɂ\xbf\x82\xcd",
+			},
+		},
+		{
+			name: "return empty string when the UTF8 string is empty",
+			args: args{
+				s: "",
+			},
+			want: want{
+				want: "",
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		test := tc
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+			if test.beforeFunc != nil {
+				test.beforeFunc(test.args)
+			}
+			if test.afterFunc != nil {
+				defer test.afterFunc(test.args)
+			}
+			checkFunc := test.checkFunc
+			if test.checkFunc == nil {
+				checkFunc = defaultCheckFunc
+			}
+
+			got, err := Utf8ToSjis(test.args.s)
+			if err := checkFunc(test.want, got, err); err != nil {
+				tt.Errorf("error = %v", err)
+			}
+		})
+	}
+}
+
+func TestUtf8ToEucjp(t *testing.T) {
+	type args struct {
+		s string
+	}
+	type want struct {
+		want string
+		err  error
+	}
+	type test struct {
+		name       string
+		args       args
+		want       want
+		checkFunc  func(want, string, error) error
+		beforeFunc func(args)
+		afterFunc  func(args)
+	}
+	defaultCheckFunc := func(w want, got string, err error) error {
+		if !errors.Is(err, w.err) {
+			return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
+		}
+		if !reflect.DeepEqual(got, w.want) {
+			return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+		}
+		return nil
+	}
+	tests := []test{
+		{
+			name: "return eucjp string from UTF8 string",
+			args: args{
+				s: "こんにちは",
+			},
+			want: want{
+				want: "\xa4\xb3\xa4\xf3\xa4ˤ\xc1\xa4\xcf",
+			},
+		},
+		{
+			name: "return empty string when the UTF8 string is empty",
+			args: args{
+				s: "",
+			},
+			want: want{
+				want: "",
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		test := tc
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+			if test.beforeFunc != nil {
+				test.beforeFunc(test.args)
+			}
+			if test.afterFunc != nil {
+				defer test.afterFunc(test.args)
+			}
+			checkFunc := test.checkFunc
+			if test.checkFunc == nil {
+				checkFunc = defaultCheckFunc
+			}
+
+			got, err := Utf8ToEucjp(test.args.s)
+			if err := checkFunc(test.want, got, err); err != nil {
+				tt.Errorf("error = %v", err)
+			}
+		})
+	}
+}
+
+func Test_encode(t *testing.T) {
+	type args struct {
+		r io.Reader
+		t transform.Transformer
+	}
+	type want struct {
+		want string
+		err  error
+	}
+	type test struct {
+		name       string
+		args       args
+		want       want
+		checkFunc  func(want, string, error) error
+		beforeFunc func(args)
+		afterFunc  func(args)
+	}
+	defaultCheckFunc := func(w want, got string, err error) error {
+		if !errors.Is(err, w.err) {
+			return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
+		}
+		if !reflect.DeepEqual(got, w.want) {
+			return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+		}
+		return nil
+	}
+	tests := []test{
+		{
+			name: "success to encode string",
+			args: args{
+				r: strings.NewReader("こんにちは"),
+				t: japanese.EUCJP.NewEncoder(),
+			},
+			want: want{
+				want: "\xa4\xb3\xa4\xf3\xa4ˤ\xc1\xa4\xcf",
+			},
+		},
+		func() test {
+			err := errors.New("invalid reader")
+			return test{
+				name: "fail to encode string",
+				args: args{
+					r: iotest.ErrReader(err),
+					t: japanese.EUCJP.NewEncoder(),
+				},
+				want: want{
+					err: err,
+				},
+			}
+		}(),
+	}
+
+	for _, tc := range tests {
+		test := tc
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+			if test.beforeFunc != nil {
+				test.beforeFunc(test.args)
+			}
+			if test.afterFunc != nil {
+				defer test.afterFunc(test.args)
+			}
+			checkFunc := test.checkFunc
+			if test.checkFunc == nil {
+				checkFunc = defaultCheckFunc
+			}
+
+			got, err := encode(test.args.r, test.args.t)
+			if err := checkFunc(test.want, got, err); err != nil {
+				tt.Errorf("error = %v", err)
+			}
+		})
+	}
+}

internal/test/data/request/doc.go

@@ -0,0 +1,18 @@
+//
+// Copyright (C) 2019-2022 vdaas.org vald team <vald@vdaas.org>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package request provides functions to generate proto request for testing
+package request

internal/test/data/request/insert.go

@@ -0,0 +1,77 @@
+//
+// Copyright (C) 2019-2022 vdaas.org vald team <vald@vdaas.org>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package request
+
+import (
+	"strconv"
+
+	"github.com/vdaas/vald/apis/grpc/v1/payload"
+	"github.com/vdaas/vald/internal/test/data/vector"
+)
+
+type ObjectType int
+
+const (
+	Uint8 ObjectType = iota
+	Float
+)
+
+func GenMultiInsertReq(t ObjectType, dist vector.Distribution, num int, dim int, cfg *payload.Insert_Config) (*payload.Insert_MultiRequest, error) {
+	var vecs [][]float32
+	var err error
+	switch t {
+	case Float:
+		vecs, err = vector.GenF32Vec(dist, num, dim)
+	case Uint8:
+		vecs, err = vector.GenUint8Vec(dist, num, dim)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	req := &payload.Insert_MultiRequest{
+		Requests: make([]*payload.Insert_Request, num),
+	}
+	for i, vec := range vecs {
+		req.Requests[i] = &payload.Insert_Request{
+			Vector: &payload.Object_Vector{
+				Id:     "uuid-" + strconv.Itoa(i+1),
+				Vector: vec,
+			},
+			Config: cfg,
+		}
+	}
+
+	return req, nil
+}
+
+// generate MultiInsert request with the same vector
+func GenSameVecMultiInsertReq(num int, vec []float32, cfg *payload.Insert_Config) *payload.Insert_MultiRequest {
+	req := &payload.Insert_MultiRequest{
+		Requests: make([]*payload.Insert_Request, num),
+	}
+	for i := 0; i < num; i++ {
+		req.Requests[i] = &payload.Insert_Request{
+			Vector: &payload.Object_Vector{
+				Id:     "uuid-" + strconv.Itoa(i+1),
+				Vector: vec,
+			},
+			Config: cfg,
+		}
+	}
+
+	return req
+}