Skip to content

Commit

Permalink
♻️ Refactor pkg test helper functions (#1678)
Browse files Browse the repository at this point in the history 
* refactor charset helper func

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* refactor vector helper func

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* refactor request helper func

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* add missing files

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* move charset to conv

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* return error from helper func

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* refactor conv helper func

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* refactor fill() and implement buildIndex()

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* fix comment

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* refactor convertXXX

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* refactor search test func

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* refactor beforeFunc test func to use buildIndex

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* 🤖 Update license headers / Format go codes and yaml files

Signed-off-by: Vdaas CI <vald@vdaas.org>

* Revert "🤖 Update license headers / Format go codes and yaml files"

This reverts commit a8b44c3.

* format code

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* fix comment

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* implement test for helper functions

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* add comments to helper functions

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* implement conv test

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* fix golangci warning

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* fix comment

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* fix golangci warning

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* split encode()

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* implement boundary value test for vector/gen

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* use io instead of ioutil package

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* Update internal/conv/conv_test.go

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* add boundary tests

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* fix test error

Signed-off-by: kevindiu <kevindiujp@gmail.com>

* add more boundary tests

Signed-off-by: kevindiu <kevindiujp@gmail.com>

Co-authored-by: Vdaas CI <vald@vdaas.org>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
  • Loading branch information
@kevindiu @vdaas-ci @github-actions
3 people authored Jun 2, 2022
1 parent 815b44a commit 626c706
Show file tree
Hide file tree
Showing 10 changed files with 2,272 additions and 693 deletions.
23 changes: 23 additions & 0 deletions internal/conv/conv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,13 @@
package conv

import (
"io"
"reflect"
"strings"
"unsafe"

"golang.org/x/text/encoding/japanese"
"golang.org/x/text/transform"
)

// Btoa converts from byte slice to string.
Expand Down Expand Up @@ -48,3 +53,21 @@ func F32stos(fs []float32) (s string) {
(*(*int)(unsafe.Pointer(uintptr(addr) + uintptr(16)))) = lf
return Btoa(buf)
}

// Utf8ToSjis converts a UTF8 string to sjis string.
func Utf8ToSjis(s string) (string, error) {
return encode(strings.NewReader(s), japanese.ShiftJIS.NewEncoder())
}

// Utf8ToEucjp converts a UTF8 string to eucjp string.
func Utf8ToEucjp(s string) (string, error) {
return encode(strings.NewReader(s), japanese.EUCJP.NewEncoder())
}

func encode(r io.Reader, t transform.Transformer) (string, error) {
b, err := io.ReadAll(transform.NewReader(r, t))
if err != nil {
return "", err
}
return string(b), nil
}
221 changes: 221 additions & 0 deletions internal/conv/conv_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,16 @@
package conv

import (
"io"
"reflect"
"strings"
"testing"
"testing/iotest"

"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/test/goleak"
"golang.org/x/text/encoding/japanese"
"golang.org/x/text/transform"
)

var testData = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
Expand Down Expand Up @@ -236,3 +241,219 @@ func TestF32stos(t *testing.T) {
})
}
}

func TestUtf8ToSjis(t *testing.T) {
type args struct {
s string
}
type want struct {
want string
err error
}
type test struct {
name string
args args
want want
checkFunc func(want, string, error) error
beforeFunc func(args)
afterFunc func(args)
}
defaultCheckFunc := func(w want, got string, err error) error {
if !errors.Is(err, w.err) {
return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
}
if !reflect.DeepEqual(got, w.want) {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
}
return nil
}
tests := []test{
{
name: "return sjis string from UTF8 string",
args: args{
s: "こんにちは",
},
want: want{
want: "\x82\xb1\x82\xf1\x82ɂ\xbf\x82\xcd",
},
},
{
name: "return empty string when the UTF8 string is empty",
args: args{
s: "",
},
want: want{
want: "",
},
},
}

for _, tc := range tests {
test := tc
t.Run(test.name, func(tt *testing.T) {
tt.Parallel()
defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
if test.beforeFunc != nil {
test.beforeFunc(test.args)
}
if test.afterFunc != nil {
defer test.afterFunc(test.args)
}
checkFunc := test.checkFunc
if test.checkFunc == nil {
checkFunc = defaultCheckFunc
}

got, err := Utf8ToSjis(test.args.s)
if err := checkFunc(test.want, got, err); err != nil {
tt.Errorf("error = %v", err)
}
})
}
}

func TestUtf8ToEucjp(t *testing.T) {
type args struct {
s string
}
type want struct {
want string
err error
}
type test struct {
name string
args args
want want
checkFunc func(want, string, error) error
beforeFunc func(args)
afterFunc func(args)
}
defaultCheckFunc := func(w want, got string, err error) error {
if !errors.Is(err, w.err) {
return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
}
if !reflect.DeepEqual(got, w.want) {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
}
return nil
}
tests := []test{
{
name: "return eucjp string from UTF8 string",
args: args{
s: "こんにちは",
},
want: want{
want: "\xa4\xb3\xa4\xf3\xa4ˤ\xc1\xa4\xcf",
},
},
{
name: "return empty string when the UTF8 string is empty",
args: args{
s: "",
},
want: want{
want: "",
},
},
}

for _, tc := range tests {
test := tc
t.Run(test.name, func(tt *testing.T) {
tt.Parallel()
defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
if test.beforeFunc != nil {
test.beforeFunc(test.args)
}
if test.afterFunc != nil {
defer test.afterFunc(test.args)
}
checkFunc := test.checkFunc
if test.checkFunc == nil {
checkFunc = defaultCheckFunc
}

got, err := Utf8ToEucjp(test.args.s)
if err := checkFunc(test.want, got, err); err != nil {
tt.Errorf("error = %v", err)
}
})
}
}

func Test_encode(t *testing.T) {
type args struct {
r io.Reader
t transform.Transformer
}
type want struct {
want string
err error
}
type test struct {
name string
args args
want want
checkFunc func(want, string, error) error
beforeFunc func(args)
afterFunc func(args)
}
defaultCheckFunc := func(w want, got string, err error) error {
if !errors.Is(err, w.err) {
return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
}
if !reflect.DeepEqual(got, w.want) {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
}
return nil
}
tests := []test{
{
name: "success to encode string",
args: args{
r: strings.NewReader("こんにちは"),
t: japanese.EUCJP.NewEncoder(),
},
want: want{
want: "\xa4\xb3\xa4\xf3\xa4ˤ\xc1\xa4\xcf",
},
},
func() test {
err := errors.New("invalid reader")
return test{
name: "fail to encode string",
args: args{
r: iotest.ErrReader(err),
t: japanese.EUCJP.NewEncoder(),
},
want: want{
err: err,
},
}
}(),
}

for _, tc := range tests {
test := tc
t.Run(test.name, func(tt *testing.T) {
tt.Parallel()
defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
if test.beforeFunc != nil {
test.beforeFunc(test.args)
}
if test.afterFunc != nil {
defer test.afterFunc(test.args)
}
checkFunc := test.checkFunc
if test.checkFunc == nil {
checkFunc = defaultCheckFunc
}

got, err := encode(test.args.r, test.args.t)
if err := checkFunc(test.want, got, err); err != nil {
tt.Errorf("error = %v", err)
}
})
}
}
18 changes: 18 additions & 0 deletions internal/test/data/request/doc.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
//
// Copyright (C) 2019-2022 vdaas.org vald team <vald@vdaas.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// Package request provides functions to generate proto request for testing
package request
77 changes: 77 additions & 0 deletions internal/test/data/request/insert.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
//
// Copyright (C) 2019-2022 vdaas.org vald team <vald@vdaas.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package request

import (
"strconv"

"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/internal/test/data/vector"
)

type ObjectType int

const (
Uint8 ObjectType = iota
Float
)

func GenMultiInsertReq(t ObjectType, dist vector.Distribution, num int, dim int, cfg *payload.Insert_Config) (*payload.Insert_MultiRequest, error) {
var vecs [][]float32
var err error
switch t {
case Float:
vecs, err = vector.GenF32Vec(dist, num, dim)
case Uint8:
vecs, err = vector.GenUint8Vec(dist, num, dim)
}
if err != nil {
return nil, err
}

req := &payload.Insert_MultiRequest{
Requests: make([]*payload.Insert_Request, num),
}
for i, vec := range vecs {
req.Requests[i] = &payload.Insert_Request{
Vector: &payload.Object_Vector{
Id: "uuid-" + strconv.Itoa(i+1),
Vector: vec,
},
Config: cfg,
}
}

return req, nil
}

// generate MultiInsert request with the same vector
func GenSameVecMultiInsertReq(num int, vec []float32, cfg *payload.Insert_Config) *payload.Insert_MultiRequest {
req := &payload.Insert_MultiRequest{
Requests: make([]*payload.Insert_Request, num),
}
for i := 0; i < num; i++ {
req.Requests[i] = &payload.Insert_Request{
Vector: &payload.Object_Vector{
Id: "uuid-" + strconv.Itoa(i+1),
Vector: vec,
},
Config: cfg,
}
}

return req
}
Loading

0 comments on commit 626c706

Please sign in to comment.