diff --git a/hack/benchmark/assets/x1b/loader.go b/hack/benchmark/assets/x1b/loader.go index 0a5e9450f9..bb898be826 100644 --- a/hack/benchmark/assets/x1b/loader.go +++ b/hack/benchmark/assets/x1b/loader.go @@ -72,7 +72,7 @@ type ivecs struct { *file } -func doOpen(fname string, elementSize int) (f *file, err error) { +func openFile(fname string, elementSize int) (f *file, err error) { fp, err := os.Open(fname) if err != nil { return nil, err @@ -164,7 +164,7 @@ func (iv *ivecs) Load(i int) (interface{}, error) { } func NewUint8Vectors(fname string) (Uint8Vectors, error) { - f, err := doOpen(fname, 1) + f, err := openFile(fname, 1) if err != nil { return nil, err } @@ -172,7 +172,7 @@ func NewUint8Vectors(fname string) (Uint8Vectors, error) { } func NewFloatVectors(fname string) (FloatVectors, error) { - f, err := doOpen(fname, 4) + f, err := openFile(fname, 4) if err != nil { return nil, err } @@ -180,7 +180,7 @@ func NewFloatVectors(fname string) (FloatVectors, error) { } func NewInt32Vectors(fname string) (Int32Vectors, error) { - f, err := doOpen(fname, 4) + f, err := openFile(fname, 4) if err != nil { return nil, err } diff --git a/hack/benchmark/assets/x1b/loader_test.go b/hack/benchmark/assets/x1b/loader_test.go index f6319a95f4..1c81de40ce 100644 --- a/hack/benchmark/assets/x1b/loader_test.go +++ b/hack/benchmark/assets/x1b/loader_test.go @@ -21,7 +21,7 @@ import ( "github.com/vdaas/vald/internal/test/goleak" ) -func Test_doOpen(t *testing.T) { +func Test_openFile(t *testing.T) { t.Parallel() type args struct { fname string @@ -94,7 +94,7 @@ func Test_doOpen(t *testing.T) { checkFunc = defaultCheckFunc } - gotF, err := doOpen(test.args.fname, test.args.elementSize) + gotF, err := openFile(test.args.fname, test.args.elementSize) if err := checkFunc(test.want, gotF, err); err != nil { tt.Errorf("error = %v", err) } diff --git a/internal/file/file.go b/internal/file/file.go index 3afff9455b..4c3577cd76 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -96,7 +96,7 @@ func doMoveDir(ctx context.Context, src, dst string, rollback bool) (err error) if len(src) == 0 || len(dst) == 0 || src == dst { return nil } - exits, fi, err := doExists(src) + exits, fi, err := ExistsWithDetail(src) if !exits || fi == nil || !fi.IsDir() || err != nil { return errors.ErrDirectoryNotFound(err, src, fi) } @@ -105,7 +105,7 @@ func doMoveDir(ctx context.Context, src, dst string, rollback bool) (err error) if err != nil { log.Debug(errors.ErrFailedToRenameDir(err, src, dst, nil, nil)) var tmpPath string - exits, fi, err := doExists(dst) + exits, fi, err := ExistsWithDetail(dst) if exits && fi.IsDir() && err == nil { tmpPath = Join(filepath.Dir(dst), "tmp-"+strconv.FormatInt(fastime.UnixNanoNow(), 10)) _ = os.RemoveAll(tmpPath) @@ -131,7 +131,7 @@ func doMoveDir(ctx context.Context, src, dst string, rollback bool) (err error) } log.Debugf("directory %s successfully moved to tmp location %s", dst, tmpPath) } - exits, fi, err = doExists(src) + exits, fi, err = ExistsWithDetail(src) if exits && fi != nil && fi.IsDir() && err == nil { err = os.Rename(src, dst) if err != nil { @@ -223,7 +223,7 @@ func CopyFileWithPerm(ctx context.Context, src, dst string, perm fs.FileMode) (n } }() - exist, fi, err := doExists(src) + exist, fi, err := ExistsWithDetail(src) switch { case !exist, fi == nil, fi.Size() == 0, fi.IsDir(): return 0, errors.Wrap(err, errors.ErrFileNotFound(src).Error()) @@ -258,23 +258,23 @@ func CopyFileWithPerm(ctx context.Context, src, dst string, perm fs.FileMode) (n } func WriteFile(ctx context.Context, target string, r io.Reader, perm fs.FileMode) (n int64, err error) { - return doWriteFile(ctx, target, r, os.O_CREATE|os.O_WRONLY|os.O_SYNC, perm) + return writeFileWithContext(ctx, target, r, os.O_CREATE|os.O_WRONLY|os.O_SYNC, perm) } func OverWriteFile(ctx context.Context, target string, r io.Reader, perm fs.FileMode) (n int64, err error) { - return doWriteFile(ctx, target, r, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_SYNC, perm) + return writeFileWithContext(ctx, target, r, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_SYNC, perm) } func AppendFile(ctx context.Context, target string, r io.Reader, perm fs.FileMode) (n int64, err error) { - return doWriteFile(ctx, target, r, os.O_CREATE|os.O_APPEND|os.O_RDWR|os.O_SYNC, perm) + return writeFileWithContext(ctx, target, r, os.O_CREATE|os.O_APPEND|os.O_RDWR|os.O_SYNC, perm) } -func doWriteFile(ctx context.Context, target string, r io.Reader, flg int, perm fs.FileMode) (n int64, err error) { +func writeFileWithContext(ctx context.Context, target string, r io.Reader, flg int, perm fs.FileMode) (n int64, err error) { if len(target) == 0 || r == nil { return 0, nil } - exist, fi, err := doExists(target) + exist, fi, err := ExistsWithDetail(target) switch { case err == nil, exist, fi != nil && fi.Size() != 0, fi != nil && fi.IsDir(): err = errors.ErrFileAlreadyExists(target) @@ -359,13 +359,23 @@ func ReadFile(path string) (n []byte, err error) { // Exists returns file existence func Exists(path string) (e bool) { - e, _, _ = doExists(path) + e, _, _ = ExistsWithDetail(path) return e } -// ExistsWithDetail returns file existence +// ExistsWithDetail returns file existence with detailed information func ExistsWithDetail(path string) (e bool, fi fs.FileInfo, err error) { - return doExists(path) + fi, err = os.Stat(path) + if err != nil { + if os.IsExist(err) { + return true, fi, nil + } + if os.IsNotExist(err) { + return false, fi, err + } + return false, fi, err + } + return true, fi, nil } // MkdirAll creates directory like mkdir -p @@ -375,7 +385,7 @@ func MkdirAll(path string, perm fs.FileMode) (err error) { fi fs.FileInfo merr, rerr error ) - exist, fi, err = doExists(path) + exist, fi, err = ExistsWithDetail(path) if exist { if err == nil && fi != nil && fi.IsDir() { return nil @@ -447,24 +457,9 @@ func CreateTemp(baseDir string) (f *os.File, err error) { return nil, errors.ErrFailedToCreateFile(err, path, nil) } -// doExists returns file existence with detailed information -func doExists(path string) (exists bool, fi fs.FileInfo, err error) { - fi, err = os.Stat(path) - if err != nil { - if os.IsExist(err) { - return true, fi, nil - } - if os.IsNotExist(err) { - return false, fi, err - } - return false, fi, err - } - return true, fi, nil -} - // ListInDir returns file list in directory func ListInDir(path string) ([]string, error) { - exists, fi, err := doExists(path) + exists, fi, err := ExistsWithDetail(path) if !exists { return nil, err } @@ -484,7 +479,7 @@ func Join(paths ...string) (path string) { return "" } if len(paths) > 1 { - path = doJoin(paths...) + path = joinFilePaths(paths...) } else { path = replacer.Replace(paths[0]) } @@ -498,7 +493,7 @@ func Join(paths ...string) (path string) { log.Warn(err) return filepath.Clean(path) } - return filepath.Clean(doJoin(root, path)) + return filepath.Clean(joinFilePaths(root, path)) } var replacer = strings.NewReplacer( @@ -508,7 +503,7 @@ var replacer = strings.NewReplacer( string(os.PathSeparator), ) -func doJoin(paths ...string) (path string) { +func joinFilePaths(paths ...string) (path string) { for i, path := range paths { if path != "" { return replacer.Replace(strings.Join(paths[i:], string(os.PathSeparator))) diff --git a/internal/file/file_test.go b/internal/file/file_test.go index 512b9181d7..a6551f5dd7 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -484,86 +484,6 @@ func TestExistsWithDetail(t *testing.T) { } } -func Test_doExists(t *testing.T) { - type args struct { - path string - } - type want struct { - wantExists bool - wantFi fs.FileInfo - err error - } - type test struct { - name string - args args - want want - checkFunc func(want, bool, fs.FileInfo, error) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want, gotExists bool, gotFi fs.FileInfo, err error) error { - if !errors.Is(err, w.err) { - return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) - } - if !reflect.DeepEqual(gotExists, w.wantExists) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotExists, w.wantExists) - } - if !reflect.DeepEqual(gotFi, w.wantFi) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotFi, w.wantFi) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - path: "", - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - path: "", - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - - gotExists, gotFi, err := doExists(test.args.path) - if err := test.checkFunc(test.want, gotExists, gotFi, err); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - func TestListInDir(t *testing.T) { type args struct { path string @@ -1122,7 +1042,7 @@ func TestJoin(t *testing.T) { } } -func Test_doJoin(t *testing.T) { +func Test_joinFilePaths(t *testing.T) { type args struct { paths []string } @@ -1187,7 +1107,7 @@ func Test_doJoin(t *testing.T) { checkFunc = defaultCheckFunc } - gotPath := doJoin(test.args.paths...) + gotPath := joinFilePaths(test.args.paths...) if err := checkFunc(test.want, gotPath); err != nil { tt.Errorf("error = %v", err) } @@ -1606,7 +1526,7 @@ func TestAppendFile(t *testing.T) { } } -func Test_doWriteFile(t *testing.T) { +func Test_writeFileWithContext(t *testing.T) { type args struct { ctx context.Context target string @@ -1687,7 +1607,7 @@ func Test_doWriteFile(t *testing.T) { checkFunc = defaultCheckFunc } - gotN, err := doWriteFile(test.args.ctx, test.args.target, test.args.r, test.args.flg, test.args.perm) + gotN, err := writeFileWithContext(test.args.ctx, test.args.target, test.args.r, test.args.flg, test.args.perm) if err := checkFunc(test.want, gotN, err); err != nil { tt.Errorf("error = %v", err) } diff --git a/internal/net/grpc/client.go b/internal/net/grpc/client.go index 83e06a9445..2af38ee697 100644 --- a/internal/net/grpc/client.go +++ b/internal/net/grpc/client.go @@ -326,7 +326,7 @@ func (g *gRPCClient) Range(ctx context.Context, case <-ctx.Done(): return false default: - g.doConnect(ssctx, p, addr, true, func(ictx context.Context, + g.connectWithBackoff(ssctx, p, addr, true, func(ictx context.Context, conn *ClientConn, copts ...CallOption, ) (interface{}, error) { return nil, f(ictx, addr, conn, copts...) @@ -360,7 +360,7 @@ func (g *gRPCClient) RangeConcurrent(ctx context.Context, case <-egctx.Done(): return nil default: - g.doConnect(ssctx, p, addr, true, func(ictx context.Context, + g.connectWithBackoff(ssctx, p, addr, true, func(ictx context.Context, conn *ClientConn, copts ...CallOption, ) (interface{}, error) { return nil, f(ictx, addr, conn, copts...) @@ -403,7 +403,7 @@ func (g *gRPCClient) OrderedRange(ctx context.Context, span.End() } }() - g.doConnect(ssctx, p, addr, true, func(ictx context.Context, + g.connectWithBackoff(ssctx, p, addr, true, func(ictx context.Context, conn *ClientConn, copts ...CallOption, ) (interface{}, error) { return nil, f(ictx, addr, conn, copts...) @@ -447,7 +447,7 @@ func (g *gRPCClient) OrderedRangeConcurrent(ctx context.Context, case <-egctx.Done(): return nil default: - g.doConnect(ssctx, p, addr, true, func(ictx context.Context, + g.connectWithBackoff(ssctx, p, addr, true, func(ictx context.Context, conn *ClientConn, copts ...CallOption, ) (interface{}, error) { return nil, f(ictx, addr, conn, copts...) @@ -475,7 +475,7 @@ func (g *gRPCClient) RoundRobin(ctx context.Context, f func(ctx context.Context, sctx = backoff.WithBackoffName(ctx, boName) } do := func(ctx context.Context, p pool.Conn, addr string, f func(ctx context.Context, conn *ClientConn, copts ...CallOption) (interface{}, error)) (r interface{}, ret bool, err error) { - r, err = g.doConnect(ctx, p, addr, false, f) + r, err = g.connectWithBackoff(ctx, p, addr, false, f) if err != nil { st, ok := status.FromError(err) if !ok || st == nil { @@ -555,10 +555,10 @@ func (g *gRPCClient) Do(ctx context.Context, addr string, log.Warnf("gRPCClient.Do operation failed, grpc pool connection for %s is invalid,\terror: %v", addr, err) return nil, err } - return g.doConnect(sctx, p, addr, true, f) + return g.connectWithBackoff(sctx, p, addr, true, f) } -func (g *gRPCClient) doConnect(ctx context.Context, p pool.Conn, addr string, enableBackoff bool, +func (g *gRPCClient) connectWithBackoff(ctx context.Context, p pool.Conn, addr string, enableBackoff bool, f func(ctx context.Context, conn *ClientConn, copts ...CallOption) (interface{}, error), ) (data interface{}, err error) { diff --git a/internal/net/grpc/client_test.go b/internal/net/grpc/client_test.go index c2b46b99e3..29012a5d6b 100644 --- a/internal/net/grpc/client_test.go +++ b/internal/net/grpc/client_test.go @@ -1495,7 +1495,7 @@ func Test_gRPCClient_Do(t *testing.T) { } } -func Test_gRPCClient_doConnect(t *testing.T) { +func Test_gRPCClient_connectWithBackoff(t *testing.T) { type args struct { ctx context.Context p pool.Conn @@ -1693,7 +1693,7 @@ func Test_gRPCClient_doConnect(t *testing.T) { stopMonitor: test.fields.stopMonitor, } - gotData, err := g.doConnect(test.args.ctx, test.args.p, test.args.addr, test.args.enableBackoff, test.args.f) + gotData, err := g.connectWithBackoff(test.args.ctx, test.args.p, test.args.addr, test.args.enableBackoff, test.args.f) if err := checkFunc(test.want, gotData, err); err != nil { tt.Errorf("error = %v", err) } diff --git a/internal/net/grpc/pool/pool.go b/internal/net/grpc/pool/pool.go index 679a0ac588..ed70c4ef54 100644 --- a/internal/net/grpc/pool/pool.go +++ b/internal/net/grpc/pool/pool.go @@ -158,11 +158,11 @@ func (p *pool) Connect(ctx context.Context) (c Conn, err error) { } if p.isIP || !p.resolveDNS { - return p.doConnect(ctx) + return p.reconnectUnhealthy(ctx) } ips, err := p.lookupIPAddr(ctx) if err != nil { - return p.doConnect(ctx) + return p.reconnectUnhealthy(ctx) } p.reconnectHash = strings.Join(ips, "-") @@ -213,7 +213,7 @@ func (p *pool) load(idx int) (pc *poolConn, ok bool) { return } -func (p *pool) doConnect(ctx context.Context) (c Conn, err error) { +func (p *pool) reconnectUnhealthy(ctx context.Context) (c Conn, err error) { p.reconnectHash = p.host failCnt := uint64(0) for i := range p.pool { @@ -357,10 +357,10 @@ func (p *pool) Do(f func(conn *ClientConn) error) error { } func (p *pool) Get() (*ClientConn, bool) { - return p.doGet(p.Len()) + return p.getHelthyConn(p.Len()) } -func (p *pool) doGet(retry uint64) (*ClientConn, bool) { +func (p *pool) getHelthyConn(retry uint64) (*ClientConn, bool) { if retry <= 0 || retry > math.MaxUint64-p.Len() || p.Len() <= 0 { log.Warnf("failed to find grpc pool connection for %s", p.addr) if p.isIP { @@ -378,7 +378,7 @@ func (p *pool) doGet(retry uint64) (*ClientConn, bool) { } } retry-- - return p.doGet(retry) + return p.getHelthyConn(retry) } func (p *pool) Len() uint64 { @@ -440,7 +440,7 @@ func (p *pool) Reconnect(ctx context.Context, force bool) (c Conn, err error) { if p.reconnectHash == "" { log.Debugf("connection history for %s not found starting first connection phase", p.addr) if p.isIP || !p.resolveDNS { - return p.doConnect(ctx) + return p.reconnectUnhealthy(ctx) } return p.Connect(ctx) } @@ -451,7 +451,7 @@ func (p *pool) Reconnect(ctx context.Context, force bool) (c Conn, err error) { if p.isIP { return nil, errors.ErrInvalidGRPCClientConn(p.addr) } - return p.doConnect(ctx) + return p.reconnectUnhealthy(ctx) } return p, nil } diff --git a/internal/net/grpc/pool/pool_test.go b/internal/net/grpc/pool/pool_test.go index fa91ab7408..0bac534651 100644 --- a/internal/net/grpc/pool/pool_test.go +++ b/internal/net/grpc/pool/pool_test.go @@ -413,7 +413,7 @@ func Test_pool_load(t *testing.T) { } } -func Test_pool_doConnect(t *testing.T) { +func Test_pool_reconnectUnhealthy(t *testing.T) { t.Parallel() type args struct { ctx context.Context @@ -555,7 +555,7 @@ func Test_pool_doConnect(t *testing.T) { reconnectHash: test.fields.reconnectHash, } - gotC, err := p.doConnect(test.args.ctx) + gotC, err := p.reconnectUnhealthy(test.args.ctx) if err := checkFunc(test.want, gotC, err); err != nil { tt.Errorf("error = %v", err) } @@ -1284,7 +1284,7 @@ func Test_pool_Get(t *testing.T) { } } -func Test_pool_doGet(t *testing.T) { +func Test_pool_getHelthyConn(t *testing.T) { t.Parallel() type args struct { retry uint64 @@ -1426,7 +1426,7 @@ func Test_pool_doGet(t *testing.T) { reconnectHash: test.fields.reconnectHash, } - got, got1 := p.doGet(test.args.retry) + got, got1 := p.getHelthyConn(test.args.retry) if err := checkFunc(test.want, got, got1); err != nil { tt.Errorf("error = %v", err) } diff --git a/internal/test/data/vector/gen.go b/internal/test/data/vector/gen.go index 1aaa7f89e1..f7c461f822 100644 --- a/internal/test/data/vector/gen.go +++ b/internal/test/data/vector/gen.go @@ -63,8 +63,8 @@ func Uint8VectorGenerator(d Distribution) (Uint8VectorGeneratorFunc, error) { } } -// doGenFloat32Vec return n float32 vectors with dim dimension. -func doGenFloat32Vec(n, dim int, gen func() float32) (ret [][]float32) { +// genF32Slice return n float32 vectors with dim dimension. +func genF32Slice(n, dim int, gen func() float32) (ret [][]float32) { ret = make([][]float32, 0, n) for i := 0; i < n; i++ { @@ -79,18 +79,18 @@ func doGenFloat32Vec(n, dim int, gen func() float32) (ret [][]float32) { // UniformDistributedFloat32VectorGenerator returns n float32 vectors with dim dimension and their values under Uniform distribution func UniformDistributedFloat32VectorGenerator(n, dim int) [][]float32 { - return doGenFloat32Vec(n, dim, rand.Float32) + return genF32Slice(n, dim, rand.Float32) } // GaussianDistributedFloat32VectorGenerator returns n float32 vectors with dim dimension and their values under Gaussian distribution func GaussianDistributedFloat32VectorGenerator(n, dim int) [][]float32 { - return doGenFloat32Vec(n, dim, func() float32 { + return genF32Slice(n, dim, func() float32 { return float32(rand.NormFloat64()) }) } -// doGenUint8Vec return n uint8 vectors with dim dimension -func doGenUint8Vec(n, dim int, gen func() uint8) (ret [][]uint8) { +// genUint8Slice return n uint8 vectors with dim dimension +func genUint8Slice(n, dim int, gen func() uint8) (ret [][]uint8) { ret = make([][]uint8, 0, n) for i := 0; i < n; i++ { @@ -105,7 +105,7 @@ func doGenUint8Vec(n, dim int, gen func() uint8) (ret [][]uint8) { // UniformDistributedUint8VectorGenerator returns n uint8 vectors with dim dimension and their values under Uniform distribution func UniformDistributedUint8VectorGenerator(n, dim int) [][]uint8 { - return doGenUint8Vec(n, dim, func() uint8 { + return genUint8Slice(n, dim, func() uint8 { return uint8(irand.LimitedUint32(math.MaxUint8)) }) } @@ -113,7 +113,7 @@ func UniformDistributedUint8VectorGenerator(n, dim int) [][]uint8 { // GaussianDistributedUint8VectorGenerator returns n uint8 vectors with dim dimension and their values under Gaussian distribution func GaussianDistributedUint8VectorGenerator(n, dim int) [][]uint8 { // NOTE: The boundary test is the main purpose for refactoring. Now, passing this function is dependent on the seed of the random generator. We should fix the randomness of the passing test. - return doGenUint8Vec(n, dim, func() uint8 { + return genUint8Slice(n, dim, func() uint8 { val := rand.NormFloat64()*gaussianSigma + gaussianMean if val < 0 { return 0 diff --git a/internal/test/data/vector/gen_test.go b/internal/test/data/vector/gen_test.go index 9f7a372bdc..1014e368d9 100644 --- a/internal/test/data/vector/gen_test.go +++ b/internal/test/data/vector/gen_test.go @@ -208,7 +208,7 @@ func TestUint8VectorGenerator(t *testing.T) { } } -func Test_doGenFloat32Vec(t *testing.T) { +func Test_genF32Slice(t *testing.T) { type args struct { n int dim int @@ -279,7 +279,7 @@ func Test_doGenFloat32Vec(t *testing.T) { checkFunc = defaultCheckFunc } - gotRet := doGenFloat32Vec(test.args.n, test.args.dim, test.args.gen) + gotRet := genF32Slice(test.args.n, test.args.dim, test.args.gen) if err := checkFunc(test.want, gotRet); err != nil { tt.Errorf("error = %v", err) } @@ -439,7 +439,7 @@ func TestGaussianDistributedFloat32VectorGenerator(t *testing.T) { } } -func Test_doGenUint8Vec(t *testing.T) { +func Test_genUint8Slice(t *testing.T) { type args struct { n int dim int @@ -510,7 +510,7 @@ func Test_doGenUint8Vec(t *testing.T) { checkFunc = defaultCheckFunc } - gotRet := doGenUint8Vec(test.args.n, test.args.dim, test.args.gen) + gotRet := genUint8Slice(test.args.n, test.args.dim, test.args.gen) if err := checkFunc(test.want, gotRet); err != nil { tt.Errorf("error = %v", err) }