diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a46b84bae5..ac5706abe5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: - name: Check out code. uses: actions/checkout@v1 with: - fetch-depth: 1 + fetch-depth: 10 - name: Run tests for cmd packages / tparse run: | make test/cmd/tparse | tee tparse.txt || cat tparse.txt diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index 042e622cb3..7917212dd0 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -1266,6 +1266,25 @@ agent: # @schema {"name": "agent.ngt.enable_in_memory_mode", "type": "boolean"} # agent.ngt.enable_in_memory_mode -- in-memory mode enabled enable_in_memory_mode: true + # @schema {"name": "agent.ngt.default_pool_size", "type": "integer"} + # agent.ngt.default_pool_size -- default create index batch pool size + default_pool_size: 10000 + # @schema {"name": "agent.ngt.default_radius", "type": "number"} + # agent.ngt.default_radius -- default radius used for search + default_radius: -1.0 + # @schema {"name": "agent.ngt.default_epsilon", "type": "number"} + # agent.ngt.default_epsilon -- default epsilon used for search + default_epsilon: 0.01 + # @schema {"name": "agent.ngt.min_load_index_timeout", "type": "string"} + # agent.ngt.min_load_index_timeout -- minimum duration of load index timeout + min_load_index_timeout: 3m + # @schema {"name": "agent.ngt.max_load_index_timeout", "type": "string"} + # agent.ngt.max_load_index_timeout -- maximum duration of load index timeout + max_load_index_timeout: 10m + # @schema {"name": "agent.ngt.load_index_timeout_factor", "type": "string"} + # agent.ngt.load_index_timeout_factor -- a factor of load index timeout. + # timeout duration will be calculated by (index count to be loaded) * (factor). + load_index_timeout_factor: 1ms # @schema {"name": "agent.sidecar", "type": "object"} sidecar: # @schema {"name": "agent.sidecar.enabled", "type": "boolean"} @@ -1372,12 +1391,18 @@ agent: memory: 100Mi # @schema {"name": "agent.sidecar.config", "type": "object"} config: + # @schema {"name": "agent.sidecar.config.watch_enabled", "type": "boolean"} + # agent.sidecar.config.watch_enabled -- auto backup triggered by file changes is enabled + watch_enabled: true + # @schema {"name": "agent.sidecar.config.auto_backup_enabled", "type": "boolean"} + # agent.sidecar.config.auto_backup_enabled -- auto backup triggered by timer is enabled + auto_backup_enabled: true # @schema {"name": "agent.sidecar.config.auto_backup_duration", "type": "string"} # agent.sidecar.config.auto_backup_duration -- auto backup duration - auto_backup_duration: 10m + auto_backup_duration: 24h # @schema {"name": "agent.sidecar.config.post_stop_timeout", "type": "string"} - # agent.sidecar.config.post_stop_timeout -- timeout duration for file changing during post stop - post_stop_timeout: 20s + # agent.sidecar.config.post_stop_timeout -- timeout for observing file changes during post stop + post_stop_timeout: 2m # @schema {"name": "agent.sidecar.config.filename", "type": "string"} # agent.sidecar.config.filename -- backup filename filename: _MY_POD_NAME_ diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile index bfb0723705..f09b7abf84 100644 --- a/dockers/agent/core/ngt/Dockerfile +++ b/dockers/agent/core/ngt/Dockerfile @@ -19,6 +19,7 @@ FROM vdaas/vald-base:latest AS builder ENV ORG vdaas ENV REPO vald ENV PKG agent/core/ngt +ENV PKG_INTERNAL agent/internal ENV APP_NAME ngt WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -30,6 +31,9 @@ COPY apis/grpc . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG} COPY pkg/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG_INTERNAL} +COPY pkg/${PKG_INTERNAL} . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile index 6ee2449d9f..f72d93b8fe 100644 --- a/dockers/agent/sidecar/Dockerfile +++ b/dockers/agent/sidecar/Dockerfile @@ -19,6 +19,7 @@ FROM vdaas/vald-base:latest AS builder ENV ORG vdaas ENV REPO vald ENV PKG agent/sidecar +ENV PKG_INTERNAL agent/internal ENV APP_NAME sidecar WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -30,6 +31,9 @@ COPY apis/grpc . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG} COPY pkg/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG_INTERNAL} +COPY pkg/${PKG_INTERNAL} . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . diff --git a/internal/compress/compress.go b/internal/compress/compress.go index 99c9c0eaea..20e90b46c9 100644 --- a/internal/compress/compress.go +++ b/internal/compress/compress.go @@ -22,6 +22,6 @@ import "io" type Compressor interface { CompressVector(vector []float32) (bytes []byte, err error) DecompressVector(bytes []byte) (vector []float32, err error) - Reader(src io.Reader) (io.Reader, error) + Reader(src io.ReadCloser) (io.ReadCloser, error) Writer(dst io.WriteCloser) (io.WriteCloser, error) } diff --git a/internal/compress/gob.go b/internal/compress/gob.go index 85b954ad51..3afdfe8968 100644 --- a/internal/compress/gob.go +++ b/internal/compress/gob.go @@ -60,7 +60,7 @@ func (g *gobCompressor) DecompressVector(bs []byte) ([]float32, error) { return vector, nil } -func (g *gobCompressor) Reader(src io.Reader) (io.Reader, error) { +func (g *gobCompressor) Reader(src io.ReadCloser) (io.ReadCloser, error) { return &gobReader{ src: src, decoder: gob.NewDecoder(src), @@ -75,7 +75,7 @@ func (g *gobCompressor) Writer(dst io.WriteCloser) (io.WriteCloser, error) { } type gobReader struct { - src io.Reader + src io.ReadCloser decoder *gob.Decoder } @@ -88,6 +88,10 @@ func (gr *gobReader) Read(p []byte) (n int, err error) { return len(p), nil } +func (gr *gobReader) Close() error { + return gr.src.Close() +} + type gobWriter struct { dst io.WriteCloser encoder *gob.Encoder diff --git a/internal/compress/gob_test.go b/internal/compress/gob_test.go index 70a48e937d..d9f5c79b81 100644 --- a/internal/compress/gob_test.go +++ b/internal/compress/gob_test.go @@ -121,7 +121,7 @@ func TestNewGob(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -197,7 +197,7 @@ func Test_gobCompressor_CompressVector(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -274,7 +274,7 @@ func Test_gobCompressor_DecompressVector(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -297,10 +297,10 @@ func Test_gobCompressor_DecompressVector(t *testing.T) { func Test_gobCompressor_Reader(t *testing.T) { type args struct { - src io.Reader + src io.ReadCloser } type want struct { - want io.Reader + want io.ReadCloser err error } type test struct { @@ -308,11 +308,11 @@ func Test_gobCompressor_Reader(t *testing.T) { args args g *gobCompressor want want - checkFunc func(want, io.Reader, error) error + checkFunc func(want, io.ReadCloser, error) error beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, got io.Reader, err error) error { + defaultCheckFunc := func(w want, got io.ReadCloser, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got error = %v, want %v", err, w.err) } @@ -351,7 +351,7 @@ func Test_gobCompressor_Reader(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -428,7 +428,7 @@ func Test_gobCompressor_Writer(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -454,7 +454,7 @@ func Test_gobReader_Read(t *testing.T) { p []byte } type fields struct { - src io.Reader + src io.ReadCloser decoder *gob.Decoder } type want struct { @@ -517,7 +517,7 @@ func Test_gobReader_Read(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -541,6 +541,84 @@ func Test_gobReader_Read(t *testing.T) { } } +func Test_gobReader_Close(t *testing.T) { + type fields struct { + src io.ReadCloser + decoder *gob.Decoder + } + type want struct { + err error + } + type test struct { + name string + fields fields + want want + checkFunc func(want, error) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + src: nil, + decoder: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + src: nil, + decoder: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + gr := &gobReader{ + src: test.fields.src, + decoder: test.fields.decoder, + } + + err := gr.Close() + if err := test.checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + func Test_gobWriter_Write(t *testing.T) { type args struct { p []byte @@ -609,7 +687,7 @@ func Test_gobWriter_Write(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -687,7 +765,7 @@ func Test_gobWriter_Close(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc() } diff --git a/internal/compress/gzip.go b/internal/compress/gzip.go index f1a825140b..0a4a08af3f 100644 --- a/internal/compress/gzip.go +++ b/internal/compress/gzip.go @@ -87,8 +87,16 @@ func (g *gzipCompressor) DecompressVector(bs []byte) ([]float32, error) { return vec, nil } -func (g *gzipCompressor) Reader(src io.Reader) (io.Reader, error) { - return gzip.NewReader(src) +func (g *gzipCompressor) Reader(src io.ReadCloser) (io.ReadCloser, error) { + r, err := gzip.NewReader(src) + if err != nil { + return nil, err + } + + return &gzipReader{ + src: src, + r: r, + }, nil } func (g *gzipCompressor) Writer(dst io.WriteCloser) (io.WriteCloser, error) { @@ -103,6 +111,24 @@ func (g *gzipCompressor) Writer(dst io.WriteCloser) (io.WriteCloser, error) { }, nil } +type gzipReader struct { + src io.ReadCloser + r io.ReadCloser +} + +func (g *gzipReader) Read(p []byte) (n int, err error) { + return g.r.Read(p) +} + +func (g *gzipReader) Close() (err error) { + err = g.r.Close() + if err != nil { + return errors.Wrap(g.src.Close(), err.Error()) + } + + return g.src.Close() +} + type gzipWriter struct { dst io.WriteCloser w io.WriteCloser diff --git a/internal/compress/gzip_test.go b/internal/compress/gzip_test.go index 63909a984d..5ad7b38a72 100644 --- a/internal/compress/gzip_test.go +++ b/internal/compress/gzip_test.go @@ -120,7 +120,7 @@ func TestNewGzip(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -208,7 +208,7 @@ func Test_gzipCompressor_CompressVector(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -300,7 +300,7 @@ func Test_gzipCompressor_DecompressVector(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -326,14 +326,14 @@ func Test_gzipCompressor_DecompressVector(t *testing.T) { func Test_gzipCompressor_Reader(t *testing.T) { type args struct { - src io.Reader + src io.ReadCloser } type fields struct { gobc Compressor compressionLevel int } type want struct { - want io.Reader + want io.ReadCloser err error } type test struct { @@ -341,11 +341,11 @@ func Test_gzipCompressor_Reader(t *testing.T) { args args fields fields want want - checkFunc func(want, io.Reader, error) error + checkFunc func(want, io.ReadCloser, error) error beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, got io.Reader, err error) error { + defaultCheckFunc := func(w want, got io.ReadCloser, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got error = %v, want %v", err, w.err) } @@ -392,7 +392,7 @@ func Test_gzipCompressor_Reader(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -484,7 +484,7 @@ func Test_gzipCompressor_Writer(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -508,6 +508,176 @@ func Test_gzipCompressor_Writer(t *testing.T) { } } +func Test_gzipReader_Read(t *testing.T) { + type args struct { + p []byte + } + type fields struct { + src io.ReadCloser + r io.ReadCloser + } + type want struct { + wantN int + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, int, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, gotN int, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(gotN, w.wantN) { + return errors.Errorf("got = %v, want %v", gotN, w.wantN) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + p: nil, + }, + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + p: nil, + }, + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + g := &gzipReader{ + src: test.fields.src, + r: test.fields.r, + } + + gotN, err := g.Read(test.args.p) + if err := test.checkFunc(test.want, gotN, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func Test_gzipReader_Close(t *testing.T) { + type fields struct { + src io.ReadCloser + r io.ReadCloser + } + type want struct { + err error + } + type test struct { + name string + fields fields + want want + checkFunc func(want, error) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + g := &gzipReader{ + src: test.fields.src, + r: test.fields.r, + } + + err := g.Close() + if err := test.checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + func Test_gzipWriter_Write(t *testing.T) { type args struct { p []byte @@ -576,7 +746,7 @@ func Test_gzipWriter_Write(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -654,7 +824,7 @@ func Test_gzipWriter_Close(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc() } diff --git a/internal/compress/lz4.go b/internal/compress/lz4.go index d7c956c611..666a67102f 100644 --- a/internal/compress/lz4.go +++ b/internal/compress/lz4.go @@ -87,8 +87,11 @@ func (l *lz4Compressor) DecompressVector(bs []byte) ([]float32, error) { return vec, nil } -func (l *lz4Compressor) Reader(src io.Reader) (io.Reader, error) { - return lz4.NewReader(src), nil +func (l *lz4Compressor) Reader(src io.ReadCloser) (io.ReadCloser, error) { + return &lz4Reader{ + src: src, + r: lz4.NewReader(src), + }, nil } func (l *lz4Compressor) Writer(dst io.WriteCloser) (io.WriteCloser, error) { @@ -98,6 +101,19 @@ func (l *lz4Compressor) Writer(dst io.WriteCloser) (io.WriteCloser, error) { }, nil } +type lz4Reader struct { + src io.ReadCloser + r io.Reader +} + +func (l *lz4Reader) Read(p []byte) (n int, err error) { + return l.r.Read(p) +} + +func (l *lz4Reader) Close() (err error) { + return l.src.Close() +} + type lz4Writer struct { dst io.WriteCloser w io.WriteCloser diff --git a/internal/compress/lz4_test.go b/internal/compress/lz4_test.go index 08b2fdc7a7..4aae8b9fbc 100644 --- a/internal/compress/lz4_test.go +++ b/internal/compress/lz4_test.go @@ -120,7 +120,7 @@ func TestNewLZ4(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -208,7 +208,7 @@ func Test_lz4Compressor_CompressVector(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -300,7 +300,7 @@ func Test_lz4Compressor_DecompressVector(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -326,14 +326,14 @@ func Test_lz4Compressor_DecompressVector(t *testing.T) { func Test_lz4Compressor_Reader(t *testing.T) { type args struct { - src io.Reader + src io.ReadCloser } type fields struct { gobc Compressor compressionLevel int } type want struct { - want io.Reader + want io.ReadCloser err error } type test struct { @@ -341,11 +341,11 @@ func Test_lz4Compressor_Reader(t *testing.T) { args args fields fields want want - checkFunc func(want, io.Reader, error) error + checkFunc func(want, io.ReadCloser, error) error beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, got io.Reader, err error) error { + defaultCheckFunc := func(w want, got io.ReadCloser, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got error = %v, want %v", err, w.err) } @@ -392,7 +392,7 @@ func Test_lz4Compressor_Reader(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -484,7 +484,7 @@ func Test_lz4Compressor_Writer(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -508,6 +508,176 @@ func Test_lz4Compressor_Writer(t *testing.T) { } } +func Test_lz4Reader_Read(t *testing.T) { + type args struct { + p []byte + } + type fields struct { + src io.ReadCloser + r io.Reader + } + type want struct { + wantN int + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, int, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, gotN int, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(gotN, w.wantN) { + return errors.Errorf("got = %v, want %v", gotN, w.wantN) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + p: nil, + }, + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + p: nil, + }, + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + l := &lz4Reader{ + src: test.fields.src, + r: test.fields.r, + } + + gotN, err := l.Read(test.args.p) + if err := test.checkFunc(test.want, gotN, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func Test_lz4Reader_Close(t *testing.T) { + type fields struct { + src io.ReadCloser + r io.Reader + } + type want struct { + err error + } + type test struct { + name string + fields fields + want want + checkFunc func(want, error) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + l := &lz4Reader{ + src: test.fields.src, + r: test.fields.r, + } + + err := l.Close() + if err := test.checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + func Test_lz4Writer_Write(t *testing.T) { type args struct { p []byte @@ -576,7 +746,7 @@ func Test_lz4Writer_Write(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -654,7 +824,7 @@ func Test_lz4Writer_Close(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc() } diff --git a/internal/compress/zstd.go b/internal/compress/zstd.go index c3cbdb9d7d..f672c57b6e 100644 --- a/internal/compress/zstd.go +++ b/internal/compress/zstd.go @@ -88,8 +88,16 @@ func (z *zstdCompressor) DecompressVector(bs []byte) ([]float32, error) { return vec, nil } -func (z *zstdCompressor) Reader(src io.Reader) (io.Reader, error) { - return zstd.NewReader(src) +func (z *zstdCompressor) Reader(src io.ReadCloser) (io.ReadCloser, error) { + r, err := zstd.NewReader(src) + if err != nil { + return nil, err + } + + return &zstdReader{ + src: src, + r: r, + }, nil } func (z *zstdCompressor) Writer(dst io.WriteCloser) (io.WriteCloser, error) { @@ -104,6 +112,19 @@ func (z *zstdCompressor) Writer(dst io.WriteCloser) (io.WriteCloser, error) { }, nil } +type zstdReader struct { + src io.ReadCloser + r io.Reader +} + +func (z *zstdReader) Read(p []byte) (n int, err error) { + return z.r.Read(p) +} + +func (z *zstdReader) Close() error { + return z.src.Close() +} + type zstdWriter struct { dst io.WriteCloser w io.WriteCloser diff --git a/internal/compress/zstd_test.go b/internal/compress/zstd_test.go index fda251b9c2..a0d4b56ba3 100644 --- a/internal/compress/zstd_test.go +++ b/internal/compress/zstd_test.go @@ -121,7 +121,7 @@ func TestNewZstd(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -209,7 +209,7 @@ func Test_zstdCompressor_CompressVector(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -301,7 +301,7 @@ func Test_zstdCompressor_DecompressVector(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -327,14 +327,14 @@ func Test_zstdCompressor_DecompressVector(t *testing.T) { func Test_zstdCompressor_Reader(t *testing.T) { type args struct { - src io.Reader + src io.ReadCloser } type fields struct { gobc Compressor eoptions []zstd.EOption } type want struct { - want io.Reader + want io.ReadCloser err error } type test struct { @@ -342,11 +342,11 @@ func Test_zstdCompressor_Reader(t *testing.T) { args args fields fields want want - checkFunc func(want, io.Reader, error) error + checkFunc func(want, io.ReadCloser, error) error beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, got io.Reader, err error) error { + defaultCheckFunc := func(w want, got io.ReadCloser, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got error = %v, want %v", err, w.err) } @@ -393,7 +393,7 @@ func Test_zstdCompressor_Reader(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -485,7 +485,7 @@ func Test_zstdCompressor_Writer(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -509,6 +509,176 @@ func Test_zstdCompressor_Writer(t *testing.T) { } } +func Test_zstdReader_Read(t *testing.T) { + type args struct { + p []byte + } + type fields struct { + src io.ReadCloser + r io.Reader + } + type want struct { + wantN int + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, int, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, gotN int, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(gotN, w.wantN) { + return errors.Errorf("got = %v, want %v", gotN, w.wantN) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + p: nil, + }, + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + p: nil, + }, + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + z := &zstdReader{ + src: test.fields.src, + r: test.fields.r, + } + + gotN, err := z.Read(test.args.p) + if err := test.checkFunc(test.want, gotN, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func Test_zstdReader_Close(t *testing.T) { + type fields struct { + src io.ReadCloser + r io.Reader + } + type want struct { + err error + } + type test struct { + name string + fields fields + want want + checkFunc func(want, error) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + src: nil, + r: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + z := &zstdReader{ + src: test.fields.src, + r: test.fields.r, + } + + err := z.Close() + if err := test.checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + func Test_zstdWriter_Write(t *testing.T) { type args struct { p []byte @@ -577,7 +747,7 @@ func Test_zstdWriter_Write(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -655,7 +825,7 @@ func Test_zstdWriter_Close(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc() } diff --git a/internal/config/config.go b/internal/config/config.go index 5671ef24e6..c2b9d18717 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,7 +25,7 @@ import ( "strings" "unsafe" - "github.com/vdaas/vald/internal/net/http/json" + "github.com/vdaas/vald/internal/encoding/json" yaml "gopkg.in/yaml.v2" ) diff --git a/internal/config/ngt.go b/internal/config/ngt.go index e9e7238c52..317f129906 100644 --- a/internal/config/ngt.go +++ b/internal/config/ngt.go @@ -57,6 +57,24 @@ type NGT struct { // EnableInMemoryMode enables on memory ngt indexing mode EnableInMemoryMode bool `yaml:"enable_in_memory_mode" json:"enable_in_memory_mode"` + + // DefaultPoolSize represent default create index batch pool size + DefaultPoolSize uint32 `yaml:"default_pool_size" json:"default_pool_size"` + + // DefaultRadius represent default radius used for search + DefaultRadius float32 `yaml:"default_radius" json:"default_radius"` + + // DefaultEpsilon represent default epsilon used for search + DefaultEpsilon float32 `yaml:"default_epsilon" json:"default_epsilon"` + + // MinLoadIndexTimeout represents minimum duration of load index timeout + MinLoadIndexTimeout string `yaml:"min_load_index_timeout" json:"min_load_index_timeout"` + + // MaxLoadIndexTimeout represents maximum duration of load index timeout + MaxLoadIndexTimeout string `yaml:"max_load_index_timeout" json:"max_load_index_timeout"` + + // LoadIndexTimeoutFactor represents a factor of load index timeout + LoadIndexTimeoutFactor string `yaml:"load_index_timeout_factor" json:"load_index_timeout_factor"` } // Bind returns NGT object whose some string value is filed value or environment value. @@ -68,5 +86,8 @@ func (n *NGT) Bind() *NGT { n.AutoIndexDurationLimit = GetActualValue(n.AutoIndexDurationLimit) n.AutoSaveIndexDuration = GetActualValue(n.AutoSaveIndexDuration) n.InitialDelayMaxDuration = GetActualValue(n.InitialDelayMaxDuration) + n.MinLoadIndexTimeout = GetActualValue(n.MinLoadIndexTimeout) + n.MaxLoadIndexTimeout = GetActualValue(n.MaxLoadIndexTimeout) + n.LoadIndexTimeoutFactor = GetActualValue(n.LoadIndexTimeoutFactor) return n } diff --git a/internal/config/sidecar.go b/internal/config/sidecar.go index d474e1d657..95ea5ecf58 100644 --- a/internal/config/sidecar.go +++ b/internal/config/sidecar.go @@ -24,6 +24,12 @@ type AgentSidecar struct { // WatchDir represents watch target directory for backup WatchDir string `yaml:"watch_dir" json:"watch_dir"` + // WatchEnabled represent auto backup triggered by file changes is enabled or not + WatchEnabled bool `yaml:"watch_enabled" json:"watch_enabled"` + + // AutoBackupEnabled represent auto backup triggered by timer is enabled or not + AutoBackupEnabled bool `yaml:"auto_backup_enabled" json:"auto_backup_enabled"` + // AutoBackupDuration represent checking loop duration for auto backup execution AutoBackupDuration string `yaml:"auto_backup_duration" json:"auto_backup_duration"` diff --git a/internal/core/ngt/option.go b/internal/core/ngt/option.go index b00b11efa3..f675bcae47 100644 --- a/internal/core/ngt/option.go +++ b/internal/core/ngt/option.go @@ -32,12 +32,16 @@ import ( type Option func(*ngt) error var ( + DefaultPoolSize = uint32(10000) + DefaultRadius = float32(-1.0) + DefaultEpsilon = float32(0.01) + defaultOpts = []Option{ WithIndexPath("/tmp/ngt-" + string(fastime.FormattedNow())), WithDimension(0), - WithDefaultRadius(-1.0), - WithDefaultEpsilon(0.01), - WithDefaultPoolSize(1), + WithDefaultRadius(DefaultRadius), + WithDefaultEpsilon(DefaultEpsilon), + WithDefaultPoolSize(DefaultPoolSize), WithCreationEdgeSize(10), WithSearchEdgeSize(40), WithObjectType(Float), diff --git a/internal/encoding/json/json.go b/internal/encoding/json/json.go new file mode 100644 index 0000000000..66d07c0069 --- /dev/null +++ b/internal/encoding/json/json.go @@ -0,0 +1,43 @@ +// +// Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package json + +import ( + "io" + + jsoniter "github.com/json-iterator/go" +) + +func Encode(w io.Writer, data interface{}) (err error) { + return jsoniter.NewEncoder(w).Encode(data) +} + +func Decode(r io.Reader, data interface{}) (err error) { + return jsoniter.NewDecoder(r).Decode(data) +} + +func Unmarshal(data []byte, i interface{}) error { + return jsoniter.Unmarshal(data, i) +} + +func Marshal(data interface{}) ([]byte, error) { + return jsoniter.Marshal(data) +} + +func MarshalIndent(data interface{}, pref, ind string) ([]byte, error) { + return jsoniter.MarshalIndent(data, pref, ind) +} diff --git a/internal/encoding/json/json_test.go b/internal/encoding/json/json_test.go new file mode 100644 index 0000000000..92c2ce8cfb --- /dev/null +++ b/internal/encoding/json/json_test.go @@ -0,0 +1,381 @@ +// +// Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package json + +import ( + "bytes" + "io" + "reflect" + "testing" + + "github.com/vdaas/vald/internal/errors" + "go.uber.org/goleak" +) + +func TestEncode(t *testing.T) { + type args struct { + w io.Writer + data interface{} + } + + type test struct { + name string + args args + checkFunc func(err error) error + } + + tests := []test{ + func() test { + buf := new(bytes.Buffer) + data := map[string]string{ + "name": "vald", + } + + return test{ + name: "returns nil", + args: args{ + w: buf, + data: data, + }, + checkFunc: func(err error) error { + if err != nil { + return errors.Errorf("err not equals. want: %v, got: %v", nil, err) + } + + if got, want := buf.String(), "{\"name\":\"vald\"}\n"; got != want { + return errors.Errorf("output data not equals. want: %v, got: %v", want, got) + } + + return nil + }, + } + }(), + + { + name: "returns error when type is invalid", + args: args{ + w: new(bytes.Buffer), + data: make(chan struct{}), + }, + checkFunc: func(err error) error { + if err == nil { + return errors.New("err is nil") + } + return nil + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := Encode(tt.args.w, tt.args.data) + if err := tt.checkFunc(err); err != nil { + t.Error(err) + } + }) + } +} + +func TestDecode(t *testing.T) { + type args struct { + r io.Reader + data map[string]string + } + + type test struct { + name string + args args + checkFunc func(err error, data map[string]string) error + } + + tests := []test{ + func() test { + buf := new(bytes.Buffer) + buf.WriteString(`{"name":"vald"}`) + + return test{ + name: "returns nil", + args: args{ + r: buf, + data: make(map[string]string, 1), + }, + checkFunc: func(err error, data map[string]string) error { + if err != nil { + return errors.Errorf("err not equals. want: %v, got: %v", nil, err) + } + + if got, want := data, map[string]string{ + "name": "vald", + }; !reflect.DeepEqual(got, want) { + return errors.Errorf("read data not equals. want: %v, got: %v", want, got) + } + + return nil + }, + } + }(), + + func() test { + buf := new(bytes.Buffer) + buf.WriteString(`1`) + + wantData := make(map[string]string, 1) + + return test{ + name: "returns error when type is invalid", + args: args{ + r: buf, + data: wantData, + }, + checkFunc: func(err error, data map[string]string) error { + if err == nil { + return errors.New("err is nil") + } + + if !reflect.DeepEqual(data, wantData) { + return errors.Errorf("data not equals. want: %v, got: %v", wantData, data) + } + + return nil + }, + } + }(), + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := Decode(tt.args.r, &tt.args.data) + if err := tt.checkFunc(err, tt.args.data); err != nil { + t.Error(err) + } + }) + } +} + +func TestMarshalIndent(t *testing.T) { + type args struct { + data interface{} + pref string + ind string + } + + type test struct { + name string + args args + checkFunc func(data []byte, err error) error + } + + tests := []test{ + { + name: "returns data and nil", + args: args{ + data: map[string]string{ + "name": "vald", + }, + pref: "", + ind: "", + }, + checkFunc: func(data []byte, err error) error { + if err != nil { + return errors.Errorf("err not equals. want: %v, got: %v", nil, err) + } + + if got, want := data, []byte(`{"name":"vald"}`); !reflect.DeepEqual(got, want) { + return errors.Errorf("data not equals. want: %v, got: %v", string(got), string(want)) + } + + return nil + }, + }, + + { + name: "returns error when type is invalid", + args: args{ + data: make(chan struct{}), + pref: "", + ind: "", + }, + checkFunc: func(data []byte, err error) error { + if err == nil { + return errors.New("err is nil") + } + + if len(data) != 0 { + return errors.New("data is not empty") + } + return nil + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := MarshalIndent(tt.args.data, tt.args.pref, tt.args.ind) + if err := tt.checkFunc(data, err); err != nil { + t.Error(err) + } + }) + } +} + +func TestUnmarshal(t *testing.T) { + type args struct { + data []byte + i interface{} + } + type want struct { + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + data: nil, + i: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + data: nil, + i: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + err := Unmarshal(test.args.data, test.args.i) + if err := test.checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func TestMarshal(t *testing.T) { + type args struct { + data interface{} + } + type want struct { + want []byte + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, []byte, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got []byte, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got = %v, want %v", got, w.want) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + data: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + data: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got, err := Marshal(test.args.data) + if err := test.checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} diff --git a/internal/errors/file.go b/internal/errors/file.go index e0a77cb18e..267a6a0e5a 100644 --- a/internal/errors/file.go +++ b/internal/errors/file.go @@ -19,4 +19,10 @@ package errors var ( ErrWatchDirNotFound = New("fs watcher watch dir not found") + + ErrFileAlreadyExists = func(path string) error { + return Errorf("file already exists: %s", path) + } + + ErrPathNotSpecified = New("the path is not specified") ) diff --git a/internal/errors/ngt.go b/internal/errors/ngt.go index fdda5277fd..ca3002c85f 100644 --- a/internal/errors/ngt.go +++ b/internal/errors/ngt.go @@ -26,6 +26,8 @@ var ( ErrIndexNotFound = New("index file not found") + ErrIndexLoadTimeout = New("index load timeout") + ErrInvalidDimensionSize = func(current, limit int) error { if limit == 0 { return Errorf("dimension size %d is invalid, the supporting dimension size must be bigger than 2", current) diff --git a/internal/file/file.go b/internal/file/file.go index 359b0f720c..21d0e849a6 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -20,14 +20,16 @@ package file import ( "os" "path/filepath" + + "github.com/vdaas/vald/internal/errors" ) // Open opens the file with the given path, flag and permission. // If the folder does not exists, create the folder. // If the file does not exist, create the file. -func Open(path string, flg int, perm os.FileMode) *os.File { +func Open(path string, flg int, perm os.FileMode) (*os.File, error) { if path == "" { - return nil + return nil, errors.ErrPathNotSpecified } var err error @@ -36,25 +38,27 @@ func Open(path string, flg int, perm os.FileMode) *os.File { if _, err = os.Stat(filepath.Dir(path)); err != nil { err = os.MkdirAll(filepath.Dir(path), perm) if err != nil { - return nil + return nil, err } } + file, err = os.Create(path) if err != nil { - return nil + return nil, err } - err = file.Close() - if err != nil { - return nil + if file != nil { + err = file.Close() + if err != nil { + return nil, err + } } } file, err = os.OpenFile(path, flg, perm) - if err != nil { - return nil + return nil, err } - return file + return file, nil } diff --git a/internal/file/file_test.go b/internal/file/file_test.go index 5750a058e8..76a24732fb 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -33,17 +33,22 @@ func TestOpen(t *testing.T) { } type want struct { want *os.File + err error } type test struct { name string args args want want - checkFunc func(want, *os.File) error + checkFunc func(want, *os.File, error) error beforeFunc func(*testing.T, args) afterFunc func(*testing.T, args) } - defaultCheckFunc := func(w want, got *os.File) error { + defaultCheckFunc := func(w want, got *os.File, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if w.want == nil { if got != nil { return errors.New("got is not nil") @@ -63,14 +68,15 @@ func TestOpen(t *testing.T) { flg: os.O_CREATE, perm: os.ModePerm, }, - checkFunc: func(_ want, got *os.File) error { + checkFunc: func(_ want, got *os.File, gotErr error) error { file, err := os.OpenFile("test/data", os.O_CREATE, os.ModePerm) if err != nil { return err } return defaultCheckFunc(want{ want: file, - }, got) + err: nil, + }, got, gotErr) }, afterFunc: func(t *testing.T, _ args) { t.Helper() @@ -87,14 +93,15 @@ func TestOpen(t *testing.T) { flg: os.O_CREATE, perm: os.ModePerm, }, - checkFunc: func(_ want, got *os.File) error { + checkFunc: func(_ want, got *os.File, gotErr error) error { file, err := os.OpenFile("test/test/data", os.O_CREATE, os.ModePerm) if err != nil { return err } return defaultCheckFunc(want{ want: file, - }, got) + err: nil, + }, got, gotErr) }, afterFunc: func(t *testing.T, _ args) { t.Helper() @@ -116,17 +123,19 @@ func TestOpen(t *testing.T) { f, _ := os.OpenFile("file.go", os.O_RDONLY, os.ModePerm) return f }(), + err: nil, }, }, { - name: "returns nil when path is empty", + name: "returns (nil, error) when path is empty", args: args{ flg: os.O_CREATE, perm: os.ModeDir, }, want: want{ want: nil, + err: errors.ErrPathNotSpecified, }, }, } @@ -144,8 +153,8 @@ func TestOpen(t *testing.T) { test.checkFunc = defaultCheckFunc } - got := Open(test.args.path, test.args.flg, test.args.perm) - if err := test.checkFunc(test.want, got); err != nil { + got, err := Open(test.args.path, test.args.flg, test.args.perm) + if err := test.checkFunc(test.want, got, err); err != nil { tt.Errorf("error = %v", err) } }) diff --git a/internal/net/http/json/json.go b/internal/net/http/json/json.go index 1f645201e8..0cd470d0c0 100644 --- a/internal/net/http/json/json.go +++ b/internal/net/http/json/json.go @@ -24,7 +24,7 @@ import ( "net/http" "os" - jsoniter "github.com/json-iterator/go" + "github.com/vdaas/vald/internal/encoding/json" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net/http/dump" @@ -40,35 +40,19 @@ type RFC7807Error struct { Error string `json:"error"` } -func Encode(w io.Writer, data interface{}) (err error) { - return jsoniter.NewEncoder(w).Encode(data) -} - -func Decode(r io.Reader, data interface{}) (err error) { - return jsoniter.NewDecoder(r).Decode(data) -} - -func Unmarshal(data []byte, i interface{}) error { - return jsoniter.Unmarshal(data, i) -} - -func MarshalIndent(data interface{}, pref, ind string) ([]byte, error) { - return jsoniter.MarshalIndent(data, pref, ind) -} - func EncodeResponse(w http.ResponseWriter, data interface{}, status int, contentTypes ...string) error { for _, ct := range contentTypes { w.Header().Add(rest.ContentType, ct) } w.WriteHeader(status) - return Encode(w, data) + return json.Encode(w, data) } // DecodeResponse decodes http response body. func DecodeResponse(res *http.Response, data interface{}) (err error) { if res != nil && res.Body != nil && data != nil && res.ContentLength != 0 { - err = Decode(res.Body, data) + err = json.Decode(res.Body, data) if err != nil { return err } @@ -94,7 +78,7 @@ func EncodeRequest(req *http.Request, } buf := new(bytes.Buffer) - if err := Encode(buf, data); err != nil { + if err := json.Encode(buf, data); err != nil { return err } @@ -107,7 +91,7 @@ func EncodeRequest(req *http.Request, // DecodeRequest decodes http request body. func DecodeRequest(r *http.Request, data interface{}) (err error) { if r != nil && r.Body != nil && r.ContentLength != 0 { - err = Decode(r.Body, data) + err = json.Decode(r.Body, data) if err != nil { return err } @@ -156,7 +140,7 @@ func ErrorHandler(w http.ResponseWriter, r *http.Request, log.Error(err) } body := make(map[string]interface{}) - err = Decode(r.Body, &body) + err = json.Decode(r.Body, &body) if err != nil { log.Error(err) } @@ -165,7 +149,7 @@ func ErrorHandler(w http.ResponseWriter, r *http.Request, log.Error(err) } - res, err := MarshalIndent(data, "", " ") + res, err := json.MarshalIndent(data, "", " ") if err != nil { return err } @@ -177,14 +161,14 @@ func ErrorHandler(w http.ResponseWriter, r *http.Request, } // Request sends http json request. -func Request(ctx context.Context, method string, url string, payloyd interface{}, data interface{}) error { +func Request(ctx context.Context, method, url string, payload, data interface{}) error { req, err := http.NewRequestWithContext(ctx, method, url, nil) if err != nil { return err } - if payloyd != nil && method != http.MethodGet { - if err := EncodeRequest(req, payloyd, rest.ApplicationJSON, rest.CharsetUTF8); err != nil { + if payload != nil && method != http.MethodGet { + if err := EncodeRequest(req, payload, rest.ApplicationJSON, rest.CharsetUTF8); err != nil { return err } } diff --git a/internal/net/http/json/json_test.go b/internal/net/http/json/json_test.go index 8d9d48feb0..4714fa0aa5 100644 --- a/internal/net/http/json/json_test.go +++ b/internal/net/http/json/json_test.go @@ -18,7 +18,6 @@ package json import ( "bytes" "context" - "io" "net/http" "net/http/httptest" "reflect" @@ -31,212 +30,6 @@ import ( "go.uber.org/goleak" ) -func TestEncode(t *testing.T) { - type args struct { - w io.Writer - data interface{} - } - - type test struct { - name string - args args - checkFunc func(err error) error - } - - tests := []test{ - func() test { - buf := new(bytes.Buffer) - data := map[string]string{ - "name": "vald", - } - - return test{ - name: "returns nil", - args: args{ - w: buf, - data: data, - }, - checkFunc: func(err error) error { - if err != nil { - return errors.Errorf("err not equals. want: %v, got: %v", nil, err) - } - - if got, want := buf.String(), "{\"name\":\"vald\"}\n"; got != want { - return errors.Errorf("output data not equals. want: %v, got: %v", want, got) - } - - return nil - }, - } - }(), - - { - name: "returns error when type is invalid", - args: args{ - w: new(bytes.Buffer), - data: make(chan struct{}), - }, - checkFunc: func(err error) error { - if err == nil { - return errors.New("err is nil") - } - return nil - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := Encode(tt.args.w, tt.args.data) - if err := tt.checkFunc(err); err != nil { - t.Error(err) - } - }) - } -} - -func TestDecode(t *testing.T) { - type args struct { - r io.Reader - data map[string]string - } - - type test struct { - name string - args args - checkFunc func(err error, data map[string]string) error - } - - tests := []test{ - func() test { - buf := new(bytes.Buffer) - buf.WriteString(`{"name":"vald"}`) - - return test{ - name: "returns nil", - args: args{ - r: buf, - data: make(map[string]string, 1), - }, - checkFunc: func(err error, data map[string]string) error { - if err != nil { - return errors.Errorf("err not equals. want: %v, got: %v", nil, err) - } - - if got, want := data, map[string]string{ - "name": "vald", - }; !reflect.DeepEqual(got, want) { - return errors.Errorf("read data not equals. want: %v, got: %v", want, got) - } - - return nil - }, - } - }(), - - func() test { - buf := new(bytes.Buffer) - buf.WriteString(`1`) - - wantData := make(map[string]string, 1) - - return test{ - name: "returns error when type is invalid", - args: args{ - r: buf, - data: wantData, - }, - checkFunc: func(err error, data map[string]string) error { - if err == nil { - return errors.New("err is nil") - } - - if !reflect.DeepEqual(data, wantData) { - return errors.Errorf("data not equals. want: %v, got: %v", wantData, data) - } - - return nil - }, - } - }(), - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := Decode(tt.args.r, &tt.args.data) - if err := tt.checkFunc(err, tt.args.data); err != nil { - t.Error(err) - } - }) - } -} - -func TestMarshalIndent(t *testing.T) { - type args struct { - data interface{} - pref string - ind string - } - - type test struct { - name string - args args - checkFunc func(data []byte, err error) error - } - - tests := []test{ - { - name: "returns data and nil", - args: args{ - data: map[string]string{ - "name": "vald", - }, - pref: "", - ind: "", - }, - checkFunc: func(data []byte, err error) error { - if err != nil { - return errors.Errorf("err not equals. want: %v, got: %v", nil, err) - } - - if got, want := data, []byte(`{"name":"vald"}`); !reflect.DeepEqual(got, want) { - return errors.Errorf("data not equals. want: %v, got: %v", string(got), string(want)) - } - - return nil - }, - }, - - { - name: "returns error when type is invalid", - args: args{ - data: make(chan struct{}), - pref: "", - ind: "", - }, - checkFunc: func(data []byte, err error) error { - if err == nil { - return errors.New("err is nil") - } - - if len(data) != 0 { - return errors.New("data is not empty") - } - return nil - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - data, err := MarshalIndent(tt.args.data, tt.args.pref, tt.args.ind) - if err := tt.checkFunc(data, err); err != nil { - t.Error(err) - } - }) - } -} - func TestEncodeResponse(t *testing.T) { type args struct { w http.ResponseWriter @@ -609,80 +402,6 @@ func TestErrorHandler(t *testing.T) { } } -func TestUnmarshal(t *testing.T) { - type args struct { - data []byte - i interface{} - } - type want struct { - err error - } - type test struct { - name string - args args - want want - checkFunc func(want, error) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want, err error) error { - if !errors.Is(err, w.err) { - return errors.Errorf("got error = %v, want %v", err, w.err) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - data: nil, - i: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - data: nil, - i: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, test := range tests { - t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - - err := Unmarshal(test.args.data, test.args.i) - if err := test.checkFunc(test.want, err); err != nil { - tt.Errorf("error = %v", err) - } - - }) - } -} - func TestDecodeResponse(t *testing.T) { type args struct { res *http.Response diff --git a/internal/timeutil/time.go b/internal/timeutil/time.go index cc1cd268e0..ce7f4cf959 100644 --- a/internal/timeutil/time.go +++ b/internal/timeutil/time.go @@ -33,3 +33,17 @@ func Parse(t string) (time.Duration, error) { } return dur, nil } + +// ParseWithDefault parses string to time.Duration and returns d when the parse failed. +func ParseWithDefault(t string, d time.Duration) time.Duration { + if t == "" { + return d + } + + parsed, err := Parse(t) + if err != nil { + return d + } + + return parsed +} diff --git a/internal/timeutil/time_test.go b/internal/timeutil/time_test.go index 924b312cfd..d7f7f06da7 100644 --- a/internal/timeutil/time_test.go +++ b/internal/timeutil/time_test.go @@ -17,8 +17,12 @@ package timeutil import ( + "reflect" "testing" "time" + + "github.com/vdaas/vald/internal/errors" + "go.uber.org/goleak" ) func TestParse(t *testing.T) { @@ -86,3 +90,80 @@ func TestParse(t *testing.T) { }) } } + +func TestParseWithDefault(t *testing.T) { + type args struct { + t string + d time.Duration + } + type want struct { + want time.Duration + } + type test struct { + name string + args args + want want + checkFunc func(want, time.Duration) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got time.Duration) error { + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got = %v, want %v", got, w.want) + } + return nil + } + tests := []test{ + { + name: "returns parsed result when t is a valid string", + args: args{ + t: "1s", + d: time.Hour, + }, + want: want{ + want: time.Second, + }, + }, + { + name: "returns default value when t is empty string", + args: args{ + t: "", + d: time.Hour, + }, + want: want{ + want: time.Hour, + }, + }, + { + name: "returns default value when t is invalid string", + args: args{ + t: "hoge", + d: time.Hour, + }, + want: want{ + want: time.Hour, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := ParseWithDefault(test.args.t, test.args.d) + if err := test.checkFunc(test.want, got); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} diff --git a/pkg/agent/core/ngt/service/ngt.go b/pkg/agent/core/ngt/service/ngt.go index 029e44a397..95dbfa8da3 100644 --- a/pkg/agent/core/ngt/service/ngt.go +++ b/pkg/agent/core/ngt/service/ngt.go @@ -20,9 +20,11 @@ package service import ( "context" "encoding/gob" + "math" "os" + "path/filepath" + "reflect" "runtime" - "strings" "sync" "sync/atomic" "time" @@ -34,11 +36,10 @@ import ( "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/observability/trace" - "github.com/vdaas/vald/internal/rand" "github.com/vdaas/vald/internal/safety" - "github.com/vdaas/vald/internal/timeutil" "github.com/vdaas/vald/pkg/agent/core/ngt/model" "github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs" + "github.com/vdaas/vald/pkg/agent/internal/metadata" ) type NGT interface { @@ -67,24 +68,42 @@ type NGT interface { } type ngt struct { - alen int + // instances + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches // insertion vector cache + dvc *vcaches // deletion vector cache + + // statuses indexing atomic.Value - saveMu sync.Mutex // creating or saving index - lim time.Duration // auto indexing time limit - dur time.Duration // auto indexing check duration - sdur time.Duration // auto save index check duration - idelay time.Duration // initial delay duration - dps uint32 // default pool size - ic uint64 // insert count - nocie uint64 // number of create index execution - eg errgroup.Group - ivc *vcaches // insertion vector cache - dvc *vcaches // deletion vector cache - path string - kvs kvs.BidiMap - core core.NGT - dcd bool // disable commit daemon - inMem bool + saveMu sync.Mutex // creating or saving index + + // counters + ic uint64 // insert count + nocie uint64 // number of create index execution + + // configurations + inMem bool // in-memory mode + + alen int // auto indexing length + + lim time.Duration // auto indexing time limit + dur time.Duration // auto indexing check duration + sdur time.Duration // auto save index check duration + + minLit time.Duration // minimum load index timeout + maxLit time.Duration // maximum load index timeout + litFactor time.Duration // load index timeout factor + + path string // index path + + poolSize uint32 // default pool size + radius float32 // default radius + epsilon float32 // default epsilon + + idelay time.Duration // initial delay duration + dcd bool // disable commit daemon } type vcache struct { @@ -96,110 +115,162 @@ const ( kvsFileName = "ngt-meta.kvsdb" ) -func New(cfg *config.NGT) (nn NGT, err error) { +func New(cfg *config.NGT, opts ...Option) (nn NGT, err error) { n := new(ngt) - n.inMem = cfg.EnableInMemoryMode - cfg.IndexPath = strings.TrimSuffix(cfg.IndexPath, "/") - opts := []core.Option{ + + for _, opt := range append(defaultOpts, opts...) { + if err := opt(n); err != nil { + return nil, errors.ErrOptionFailed(err, reflect.ValueOf(opt)) + } + } + + n.kvs = kvs.New() + + err = n.initNGT( core.WithInMemoryMode(n.inMem), - core.WithIndexPath(cfg.IndexPath), + core.WithIndexPath(n.path), + core.WithDefaultPoolSize(n.poolSize), + core.WithDefaultRadius(n.radius), + core.WithDefaultEpsilon(n.epsilon), core.WithDimension(cfg.Dimension), core.WithDistanceTypeByString(cfg.DistanceType), core.WithObjectTypeByString(cfg.ObjectType), core.WithBulkInsertChunkSize(cfg.BulkInsertChunkSize), core.WithCreationEdgeSize(cfg.CreationEdgeSize), core.WithSearchEdgeSize(cfg.SearchEdgeSize), + ) + if err != nil { + return nil, err } - if !n.inMem && len(cfg.IndexPath) != 0 { - n.path = cfg.IndexPath + if n.dur == 0 || n.alen == 0 { + n.dcd = true + } + if n.ivc == nil { + n.ivc = new(vcaches) + } + if n.dvc == nil { + n.dvc = new(vcaches) } - n.kvs = kvs.New() + n.indexing.Store(false) - if _, err = os.Stat(cfg.IndexPath); os.IsNotExist(err) || n.inMem { + return n, nil +} + +func (n *ngt) initNGT(opts ...core.Option) (err error) { + if _, err = os.Stat(n.path); os.IsNotExist(err) || n.inMem { n.core, err = core.New(opts...) - } else { - eg, _ := errgroup.New(context.Background()) - eg.Go(safety.RecoverFunc(func() (err error) { - n.core, err = core.Load(opts...) - return err - })) - eg.Go(safety.RecoverFunc(func() (err error) { - if len(n.path) != 0 && !n.inMem { - m := make(map[string]uint32) - gob.Register(map[string]uint32{}) - f := file.Open(n.path+"/"+kvsFileName, os.O_RDONLY|os.O_SYNC, os.ModePerm) - defer f.Close() - err = gob.NewDecoder(f).Decode(&m) - if err != nil { - return err - } - for k, id := range m { - n.kvs.Set(k, id) - } - } - return nil - })) - err = eg.Wait() + return err } + + log.Debugf("load index from %s", n.path) + + agentMetadata, err := metadata.Load(filepath.Join(n.path, metadata.AgentMetadataFileName)) if err != nil { - return nil, err + log.Warnf("cannot read metadata from %s: %s", metadata.AgentMetadataFileName, err) + } + + var timeout time.Duration + if agentMetadata != nil && agentMetadata.NGT != nil { + log.Debugf("the backup index size is %d. starting to load...", agentMetadata.NGT.IndexCount) + timeout = time.Duration( + math.Min( + math.Max( + float64(agentMetadata.NGT.IndexCount)*float64(n.litFactor), + float64(n.minLit), + ), + float64(n.maxLit), + ), + ) + } else { + log.Debugf("cannot inspect the backup index size. starting to load.") + timeout = time.Duration(math.Min(float64(n.minLit), float64(n.maxLit))) } - if cfg.AutoIndexCheckDuration != "" { - d, err := timeutil.Parse(cfg.AutoIndexCheckDuration) - if err != nil { - d = 0 - } - n.dur = d - } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() - if cfg.AutoIndexDurationLimit != "" { - d, err := timeutil.Parse(cfg.AutoIndexDurationLimit) - if err != nil { - d = 0 - } - n.lim = d - } + eg, _ := errgroup.New(ctx) + eg.Go(safety.RecoverFunc(func() (err error) { + n.core, err = core.Load(opts...) + return err + })) + + eg.Go(safety.RecoverFunc(n.loadKVS)) - if cfg.AutoSaveIndexDuration != "" { - d, err := timeutil.Parse(cfg.AutoSaveIndexDuration) + ech := make(chan error, 1) + + // NOTE: when it exceeds the timeout while loading, + // it should exit this function and leave this goroutine running. + go func() { + defer close(ech) + + err = safety.RecoverFunc(func() (err error) { + err = eg.Wait() + if err != nil { + return err + } + cancel() + return nil + })() if err != nil { - d = 0 + ech <- err } - n.sdur = d - } + }() - if cfg.InitialDelayMaxDuration != "" { - d, err := timeutil.Parse(cfg.InitialDelayMaxDuration) - if err != nil { - d = 0 + select { + case err := <-ech: + return err + case <-ctx.Done(): + if ctx.Err() == context.DeadlineExceeded { + log.Errorf("cannot load index backup data within the timeout %s. the process is going to be killed.", timeout) + + err := metadata.Store( + filepath.Join(n.path, metadata.AgentMetadataFileName), + &metadata.Metadata{ + IsInvalid: true, + NGT: &metadata.NGT{ + IndexCount: 0, + }, + }, + ) + if err != nil { + return err + } + + return errors.ErrIndexLoadTimeout } - n.idelay = time.Duration( - int64(rand.LimitedUint32(uint64(d/time.Second))), - ) * time.Second } - n.alen = cfg.AutoIndexLength + return nil +} - n.eg = errgroup.Get() +func (n *ngt) loadKVS() error { + gob.Register(map[string]uint32{}) - if n.dur == 0 || n.alen == 0 { - n.dcd = true - } - if n.ivc == nil { - n.ivc = new(vcaches) + f, err := file.Open( + filepath.Join(n.path, kvsFileName), + os.O_RDONLY|os.O_SYNC, + os.ModePerm, + ) + if err != nil { + return err } - if n.dvc == nil { - n.dvc = new(vcaches) + + defer f.Close() + + m := make(map[string]uint32) + err = gob.NewDecoder(f).Decode(&m) + if err != nil { + return err } - if in, ok := n.indexing.Load().(bool); !ok || in { - n.indexing.Store(false) + for k, id := range m { + n.kvs.Set(k, id) } - return n, nil + return nil } func (n *ngt) Start(ctx context.Context) <-chan error { @@ -235,7 +306,7 @@ func (n *ngt) Start(ctx context.Context) <-chan error { err = nil select { case <-ctx.Done(): - err = n.CreateIndex(ctx, n.dps) + err = n.CreateIndex(ctx, n.poolSize) if err != nil { ech <- err return errors.Wrap(ctx.Err(), err.Error()) @@ -243,10 +314,10 @@ func (n *ngt) Start(ctx context.Context) <-chan error { return ctx.Err() case <-tick.C: if int(atomic.LoadUint64(&n.ic)) >= n.alen { - err = n.CreateIndex(ctx, n.dps) + err = n.CreateIndex(ctx, n.poolSize) } case <-limit.C: - err = n.CreateAndSaveIndex(ctx, n.dps) + err = n.CreateAndSaveIndex(ctx, n.poolSize) case <-sTick.C: err = n.SaveIndex(ctx) } @@ -566,32 +637,60 @@ func (n *ngt) SaveIndex(ctx context.Context) (err error) { defer n.saveMu.Unlock() if len(n.path) != 0 && !n.inMem { - eg, ctx := errgroup.New(ctx) - eg.Go(safety.RecoverFunc(func() error { - if len(n.path) != 0 { - m := make(map[string]uint32, n.kvs.Len()) - var mu sync.Mutex - n.kvs.Range(ctx, func(key string, id uint32) bool { - mu.Lock() - m[key] = id - mu.Unlock() - return true - }) - f := file.Open(n.path+"/"+kvsFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) - defer f.Close() - gob.Register(map[string]uint32{}) - return gob.NewEncoder(f).Encode(&m) - } - return nil - })) - eg.Go(safety.RecoverFunc(func() error { - return n.core.SaveIndex() - })) - err = eg.Wait() + err = n.saveIndex(ctx) } + return } +func (n *ngt) saveIndex(ctx context.Context) (err error) { + eg, ctx := errgroup.New(ctx) + + eg.Go(safety.RecoverFunc(func() error { + if n.path != "" { + m := make(map[string]uint32, n.kvs.Len()) + var mu sync.Mutex + n.kvs.Range(ctx, func(key string, id uint32) bool { + mu.Lock() + m[key] = id + mu.Unlock() + return true + }) + f, err := file.Open( + filepath.Join(n.path, kvsFileName), + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, + os.ModePerm, + ) + if err != nil { + return err + } + defer f.Close() + gob.Register(map[string]uint32{}) + return gob.NewEncoder(f).Encode(&m) + } + return nil + })) + + eg.Go(safety.RecoverFunc(func() error { + return n.core.SaveIndex() + })) + + err = eg.Wait() + if err != nil { + return err + } + + return metadata.Store( + filepath.Join(n.path, metadata.AgentMetadataFileName), + &metadata.Metadata{ + IsInvalid: false, + NGT: &metadata.NGT{ + IndexCount: n.Len(), + }, + }, + ) +} + func (n *ngt) CreateAndSaveIndex(ctx context.Context, poolSize uint32) (err error) { ctx, span := trace.StartSpan(ctx, "vald/agent-ngt/service/NGT.CreateAndSaveIndex") defer func() { @@ -674,7 +773,7 @@ func (n *ngt) DeleteVCacheLen() uint64 { func (n *ngt) Close(ctx context.Context) (err error) { if len(n.path) != 0 { - err = n.CreateAndSaveIndex(ctx, n.dps) + err = n.CreateAndSaveIndex(ctx, n.poolSize) } n.core.Close() return diff --git a/pkg/agent/core/ngt/service/ngt_test.go b/pkg/agent/core/ngt/service/ngt_test.go index 8cc5c4af7b..3b90b09008 100644 --- a/pkg/agent/core/ngt/service/ngt_test.go +++ b/pkg/agent/core/ngt/service/ngt_test.go @@ -20,6 +20,7 @@ package service import ( "context" "reflect" + "sync" "sync/atomic" "testing" "time" @@ -30,13 +31,13 @@ import ( "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/pkg/agent/core/ngt/model" "github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs" - "go.uber.org/goleak" ) func TestNew(t *testing.T) { type args struct { - cfg *config.NGT + cfg *config.NGT + opts []Option } type want struct { wantNn NGT @@ -66,6 +67,7 @@ func TestNew(t *testing.T) { name: "test_case_1", args: args { cfg: nil, + opts: nil, }, want: want{}, checkFunc: defaultCheckFunc, @@ -79,6 +81,7 @@ func TestNew(t *testing.T) { name: "test_case_2", args: args { cfg: nil, + opts: nil, }, want: want{}, checkFunc: defaultCheckFunc, @@ -89,7 +92,7 @@ func TestNew(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -100,7 +103,7 @@ func TestNew(t *testing.T) { test.checkFunc = defaultCheckFunc } - gotNn, err := New(test.args.cfg) + gotNn, err := New(test.args.cfg, test.args.opts...) if err := test.checkFunc(test.want, gotNn, err); err != nil { tt.Errorf("error = %v", err) } @@ -109,25 +112,368 @@ func TestNew(t *testing.T) { } } +func Test_ngt_initNGT(t *testing.T) { + type args struct { + opts []core.Option + } + type fields struct { + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool + } + type want struct { + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + opts: nil, + }, + fields: fields { + core: nil, + eg: nil, + kvs: nil, + ivc: vcaches{}, + dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, + path: "", + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, + dcd: false, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + opts: nil, + }, + fields: fields { + core: nil, + eg: nil, + kvs: nil, + ivc: vcaches{}, + dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, + path: "", + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, + dcd: false, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + n := &ngt{ + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, + } + + err := n.initNGT(test.args.opts...) + if err := test.checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func Test_ngt_loadKVS(t *testing.T) { + type fields struct { + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool + } + type want struct { + err error + } + type test struct { + name string + fields fields + want want + checkFunc func(want, error) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + core: nil, + eg: nil, + kvs: nil, + ivc: vcaches{}, + dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, + path: "", + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, + dcd: false, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + core: nil, + eg: nil, + kvs: nil, + ivc: vcaches{}, + dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, + path: "", + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, + dcd: false, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + n := &ngt{ + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, + } + + err := n.loadKVS() + if err := test.checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + func Test_ngt_Start(t *testing.T) { type args struct { ctx context.Context } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { want <-chan error @@ -156,19 +502,28 @@ func Test_ngt_Start(t *testing.T) { ctx: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -185,19 +540,28 @@ func Test_ngt_Start(t *testing.T) { ctx: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -209,7 +573,7 @@ func Test_ngt_Start(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -220,20 +584,29 @@ func Test_ngt_Start(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } got := n.Start(test.args.ctx) @@ -253,20 +626,29 @@ func Test_ngt_Search(t *testing.T) { radius float32 } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { want []model.Distance @@ -302,19 +684,28 @@ func Test_ngt_Search(t *testing.T) { radius: 0, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -334,19 +725,28 @@ func Test_ngt_Search(t *testing.T) { radius: 0, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -358,7 +758,7 @@ func Test_ngt_Search(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -369,20 +769,29 @@ func Test_ngt_Search(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } got, err := n.Search(test.args.vec, test.args.size, test.args.epsilon, test.args.radius) @@ -402,20 +811,29 @@ func Test_ngt_SearchByID(t *testing.T) { radius float32 } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { wantDst []model.Distance @@ -451,19 +869,205 @@ func Test_ngt_SearchByID(t *testing.T) { radius: 0, }, fields: fields { - alen: 0, + core: nil, + eg: nil, + kvs: nil, + ivc: vcaches{}, + dvc: vcaches{}, indexing: nil, - lim: nil, - dur: nil, - dps: 0, + saveMu: sync.Mutex{}, ic: 0, nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, + path: "", + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, + dcd: false, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + uuid: "", + size: 0, + epsilon: 0, + radius: 0, + }, + fields: fields { + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, + dcd: false, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + n := &ngt{ + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, + } + + gotDst, err := n.SearchByID(test.args.uuid, test.args.size, test.args.epsilon, test.args.radius) + if err := test.checkFunc(test.want, gotDst, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func Test_ngt_Insert(t *testing.T) { + type args struct { + uuid string + vec []float32 + } + type fields struct { + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool + } + type want struct { + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + uuid: "", + vec: nil, + }, + fields: fields { core: nil, + eg: nil, + kvs: nil, + ivc: vcaches{}, + dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, + path: "", + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -478,24 +1082,31 @@ func Test_ngt_SearchByID(t *testing.T) { name: "test_case_2", args: args { uuid: "", - size: 0, - epsilon: 0, - radius: 0, + vec: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -507,7 +1118,7 @@ func Test_ngt_SearchByID(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -518,24 +1129,33 @@ func Test_ngt_SearchByID(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - gotDst, err := n.SearchByID(test.args.uuid, test.args.size, test.args.epsilon, test.args.radius) - if err := test.checkFunc(test.want, gotDst, err); err != nil { + err := n.Insert(test.args.uuid, test.args.vec) + if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -543,26 +1163,37 @@ func Test_ngt_SearchByID(t *testing.T) { } } -func Test_ngt_Insert(t *testing.T) { +func Test_ngt_insert(t *testing.T) { type args struct { - uuid string - vec []float32 + uuid string + vec []float32 + t int64 + validation bool } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -590,21 +1221,32 @@ func Test_ngt_Insert(t *testing.T) { args: args { uuid: "", vec: nil, + t: 0, + validation: false, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -620,21 +1262,32 @@ func Test_ngt_Insert(t *testing.T) { args: args { uuid: "", vec: nil, + t: 0, + validation: false, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -646,7 +1299,7 @@ func Test_ngt_Insert(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -657,23 +1310,32 @@ func Test_ngt_Insert(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - err := n.Insert(test.args.uuid, test.args.vec) + err := n.insert(test.args.uuid, test.args.vec, test.args.t, test.args.validation) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -682,28 +1344,34 @@ func Test_ngt_Insert(t *testing.T) { } } -func Test_ngt_insert(t *testing.T) { +func Test_ngt_InsertMultiple(t *testing.T) { type args struct { - uuid string - vec []float32 - t int64 - validation bool + vecs map[string][]float32 } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -729,25 +1397,31 @@ func Test_ngt_insert(t *testing.T) { { name: "test_case_1", args: args { - uuid: "", - vec: nil, - t: 0, - validation: false, + vecs: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -761,25 +1435,31 @@ func Test_ngt_insert(t *testing.T) { return test { name: "test_case_2", args: args { - uuid: "", - vec: nil, - t: 0, - validation: false, + vecs: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -791,7 +1471,7 @@ func Test_ngt_insert(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -802,23 +1482,32 @@ func Test_ngt_insert(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - err := n.insert(test.args.uuid, test.args.vec, test.args.t, test.args.validation) + err := n.InsertMultiple(test.args.vecs) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -827,25 +1516,35 @@ func Test_ngt_insert(t *testing.T) { } } -func Test_ngt_InsertMultiple(t *testing.T) { +func Test_ngt_Update(t *testing.T) { type args struct { - vecs map[string][]float32 + uuid string + vec []float32 } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -871,22 +1570,32 @@ func Test_ngt_InsertMultiple(t *testing.T) { { name: "test_case_1", args: args { - vecs: nil, + uuid: "", + vec: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -900,22 +1609,32 @@ func Test_ngt_InsertMultiple(t *testing.T) { return test { name: "test_case_2", args: args { - vecs: nil, + uuid: "", + vec: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -927,7 +1646,7 @@ func Test_ngt_InsertMultiple(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -938,23 +1657,32 @@ func Test_ngt_InsertMultiple(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - err := n.InsertMultiple(test.args.vecs) + err := n.Update(test.args.uuid, test.args.vec) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -963,26 +1691,34 @@ func Test_ngt_InsertMultiple(t *testing.T) { } } -func Test_ngt_Update(t *testing.T) { +func Test_ngt_UpdateMultiple(t *testing.T) { type args struct { - uuid string - vec []float32 + vecs map[string][]float32 } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -1008,23 +1744,31 @@ func Test_ngt_Update(t *testing.T) { { name: "test_case_1", args: args { - uuid: "", - vec: nil, + vecs: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1038,23 +1782,31 @@ func Test_ngt_Update(t *testing.T) { return test { name: "test_case_2", args: args { - uuid: "", - vec: nil, + vecs: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1066,7 +1818,7 @@ func Test_ngt_Update(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -1077,23 +1829,32 @@ func Test_ngt_Update(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - err := n.Update(test.args.uuid, test.args.vec) + err := n.UpdateMultiple(test.args.vecs) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -1102,25 +1863,34 @@ func Test_ngt_Update(t *testing.T) { } } -func Test_ngt_UpdateMultiple(t *testing.T) { +func Test_ngt_Delete(t *testing.T) { type args struct { - vecs map[string][]float32 + uuid string } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -1146,22 +1916,31 @@ func Test_ngt_UpdateMultiple(t *testing.T) { { name: "test_case_1", args: args { - vecs: nil, - }, - fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + uuid: "", + }, + fields: fields { + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1175,22 +1954,31 @@ func Test_ngt_UpdateMultiple(t *testing.T) { return test { name: "test_case_2", args: args { - vecs: nil, + uuid: "", }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1202,7 +1990,7 @@ func Test_ngt_UpdateMultiple(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -1213,23 +2001,32 @@ func Test_ngt_UpdateMultiple(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - err := n.UpdateMultiple(test.args.vecs) + err := n.Delete(test.args.uuid) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -1238,25 +2035,35 @@ func Test_ngt_UpdateMultiple(t *testing.T) { } } -func Test_ngt_Delete(t *testing.T) { +func Test_ngt_delete(t *testing.T) { type args struct { uuid string + t int64 } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -1283,21 +2090,31 @@ func Test_ngt_Delete(t *testing.T) { name: "test_case_1", args: args { uuid: "", + t: 0, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1312,21 +2129,31 @@ func Test_ngt_Delete(t *testing.T) { name: "test_case_2", args: args { uuid: "", + t: 0, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1338,7 +2165,7 @@ func Test_ngt_Delete(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -1349,23 +2176,32 @@ func Test_ngt_Delete(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - err := n.Delete(test.args.uuid) + err := n.delete(test.args.uuid, test.args.t) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -1374,26 +2210,34 @@ func Test_ngt_Delete(t *testing.T) { } } -func Test_ngt_delete(t *testing.T) { +func Test_ngt_DeleteMultiple(t *testing.T) { type args struct { - uuid string - t int64 + uuids []string } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -1419,23 +2263,31 @@ func Test_ngt_delete(t *testing.T) { { name: "test_case_1", args: args { - uuid: "", - t: 0, + uuids: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1449,23 +2301,31 @@ func Test_ngt_delete(t *testing.T) { return test { name: "test_case_2", args: args { - uuid: "", - t: 0, + uuids: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1477,7 +2337,7 @@ func Test_ngt_delete(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -1488,23 +2348,32 @@ func Test_ngt_delete(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - err := n.delete(test.args.uuid, test.args.t) + err := n.DeleteMultiple(test.args.uuids...) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -1513,42 +2382,55 @@ func Test_ngt_delete(t *testing.T) { } } -func Test_ngt_DeleteMultiple(t *testing.T) { +func Test_ngt_GetObject(t *testing.T) { type args struct { - uuids []string + uuid string } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { - err error + wantVec []float32 + err error } type test struct { name string args args fields fields want want - checkFunc func(want, error) error + checkFunc func(want, []float32, error) error beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, err error) error { + defaultCheckFunc := func(w want, gotVec []float32, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got error = %v, want %v", err, w.err) } + if !reflect.DeepEqual(gotVec, w.wantVec) { + return errors.Errorf("got = %v, want %v", gotVec, w.wantVec) + } return nil } tests := []test{ @@ -1557,22 +2439,31 @@ func Test_ngt_DeleteMultiple(t *testing.T) { { name: "test_case_1", args: args { - uuids: nil, + uuid: "", }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1586,22 +2477,31 @@ func Test_ngt_DeleteMultiple(t *testing.T) { return test { name: "test_case_2", args: args { - uuids: nil, + uuid: "", }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1613,7 +2513,7 @@ func Test_ngt_DeleteMultiple(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -1624,24 +2524,33 @@ func Test_ngt_DeleteMultiple(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - err := n.DeleteMultiple(test.args.uuids...) - if err := test.checkFunc(test.want, err); err != nil { + gotVec, err := n.GetObject(test.args.uuid) + if err := test.checkFunc(test.want, gotVec, err); err != nil { tt.Errorf("error = %v", err) } @@ -1649,46 +2558,52 @@ func Test_ngt_DeleteMultiple(t *testing.T) { } } -func Test_ngt_GetObject(t *testing.T) { +func Test_ngt_CreateIndex(t *testing.T) { type args struct { - uuid string + ctx context.Context + poolSize uint32 } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { - wantVec []float32 - err error + err error } type test struct { name string args args fields fields want want - checkFunc func(want, []float32, error) error + checkFunc func(want, error) error beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, gotVec []float32, err error) error { + defaultCheckFunc := func(w want, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got error = %v, want %v", err, w.err) } - if !reflect.DeepEqual(gotVec, w.wantVec) { - return errors.Errorf("got = %v, want %v", gotVec, w.wantVec) - } return nil } tests := []test{ @@ -1697,22 +2612,32 @@ func Test_ngt_GetObject(t *testing.T) { { name: "test_case_1", args: args { - uuid: "", + ctx: nil, + poolSize: 0, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1726,22 +2651,32 @@ func Test_ngt_GetObject(t *testing.T) { return test { name: "test_case_2", args: args { - uuid: "", + ctx: nil, + poolSize: 0, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1753,7 +2688,7 @@ func Test_ngt_GetObject(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -1764,24 +2699,33 @@ func Test_ngt_GetObject(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - - gotVec, err := n.GetObject(test.args.uuid) - if err := test.checkFunc(test.want, gotVec, err); err != nil { + + err := n.CreateIndex(test.args.ctx, test.args.poolSize) + if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -1789,26 +2733,34 @@ func Test_ngt_GetObject(t *testing.T) { } } -func Test_ngt_CreateIndex(t *testing.T) { +func Test_ngt_SaveIndex(t *testing.T) { type args struct { - ctx context.Context - poolSize uint32 + ctx context.Context } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -1834,22 +2786,31 @@ func Test_ngt_CreateIndex(t *testing.T) { { name: "test_case_1", args: args { - poolSize: 0, + ctx: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1863,22 +2824,31 @@ func Test_ngt_CreateIndex(t *testing.T) { return test { name: "test_case_2", args: args { - poolSize: 0, + ctx: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -1890,7 +2860,7 @@ func Test_ngt_CreateIndex(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -1901,23 +2871,32 @@ func Test_ngt_CreateIndex(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } - err := n.CreateIndex(test.args.ctx, test.args.poolSize) + err := n.SaveIndex(test.args.ctx) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -1926,25 +2905,34 @@ func Test_ngt_CreateIndex(t *testing.T) { } } -func Test_ngt_SaveIndex(t *testing.T) { +func Test_ngt_saveIndex(t *testing.T) { type args struct { ctx context.Context } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -1973,19 +2961,28 @@ func Test_ngt_SaveIndex(t *testing.T) { ctx: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2002,19 +2999,28 @@ func Test_ngt_SaveIndex(t *testing.T) { ctx: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2026,7 +3032,7 @@ func Test_ngt_SaveIndex(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -2037,23 +3043,32 @@ func Test_ngt_SaveIndex(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, - } - - err := n.SaveIndex(test.args.ctx) + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, + } + + err := n.saveIndex(test.args.ctx) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -2068,20 +3083,29 @@ func Test_ngt_CreateAndSaveIndex(t *testing.T) { poolSize uint32 } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -2111,19 +3135,28 @@ func Test_ngt_CreateAndSaveIndex(t *testing.T) { poolSize: 0, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2141,19 +3174,28 @@ func Test_ngt_CreateAndSaveIndex(t *testing.T) { poolSize: 0, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2165,7 +3207,7 @@ func Test_ngt_CreateAndSaveIndex(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -2176,20 +3218,29 @@ func Test_ngt_CreateAndSaveIndex(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } err := n.CreateAndSaveIndex(test.args.ctx, test.args.poolSize) @@ -2206,20 +3257,29 @@ func Test_ngt_Exists(t *testing.T) { uuid string } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { wantOid uint32 @@ -2252,19 +3312,28 @@ func Test_ngt_Exists(t *testing.T) { uuid: "", }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2281,19 +3350,28 @@ func Test_ngt_Exists(t *testing.T) { uuid: "", }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2305,7 +3383,7 @@ func Test_ngt_Exists(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -2316,20 +3394,29 @@ func Test_ngt_Exists(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } gotOid, gotOk := n.Exists(test.args.uuid) @@ -2346,20 +3433,29 @@ func Test_ngt_insertCache(t *testing.T) { uuid string } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { want *vcache @@ -2392,19 +3488,28 @@ func Test_ngt_insertCache(t *testing.T) { uuid: "", }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2421,19 +3526,28 @@ func Test_ngt_insertCache(t *testing.T) { uuid: "", }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2445,7 +3559,7 @@ func Test_ngt_insertCache(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -2456,20 +3570,29 @@ func Test_ngt_insertCache(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } got, got1 := n.insertCache(test.args.uuid) @@ -2483,20 +3606,29 @@ func Test_ngt_insertCache(t *testing.T) { func Test_ngt_IsIndexing(t *testing.T) { type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { want bool @@ -2521,19 +3653,28 @@ func Test_ngt_IsIndexing(t *testing.T) { { name: "test_case_1", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2547,19 +3688,28 @@ func Test_ngt_IsIndexing(t *testing.T) { return test { name: "test_case_2", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2571,7 +3721,7 @@ func Test_ngt_IsIndexing(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc() } @@ -2582,20 +3732,29 @@ func Test_ngt_IsIndexing(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } got := n.IsIndexing() @@ -2612,20 +3771,29 @@ func Test_ngt_UUIDs(t *testing.T) { ctx context.Context } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { wantUuids []string @@ -2654,19 +3822,28 @@ func Test_ngt_UUIDs(t *testing.T) { ctx: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2683,19 +3860,28 @@ func Test_ngt_UUIDs(t *testing.T) { ctx: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2707,7 +3893,7 @@ func Test_ngt_UUIDs(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -2718,20 +3904,29 @@ func Test_ngt_UUIDs(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } gotUuids := n.UUIDs(test.args.ctx) @@ -2745,20 +3940,29 @@ func Test_ngt_UUIDs(t *testing.T) { func Test_ngt_UncommittedUUIDs(t *testing.T) { type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { wantUuids []string @@ -2783,19 +3987,28 @@ func Test_ngt_UncommittedUUIDs(t *testing.T) { { name: "test_case_1", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2809,19 +4022,28 @@ func Test_ngt_UncommittedUUIDs(t *testing.T) { return test { name: "test_case_2", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2833,7 +4055,7 @@ func Test_ngt_UncommittedUUIDs(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc() } @@ -2844,20 +4066,29 @@ func Test_ngt_UncommittedUUIDs(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } gotUuids := n.UncommittedUUIDs() @@ -2871,20 +4102,29 @@ func Test_ngt_UncommittedUUIDs(t *testing.T) { func Test_ngt_NumberOfCreateIndexExecution(t *testing.T) { type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { want uint64 @@ -2909,19 +4149,28 @@ func Test_ngt_NumberOfCreateIndexExecution(t *testing.T) { { name: "test_case_1", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2935,19 +4184,28 @@ func Test_ngt_NumberOfCreateIndexExecution(t *testing.T) { return test { name: "test_case_2", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -2959,7 +4217,7 @@ func Test_ngt_NumberOfCreateIndexExecution(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc() } @@ -2970,20 +4228,29 @@ func Test_ngt_NumberOfCreateIndexExecution(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } got := n.NumberOfCreateIndexExecution() @@ -2997,20 +4264,29 @@ func Test_ngt_NumberOfCreateIndexExecution(t *testing.T) { func Test_ngt_Len(t *testing.T) { type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { want uint64 @@ -3035,19 +4311,28 @@ func Test_ngt_Len(t *testing.T) { { name: "test_case_1", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -3061,19 +4346,28 @@ func Test_ngt_Len(t *testing.T) { return test { name: "test_case_2", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -3085,7 +4379,7 @@ func Test_ngt_Len(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc() } @@ -3096,20 +4390,29 @@ func Test_ngt_Len(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } got := n.Len() @@ -3123,20 +4426,29 @@ func Test_ngt_Len(t *testing.T) { func Test_ngt_InsertVCacheLen(t *testing.T) { type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { want uint64 @@ -3161,19 +4473,28 @@ func Test_ngt_InsertVCacheLen(t *testing.T) { { name: "test_case_1", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -3187,19 +4508,28 @@ func Test_ngt_InsertVCacheLen(t *testing.T) { return test { name: "test_case_2", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -3211,7 +4541,7 @@ func Test_ngt_InsertVCacheLen(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc() } @@ -3222,20 +4552,29 @@ func Test_ngt_InsertVCacheLen(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } got := n.InsertVCacheLen() @@ -3249,20 +4588,29 @@ func Test_ngt_InsertVCacheLen(t *testing.T) { func Test_ngt_DeleteVCacheLen(t *testing.T) { type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { want uint64 @@ -3287,19 +4635,28 @@ func Test_ngt_DeleteVCacheLen(t *testing.T) { { name: "test_case_1", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -3313,19 +4670,28 @@ func Test_ngt_DeleteVCacheLen(t *testing.T) { return test { name: "test_case_2", fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -3337,7 +4703,7 @@ func Test_ngt_DeleteVCacheLen(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc() } @@ -3348,20 +4714,29 @@ func Test_ngt_DeleteVCacheLen(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } got := n.DeleteVCacheLen() @@ -3378,20 +4753,29 @@ func Test_ngt_Close(t *testing.T) { ctx context.Context } type fields struct { - alen int - indexing atomic.Value - lim time.Duration - dur time.Duration - dps uint32 - ic uint64 - nocie uint64 - eg errgroup.Group - ivc *vcaches - dvc *vcaches - path string - kvs kvs.BidiMap - core core.NGT - dcd bool + core core.NGT + eg errgroup.Group + kvs kvs.BidiMap + ivc *vcaches + dvc *vcaches + indexing atomic.Value + saveMu sync.Mutex + ic uint64 + nocie uint64 + inMem bool + alen int + lim time.Duration + dur time.Duration + sdur time.Duration + minLit time.Duration + maxLit time.Duration + litFactor time.Duration + path string + poolSize uint32 + radius float32 + epsilon float32 + idelay time.Duration + dcd bool } type want struct { err error @@ -3420,19 +4804,28 @@ func Test_ngt_Close(t *testing.T) { ctx: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -3449,19 +4842,28 @@ func Test_ngt_Close(t *testing.T) { ctx: nil, }, fields: fields { - alen: 0, - indexing: nil, - lim: nil, - dur: nil, - dps: 0, - ic: 0, - nocie: 0, + core: nil, eg: nil, + kvs: nil, ivc: vcaches{}, dvc: vcaches{}, + indexing: nil, + saveMu: sync.Mutex{}, + ic: 0, + nocie: 0, + inMem: false, + alen: 0, + lim: nil, + dur: nil, + sdur: nil, + minLit: nil, + maxLit: nil, + litFactor: nil, path: "", - kvs: nil, - core: nil, + poolSize: 0, + radius: 0, + epsilon: 0, + idelay: nil, dcd: false, }, want: want{}, @@ -3473,7 +4875,7 @@ func Test_ngt_Close(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -3484,20 +4886,29 @@ func Test_ngt_Close(t *testing.T) { test.checkFunc = defaultCheckFunc } n := &ngt{ - alen: test.fields.alen, - indexing: test.fields.indexing, - lim: test.fields.lim, - dur: test.fields.dur, - dps: test.fields.dps, - ic: test.fields.ic, - nocie: test.fields.nocie, - eg: test.fields.eg, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - path: test.fields.path, - kvs: test.fields.kvs, - core: test.fields.core, - dcd: test.fields.dcd, + core: test.fields.core, + eg: test.fields.eg, + kvs: test.fields.kvs, + ivc: test.fields.ivc, + dvc: test.fields.dvc, + indexing: test.fields.indexing, + saveMu: test.fields.saveMu, + ic: test.fields.ic, + nocie: test.fields.nocie, + inMem: test.fields.inMem, + alen: test.fields.alen, + lim: test.fields.lim, + dur: test.fields.dur, + sdur: test.fields.sdur, + minLit: test.fields.minLit, + maxLit: test.fields.maxLit, + litFactor: test.fields.litFactor, + path: test.fields.path, + poolSize: test.fields.poolSize, + radius: test.fields.radius, + epsilon: test.fields.epsilon, + idelay: test.fields.idelay, + dcd: test.fields.dcd, } err := n.Close(test.args.ctx) diff --git a/pkg/agent/core/ngt/service/option.go b/pkg/agent/core/ngt/service/option.go new file mode 100644 index 0000000000..b060fb9213 --- /dev/null +++ b/pkg/agent/core/ngt/service/option.go @@ -0,0 +1,223 @@ +// +// Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package service + +import ( + "path/filepath" + "strings" + "time" + + core "github.com/vdaas/vald/internal/core/ngt" + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/rand" + "github.com/vdaas/vald/internal/timeutil" +) + +type Option func(n *ngt) error + +var ( + defaultOpts = []Option{ + WithErrGroup(errgroup.Get()), + WithAutoIndexCheckDuration("30m"), + WithAutoIndexDurationLimit("24h"), + WithAutoSaveIndexDuration("35m"), + WithAutoIndexLength(100), + WithInitialDelayMaxDuration("3m"), + WithMinLoadIndexTimeout("3m"), + WithMaxLoadIndexTimeout("10m"), + WithLoadIndexTimeoutFactor("1ms"), + WithDefaultPoolSize(core.DefaultPoolSize), + WithDefaultRadius(core.DefaultRadius), + WithDefaultEpsilon(core.DefaultEpsilon), + } +) + +func WithErrGroup(eg errgroup.Group) Option { + return func(n *ngt) error { + if eg != nil { + n.eg = eg + } + + return nil + } +} + +func WithEnableInMemoryMode(enabled bool) Option { + return func(n *ngt) error { + n.inMem = enabled + + return nil + } +} + +func WithIndexPath(path string) Option { + return func(n *ngt) error { + n.path = filepath.Clean(strings.TrimSuffix(path, "/")) + + return nil + } +} + +func WithAutoIndexCheckDuration(dur string) Option { + return func(n *ngt) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + n.dur = d + + return nil + } +} + +func WithAutoIndexDurationLimit(dur string) Option { + return func(n *ngt) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + n.lim = d + + return nil + } +} + +func WithAutoSaveIndexDuration(dur string) Option { + return func(n *ngt) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + n.sdur = d + + return nil + } +} + +func WithAutoIndexLength(l int) Option { + return func(n *ngt) error { + n.alen = l + + return nil + } +} + +func WithInitialDelayMaxDuration(dur string) Option { + return func(n *ngt) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + n.idelay = time.Duration(int64(rand.LimitedUint32(uint64(d/time.Second)))) * time.Second + + return nil + } +} + +func WithMinLoadIndexTimeout(dur string) Option { + return func(n *ngt) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + n.minLit = d + + return nil + } +} + +func WithMaxLoadIndexTimeout(dur string) Option { + return func(n *ngt) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + n.maxLit = d + + return nil + } +} + +func WithLoadIndexTimeoutFactor(dur string) Option { + return func(n *ngt) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + n.litFactor = d + + return nil + } +} + +func WithDefaultPoolSize(ps uint32) Option { + return func(n *ngt) error { + n.poolSize = ps + + return nil + } +} + +func WithDefaultRadius(rad float32) Option { + return func(n *ngt) error { + n.radius = rad + + return nil + } +} +func WithDefaultEpsilon(epsilon float32) Option { + return func(n *ngt) error { + n.epsilon = epsilon + + return nil + } +} diff --git a/pkg/agent/core/ngt/service/option_test.go b/pkg/agent/core/ngt/service/option_test.go new file mode 100644 index 0000000000..e2d5b7475c --- /dev/null +++ b/pkg/agent/core/ngt/service/option_test.go @@ -0,0 +1,1620 @@ +// +// Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package service + +import ( + "testing" + + "github.com/vdaas/vald/internal/errgroup" + "go.uber.org/goleak" +) + +func TestWithErrGroup(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + eg errgroup.Group + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + eg: nil, + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + eg: nil, + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithErrGroup(test.args.eg) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithErrGroup(test.args.eg) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithEnableInMemoryMode(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + enabled bool + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + enabled: false, + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + enabled: false, + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithEnableInMemoryMode(test.args.enabled) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithEnableInMemoryMode(test.args.enabled) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithIndexPath(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + path string + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + path: "", + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + path: "", + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithIndexPath(test.args.path) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithIndexPath(test.args.path) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithAutoIndexCheckDuration(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + dur string + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithAutoIndexCheckDuration(test.args.dur) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithAutoIndexCheckDuration(test.args.dur) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithAutoIndexDurationLimit(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + dur string + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithAutoIndexDurationLimit(test.args.dur) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithAutoIndexDurationLimit(test.args.dur) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithAutoSaveIndexDuration(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + dur string + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithAutoSaveIndexDuration(test.args.dur) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithAutoSaveIndexDuration(test.args.dur) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithAutoIndexLength(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + l int + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + l: 0, + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + l: 0, + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithAutoIndexLength(test.args.l) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithAutoIndexLength(test.args.l) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithInitialDelayMaxDuration(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + dur string + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithInitialDelayMaxDuration(test.args.dur) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithInitialDelayMaxDuration(test.args.dur) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithMinLoadIndexTimeout(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + dur string + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithMinLoadIndexTimeout(test.args.dur) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithMinLoadIndexTimeout(test.args.dur) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithMaxLoadIndexTimeout(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + dur string + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithMaxLoadIndexTimeout(test.args.dur) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithMaxLoadIndexTimeout(test.args.dur) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithLoadIndexTimeoutFactor(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + dur string + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + dur: "", + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithLoadIndexTimeoutFactor(test.args.dur) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithLoadIndexTimeoutFactor(test.args.dur) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithDefaultPoolSize(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + ps uint32 + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + ps: 0, + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + ps: 0, + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithDefaultPoolSize(test.args.ps) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithDefaultPoolSize(test.args.ps) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithDefaultRadius(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + rad float32 + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + rad: 0, + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + rad: 0, + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithDefaultRadius(test.args.rad) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithDefaultRadius(test.args.rad) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithDefaultEpsilon(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + epsilon float32 + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + epsilon: 0, + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + epsilon: 0, + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithDefaultEpsilon(test.args.epsilon) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithDefaultEpsilon(test.args.epsilon) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} diff --git a/pkg/agent/core/ngt/usecase/agentd.go b/pkg/agent/core/ngt/usecase/agentd.go index d1220b7581..97a6a775ba 100644 --- a/pkg/agent/core/ngt/usecase/agentd.go +++ b/pkg/agent/core/ngt/usecase/agentd.go @@ -46,7 +46,23 @@ type run struct { } func New(cfg *config.Data) (r runner.Runner, err error) { - ngt, err := service.New(cfg.NGT) + ngt, err := service.New( + cfg.NGT, + service.WithErrGroup(errgroup.Get()), + service.WithEnableInMemoryMode(cfg.NGT.EnableInMemoryMode), + service.WithIndexPath(cfg.NGT.IndexPath), + service.WithAutoIndexCheckDuration(cfg.NGT.AutoIndexCheckDuration), + service.WithAutoIndexDurationLimit(cfg.NGT.AutoIndexDurationLimit), + service.WithAutoSaveIndexDuration(cfg.NGT.AutoSaveIndexDuration), + service.WithAutoIndexLength(cfg.NGT.AutoIndexLength), + service.WithInitialDelayMaxDuration(cfg.NGT.InitialDelayMaxDuration), + service.WithMinLoadIndexTimeout(cfg.NGT.MinLoadIndexTimeout), + service.WithMaxLoadIndexTimeout(cfg.NGT.MaxLoadIndexTimeout), + service.WithLoadIndexTimeoutFactor(cfg.NGT.LoadIndexTimeoutFactor), + service.WithDefaultPoolSize(cfg.NGT.DefaultPoolSize), + service.WithDefaultRadius(cfg.NGT.DefaultRadius), + service.WithDefaultEpsilon(cfg.NGT.DefaultEpsilon), + ) if err != nil { return nil, err } diff --git a/pkg/agent/internal/metadata/metadata.go b/pkg/agent/internal/metadata/metadata.go new file mode 100644 index 0000000000..a079a5d208 --- /dev/null +++ b/pkg/agent/internal/metadata/metadata.go @@ -0,0 +1,64 @@ +// +// Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package metadata provides agent metadata structs and info. +package metadata + +import ( + "os" + + "github.com/vdaas/vald/internal/encoding/json" + "github.com/vdaas/vald/internal/file" +) + +const ( + AgentMetadataFileName = "metadata.json" +) + +type Metadata struct { + IsInvalid bool `json:"is_invalid" yaml:"is_invalid"` + NGT *NGT `json:"ngt,omitempty" yaml:"ngt"` +} + +type NGT struct { + IndexCount uint64 `json:"index_count" yaml:"index_count"` +} + +func Load(path string) (*Metadata, error) { + f, err := file.Open(path, os.O_RDONLY|os.O_SYNC, os.ModePerm) + if err != nil { + return nil, err + } + defer f.Close() + + var meta Metadata + err = json.Decode(f, &meta) + if err != nil { + return nil, err + } + + return &meta, nil +} + +func Store(path string, meta *Metadata) error { + f, err := file.Open(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer f.Close() + + return json.Encode(f, &meta) +} diff --git a/pkg/agent/sidecar/service/observer/observer.go b/pkg/agent/sidecar/service/observer/observer.go index bddf2a238f..fdf23397bb 100644 --- a/pkg/agent/sidecar/service/observer/observer.go +++ b/pkg/agent/sidecar/service/observer/observer.go @@ -25,6 +25,7 @@ import ( "path/filepath" "reflect" "sync" + "syscall" "time" "github.com/vdaas/vald/internal/errgroup" @@ -34,6 +35,7 @@ import ( "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/pkg/agent/internal/metadata" "github.com/vdaas/vald/pkg/agent/sidecar/service/storage" ) @@ -48,8 +50,13 @@ type observer struct { eg errgroup.Group checkDuration time.Duration + metadataPath string + postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool + storage storage.Storage ch chan struct{} @@ -82,15 +89,19 @@ func (o *observer) Start(ctx context.Context) (<-chan error, error) { var wech, tech, sech, bech <-chan error var err error - wech, err = o.w.Start(ctx) - if err != nil { - return nil, err + if o.watchEnabled { + wech, err = o.w.Start(ctx) + if err != nil { + return nil, err + } } - tech, err = o.startTicker(ctx) - if err != nil { - close(ech) - return nil, err + if o.tickerEnabled { + tech, err = o.startTicker(ctx) + if err != nil { + close(ech) + return nil, err + } } sech, err = o.storage.Start(ctx) @@ -139,12 +150,30 @@ func (o *observer) PostStop(ctx context.Context) (err error) { return nil } - ticker := time.NewTicker(o.postStopTimeout / 5) - defer ticker.Stop() + backup := func() error { + metadata, err := metadata.Load(o.metadataPath) + if err != nil { + return err + } + + if metadata.IsInvalid { + log.Warn("backup skipped because the files are invalid") + return nil + } + + return o.backup(ctx) + } t := time.Now() + ch := make(chan struct{}, 1) + defer close(ch) f := func(ctx context.Context, name string) error { + if name == o.metadataPath { + ch <- struct{}{} + return nil + } + t = time.Now() return nil } @@ -173,13 +202,23 @@ func (o *observer) PostStop(ctx context.Context) (err error) { } }() + ticker := time.NewTicker(func() time.Duration { + if o.postStopTimeout/5 < 2*time.Second { + return o.postStopTimeout / 5 + } + return 2 * time.Second + }()) + defer ticker.Stop() + for { select { case <-ctx.Done(): return finalize() + case <-ch: + return backup() case <-ticker.C: if time.Since(t) > o.postStopTimeout { - return o.backup(ctx) + return backup() } } } @@ -206,6 +245,17 @@ func (o *observer) startTicker(ctx context.Context) (<-chan error, error) { case <-ctx.Done(): return finalize() case <-ct.C: + metadata, err := metadata.Load(o.metadataPath) + if err != nil { + ech <- err + continue + } + + if metadata.IsInvalid { + log.Warn("backup skipped because the files are invalid") + continue + } + err = o.requestBackup(ctx) if err != nil { ech <- err @@ -268,7 +318,16 @@ func (o *observer) onWrite(ctx context.Context, name string) error { } }() - return o.requestBackup(ctx) + ok, err := o.isValidMetadata(name) + if err != nil { + return err + } + + if ok { + return o.requestBackup(ctx) + } + + return o.terminate() } func (o *observer) onCreate(ctx context.Context, name string) error { @@ -279,7 +338,40 @@ func (o *observer) onCreate(ctx context.Context, name string) error { } }() - return o.requestBackup(ctx) + ok, err := o.isValidMetadata(name) + if err != nil { + return err + } + + if ok { + return o.requestBackup(ctx) + } + + return o.terminate() +} + +func (o *observer) isValidMetadata(name string) (bool, error) { + if name != o.metadataPath { + return false, nil + } + + metadata, err := metadata.Load(o.metadataPath) + if err != nil { + return false, err + } + + return !metadata.IsInvalid, nil +} + +func (o *observer) terminate() error { + log.Error("the process will be terminated because the files are invalid") + + p, err := os.FindProcess(os.Getpid()) + if err != nil { + return err + } + + return p.Signal(syscall.SIGTERM) } func (o *observer) requestBackup(ctx context.Context) error { diff --git a/pkg/agent/sidecar/service/observer/observer_test.go b/pkg/agent/sidecar/service/observer/observer_test.go index f921af0387..eb83686a0d 100644 --- a/pkg/agent/sidecar/service/observer/observer_test.go +++ b/pkg/agent/sidecar/service/observer/observer_test.go @@ -85,7 +85,7 @@ func TestNew(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -110,12 +110,16 @@ func Test_observer_Start(t *testing.T) { ctx context.Context } type fields struct { - w watch.Watcher - dir string - eg errgroup.Group - checkDuration time.Duration - storage storage.Storage - ch chan struct{} + w watch.Watcher + dir string + eg errgroup.Group + checkDuration time.Duration + metadataPath string + postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool + storage storage.Storage + ch chan struct{} } type want struct { want <-chan error @@ -152,6 +156,10 @@ func Test_observer_Start(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -173,6 +181,10 @@ func Test_observer_Start(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -185,7 +197,7 @@ func Test_observer_Start(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -196,12 +208,16 @@ func Test_observer_Start(t *testing.T) { test.checkFunc = defaultCheckFunc } o := &observer{ - w: test.fields.w, - dir: test.fields.dir, - eg: test.fields.eg, - checkDuration: test.fields.checkDuration, - storage: test.fields.storage, - ch: test.fields.ch, + w: test.fields.w, + dir: test.fields.dir, + eg: test.fields.eg, + checkDuration: test.fields.checkDuration, + metadataPath: test.fields.metadataPath, + postStopTimeout: test.fields.postStopTimeout, + watchEnabled: test.fields.watchEnabled, + tickerEnabled: test.fields.tickerEnabled, + storage: test.fields.storage, + ch: test.fields.ch, } got, err := o.Start(test.args.ctx) @@ -213,17 +229,141 @@ func Test_observer_Start(t *testing.T) { } } +func Test_observer_PostStop(t *testing.T) { + type args struct { + ctx context.Context + } + type fields struct { + w watch.Watcher + dir string + eg errgroup.Group + checkDuration time.Duration + metadataPath string + postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool + storage storage.Storage + ch chan struct{} + } + type want struct { + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + ctx: nil, + }, + fields: fields { + w: nil, + dir: "", + eg: nil, + checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, + storage: nil, + ch: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + ctx: nil, + }, + fields: fields { + w: nil, + dir: "", + eg: nil, + checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, + storage: nil, + ch: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + o := &observer{ + w: test.fields.w, + dir: test.fields.dir, + eg: test.fields.eg, + checkDuration: test.fields.checkDuration, + metadataPath: test.fields.metadataPath, + postStopTimeout: test.fields.postStopTimeout, + watchEnabled: test.fields.watchEnabled, + tickerEnabled: test.fields.tickerEnabled, + storage: test.fields.storage, + ch: test.fields.ch, + } + + err := o.PostStop(test.args.ctx) + if err := test.checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + func Test_observer_startTicker(t *testing.T) { type args struct { ctx context.Context } type fields struct { - w watch.Watcher - dir string - eg errgroup.Group - checkDuration time.Duration - storage storage.Storage - ch chan struct{} + w watch.Watcher + dir string + eg errgroup.Group + checkDuration time.Duration + metadataPath string + postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool + storage storage.Storage + ch chan struct{} } type want struct { want <-chan error @@ -260,6 +400,10 @@ func Test_observer_startTicker(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -281,6 +425,10 @@ func Test_observer_startTicker(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -293,7 +441,7 @@ func Test_observer_startTicker(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -304,12 +452,16 @@ func Test_observer_startTicker(t *testing.T) { test.checkFunc = defaultCheckFunc } o := &observer{ - w: test.fields.w, - dir: test.fields.dir, - eg: test.fields.eg, - checkDuration: test.fields.checkDuration, - storage: test.fields.storage, - ch: test.fields.ch, + w: test.fields.w, + dir: test.fields.dir, + eg: test.fields.eg, + checkDuration: test.fields.checkDuration, + metadataPath: test.fields.metadataPath, + postStopTimeout: test.fields.postStopTimeout, + watchEnabled: test.fields.watchEnabled, + tickerEnabled: test.fields.tickerEnabled, + storage: test.fields.storage, + ch: test.fields.ch, } got, err := o.startTicker(test.args.ctx) @@ -326,12 +478,16 @@ func Test_observer_startBackupLoop(t *testing.T) { ctx context.Context } type fields struct { - w watch.Watcher - dir string - eg errgroup.Group - checkDuration time.Duration - storage storage.Storage - ch chan struct{} + w watch.Watcher + dir string + eg errgroup.Group + checkDuration time.Duration + metadataPath string + postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool + storage storage.Storage + ch chan struct{} } type want struct { want <-chan error @@ -368,6 +524,10 @@ func Test_observer_startBackupLoop(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -389,6 +549,10 @@ func Test_observer_startBackupLoop(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -401,7 +565,7 @@ func Test_observer_startBackupLoop(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -412,12 +576,16 @@ func Test_observer_startBackupLoop(t *testing.T) { test.checkFunc = defaultCheckFunc } o := &observer{ - w: test.fields.w, - dir: test.fields.dir, - eg: test.fields.eg, - checkDuration: test.fields.checkDuration, - storage: test.fields.storage, - ch: test.fields.ch, + w: test.fields.w, + dir: test.fields.dir, + eg: test.fields.eg, + checkDuration: test.fields.checkDuration, + metadataPath: test.fields.metadataPath, + postStopTimeout: test.fields.postStopTimeout, + watchEnabled: test.fields.watchEnabled, + tickerEnabled: test.fields.tickerEnabled, + storage: test.fields.storage, + ch: test.fields.ch, } got, err := o.startBackupLoop(test.args.ctx) @@ -435,12 +603,16 @@ func Test_observer_onWrite(t *testing.T) { name string } type fields struct { - w watch.Watcher - dir string - eg errgroup.Group - checkDuration time.Duration - storage storage.Storage - ch chan struct{} + w watch.Watcher + dir string + eg errgroup.Group + checkDuration time.Duration + metadataPath string + postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool + storage storage.Storage + ch chan struct{} } type want struct { err error @@ -474,6 +646,10 @@ func Test_observer_onWrite(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -496,6 +672,10 @@ func Test_observer_onWrite(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -508,7 +688,7 @@ func Test_observer_onWrite(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -519,12 +699,16 @@ func Test_observer_onWrite(t *testing.T) { test.checkFunc = defaultCheckFunc } o := &observer{ - w: test.fields.w, - dir: test.fields.dir, - eg: test.fields.eg, - checkDuration: test.fields.checkDuration, - storage: test.fields.storage, - ch: test.fields.ch, + w: test.fields.w, + dir: test.fields.dir, + eg: test.fields.eg, + checkDuration: test.fields.checkDuration, + metadataPath: test.fields.metadataPath, + postStopTimeout: test.fields.postStopTimeout, + watchEnabled: test.fields.watchEnabled, + tickerEnabled: test.fields.tickerEnabled, + storage: test.fields.storage, + ch: test.fields.ch, } err := o.onWrite(test.args.ctx, test.args.name) @@ -542,12 +726,16 @@ func Test_observer_onCreate(t *testing.T) { name string } type fields struct { - w watch.Watcher - dir string - eg errgroup.Group - checkDuration time.Duration - storage storage.Storage - ch chan struct{} + w watch.Watcher + dir string + eg errgroup.Group + checkDuration time.Duration + metadataPath string + postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool + storage storage.Storage + ch chan struct{} } type want struct { err error @@ -581,6 +769,10 @@ func Test_observer_onCreate(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -603,6 +795,10 @@ func Test_observer_onCreate(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -615,7 +811,7 @@ func Test_observer_onCreate(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -626,12 +822,16 @@ func Test_observer_onCreate(t *testing.T) { test.checkFunc = defaultCheckFunc } o := &observer{ - w: test.fields.w, - dir: test.fields.dir, - eg: test.fields.eg, - checkDuration: test.fields.checkDuration, - storage: test.fields.storage, - ch: test.fields.ch, + w: test.fields.w, + dir: test.fields.dir, + eg: test.fields.eg, + checkDuration: test.fields.checkDuration, + metadataPath: test.fields.metadataPath, + postStopTimeout: test.fields.postStopTimeout, + watchEnabled: test.fields.watchEnabled, + tickerEnabled: test.fields.tickerEnabled, + storage: test.fields.storage, + ch: test.fields.ch, } err := o.onCreate(test.args.ctx, test.args.name) @@ -643,34 +843,42 @@ func Test_observer_onCreate(t *testing.T) { } } -func Test_observer_requestBackup(t *testing.T) { +func Test_observer_isValidMetadata(t *testing.T) { type args struct { - ctx context.Context + name string } type fields struct { - w watch.Watcher - dir string - eg errgroup.Group - checkDuration time.Duration - storage storage.Storage - ch chan struct{} + w watch.Watcher + dir string + eg errgroup.Group + checkDuration time.Duration + metadataPath string + postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool + storage storage.Storage + ch chan struct{} } type want struct { - err error + want bool + err error } type test struct { name string args args fields fields want want - checkFunc func(want, error) error + checkFunc func(want, bool, error) error beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, err error) error { + defaultCheckFunc := func(w want, got bool, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got error = %v, want %v", err, w.err) } + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got = %v, want %v", got, w.want) + } return nil } tests := []test{ @@ -679,13 +887,17 @@ func Test_observer_requestBackup(t *testing.T) { { name: "test_case_1", args: args { - ctx: nil, + name: "", }, fields: fields { w: nil, dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -700,13 +912,17 @@ func Test_observer_requestBackup(t *testing.T) { return test { name: "test_case_2", args: args { - ctx: nil, + name: "", }, fields: fields { w: nil, dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -719,7 +935,7 @@ func Test_observer_requestBackup(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -730,15 +946,129 @@ func Test_observer_requestBackup(t *testing.T) { test.checkFunc = defaultCheckFunc } o := &observer{ - w: test.fields.w, - dir: test.fields.dir, - eg: test.fields.eg, - checkDuration: test.fields.checkDuration, - storage: test.fields.storage, - ch: test.fields.ch, + w: test.fields.w, + dir: test.fields.dir, + eg: test.fields.eg, + checkDuration: test.fields.checkDuration, + metadataPath: test.fields.metadataPath, + postStopTimeout: test.fields.postStopTimeout, + watchEnabled: test.fields.watchEnabled, + tickerEnabled: test.fields.tickerEnabled, + storage: test.fields.storage, + ch: test.fields.ch, } - err := o.requestBackup(test.args.ctx) + got, err := o.isValidMetadata(test.args.name) + if err := test.checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + + }) + } +} + +func Test_observer_terminate(t *testing.T) { + type fields struct { + w watch.Watcher + dir string + eg errgroup.Group + checkDuration time.Duration + metadataPath string + postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool + storage storage.Storage + ch chan struct{} + } + type want struct { + err error + } + type test struct { + name string + fields fields + want want + checkFunc func(want, error) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + w: nil, + dir: "", + eg: nil, + checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, + storage: nil, + ch: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + w: nil, + dir: "", + eg: nil, + checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, + storage: nil, + ch: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + o := &observer{ + w: test.fields.w, + dir: test.fields.dir, + eg: test.fields.eg, + checkDuration: test.fields.checkDuration, + metadataPath: test.fields.metadataPath, + postStopTimeout: test.fields.postStopTimeout, + watchEnabled: test.fields.watchEnabled, + tickerEnabled: test.fields.tickerEnabled, + storage: test.fields.storage, + ch: test.fields.ch, + } + + err := o.terminate() if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -747,17 +1077,21 @@ func Test_observer_requestBackup(t *testing.T) { } } -func Test_observer_backup(t *testing.T) { +func Test_observer_requestBackup(t *testing.T) { type args struct { ctx context.Context } type fields struct { - w watch.Watcher - dir string - eg errgroup.Group - checkDuration time.Duration - storage storage.Storage - ch chan struct{} + w watch.Watcher + dir string + eg errgroup.Group + checkDuration time.Duration + metadataPath string + postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool + storage storage.Storage + ch chan struct{} } type want struct { err error @@ -790,6 +1124,10 @@ func Test_observer_backup(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -811,6 +1149,10 @@ func Test_observer_backup(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", + postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -823,7 +1165,7 @@ func Test_observer_backup(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -834,15 +1176,19 @@ func Test_observer_backup(t *testing.T) { test.checkFunc = defaultCheckFunc } o := &observer{ - w: test.fields.w, - dir: test.fields.dir, - eg: test.fields.eg, - checkDuration: test.fields.checkDuration, - storage: test.fields.storage, - ch: test.fields.ch, + w: test.fields.w, + dir: test.fields.dir, + eg: test.fields.eg, + checkDuration: test.fields.checkDuration, + metadataPath: test.fields.metadataPath, + postStopTimeout: test.fields.postStopTimeout, + watchEnabled: test.fields.watchEnabled, + tickerEnabled: test.fields.tickerEnabled, + storage: test.fields.storage, + ch: test.fields.ch, } - err := o.backup(test.args.ctx) + err := o.requestBackup(test.args.ctx) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } @@ -851,7 +1197,7 @@ func Test_observer_backup(t *testing.T) { } } -func Test_observer_PostStop(t *testing.T) { +func Test_observer_backup(t *testing.T) { type args struct { ctx context.Context } @@ -860,7 +1206,10 @@ func Test_observer_PostStop(t *testing.T) { dir string eg errgroup.Group checkDuration time.Duration + metadataPath string postStopTimeout time.Duration + watchEnabled bool + tickerEnabled bool storage storage.Storage ch chan struct{} } @@ -895,7 +1244,10 @@ func Test_observer_PostStop(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -917,7 +1269,10 @@ func Test_observer_PostStop(t *testing.T) { dir: "", eg: nil, checkDuration: nil, + metadataPath: "", postStopTimeout: nil, + watchEnabled: false, + tickerEnabled: false, storage: nil, ch: nil, }, @@ -930,7 +1285,7 @@ func Test_observer_PostStop(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -945,12 +1300,15 @@ func Test_observer_PostStop(t *testing.T) { dir: test.fields.dir, eg: test.fields.eg, checkDuration: test.fields.checkDuration, + metadataPath: test.fields.metadataPath, postStopTimeout: test.fields.postStopTimeout, + watchEnabled: test.fields.watchEnabled, + tickerEnabled: test.fields.tickerEnabled, storage: test.fields.storage, ch: test.fields.ch, } - err := o.PostStop(test.args.ctx) + err := o.backup(test.args.ctx) if err := test.checkFunc(test.want, err); err != nil { tt.Errorf("error = %v", err) } diff --git a/pkg/agent/sidecar/service/observer/option.go b/pkg/agent/sidecar/service/observer/option.go index 61ebe56b09..148f890f71 100644 --- a/pkg/agent/sidecar/service/observer/option.go +++ b/pkg/agent/sidecar/service/observer/option.go @@ -18,8 +18,11 @@ package observer import ( + "path/filepath" + "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/timeutil" + "github.com/vdaas/vald/pkg/agent/internal/metadata" "github.com/vdaas/vald/pkg/agent/sidecar/service/storage" ) @@ -29,7 +32,9 @@ var ( defaultOpts = []Option{ WithErrGroup(errgroup.Get()), WithBackupDuration("10m"), - WithPostStopTimeout("20s"), + WithPostStopTimeout("2m"), + WithWatch(true), + WithTicker(true), } ) @@ -61,6 +66,22 @@ func WithPostStopTimeout(dur string) Option { } } +func WithWatch(enabled bool) Option { + return func(o *observer) error { + o.watchEnabled = enabled + + return nil + } +} + +func WithTicker(enabled bool) Option { + return func(o *observer) error { + o.tickerEnabled = enabled + + return nil + } +} + func WithErrGroup(eg errgroup.Group) Option { return func(o *observer) error { if eg != nil { @@ -77,6 +98,7 @@ func WithDir(dir string) Option { } o.dir = dir + o.metadataPath = filepath.Join(dir, metadata.AgentMetadataFileName) return nil } diff --git a/pkg/agent/sidecar/service/observer/option_test.go b/pkg/agent/sidecar/service/observer/option_test.go index a75bce7e76..6bebf5c016 100644 --- a/pkg/agent/sidecar/service/observer/option_test.go +++ b/pkg/agent/sidecar/service/observer/option_test.go @@ -26,6 +26,7 @@ import ( ) func TestWithBackupDuration(t *testing.T) { + // Change interface type to the type of object you are testing type T = interface{} type args struct { dur string @@ -63,7 +64,7 @@ func TestWithBackupDuration(t *testing.T) { /* defaultCheckFunc := func(w want, obj *T) error { if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.c) + return errors.Errorf("got = %v, want %v", obj, w.obj) } return nil } @@ -101,7 +102,7 @@ func TestWithBackupDuration(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -122,7 +123,7 @@ func TestWithBackupDuration(t *testing.T) { } */ - // Uncomment this block if the option returns an error, otherwise delete it + // Uncomment this block if the option do not return an error, otherwise delete it /* if test.checkFunc == nil { test.checkFunc = defaultCheckFunc @@ -130,7 +131,7 @@ func TestWithBackupDuration(t *testing.T) { got := WithBackupDuration(test.args.dur) obj := new(T) got(obj) - if err := test.checkFunc(tt.want, obj); err != nil { + if err := test.checkFunc(test.want, obj); err != nil { tt.Errorf("error = %v", err) } */ @@ -138,7 +139,8 @@ func TestWithBackupDuration(t *testing.T) { } } -func TestWithBackupDurationLimit(t *testing.T) { +func TestWithPostStopTimeout(t *testing.T) { + // Change interface type to the type of object you are testing type T = interface{} type args struct { dur string @@ -176,7 +178,7 @@ func TestWithBackupDurationLimit(t *testing.T) { /* defaultCheckFunc := func(w want, obj *T) error { if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.c) + return errors.Errorf("got = %v, want %v", obj, w.obj) } return nil } @@ -214,7 +216,7 @@ func TestWithBackupDurationLimit(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -228,22 +230,22 @@ func TestWithBackupDurationLimit(t *testing.T) { test.checkFunc = defaultCheckFunc } - got := WithBackupDurationLimit(test.args.dur) + got := WithPostStopTimeout(test.args.dur) obj := new(T) if err := test.checkFunc(test.want, obj, got(obj)); err != nil { tt.Errorf("error = %v", err) } */ - // Uncomment this block if the option returns an error, otherwise delete it + // Uncomment this block if the option do not return an error, otherwise delete it /* if test.checkFunc == nil { test.checkFunc = defaultCheckFunc } - got := WithBackupDurationLimit(test.args.dur) + got := WithPostStopTimeout(test.args.dur) obj := new(T) got(obj) - if err := test.checkFunc(tt.want, obj); err != nil { + if err := test.checkFunc(test.want, obj); err != nil { tt.Errorf("error = %v", err) } */ @@ -251,10 +253,11 @@ func TestWithBackupDurationLimit(t *testing.T) { } } -func TestWithErrGroup(t *testing.T) { +func TestWithWatch(t *testing.T) { + // Change interface type to the type of object you are testing type T = interface{} type args struct { - eg errgroup.Group + enabled bool } type want struct { obj *T @@ -289,7 +292,7 @@ func TestWithErrGroup(t *testing.T) { /* defaultCheckFunc := func(w want, obj *T) error { if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.c) + return errors.Errorf("got = %v, want %v", obj, w.obj) } return nil } @@ -301,7 +304,7 @@ func TestWithErrGroup(t *testing.T) { { name: "test_case_1", args: args { - eg: nil, + enabled: false, }, want: want { obj: new(T), @@ -315,7 +318,7 @@ func TestWithErrGroup(t *testing.T) { return test { name: "test_case_2", args: args { - eg: nil, + enabled: false, }, want: want { obj: new(T), @@ -327,7 +330,7 @@ func TestWithErrGroup(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -341,22 +344,22 @@ func TestWithErrGroup(t *testing.T) { test.checkFunc = defaultCheckFunc } - got := WithErrGroup(test.args.eg) + got := WithWatch(test.args.enabled) obj := new(T) if err := test.checkFunc(test.want, obj, got(obj)); err != nil { tt.Errorf("error = %v", err) } */ - // Uncomment this block if the option returns an error, otherwise delete it + // Uncomment this block if the option do not return an error, otherwise delete it /* if test.checkFunc == nil { test.checkFunc = defaultCheckFunc } - got := WithErrGroup(test.args.eg) + got := WithWatch(test.args.enabled) obj := new(T) got(obj) - if err := test.checkFunc(tt.want, obj); err != nil { + if err := test.checkFunc(test.want, obj); err != nil { tt.Errorf("error = %v", err) } */ @@ -364,10 +367,11 @@ func TestWithErrGroup(t *testing.T) { } } -func TestWithDir(t *testing.T) { +func TestWithTicker(t *testing.T) { + // Change interface type to the type of object you are testing type T = interface{} type args struct { - dir string + enabled bool } type want struct { obj *T @@ -402,7 +406,7 @@ func TestWithDir(t *testing.T) { /* defaultCheckFunc := func(w want, obj *T) error { if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.c) + return errors.Errorf("got = %v, want %v", obj, w.obj) } return nil } @@ -414,7 +418,7 @@ func TestWithDir(t *testing.T) { { name: "test_case_1", args: args { - dir: "", + enabled: false, }, want: want { obj: new(T), @@ -428,7 +432,7 @@ func TestWithDir(t *testing.T) { return test { name: "test_case_2", args: args { - dir: "", + enabled: false, }, want: want { obj: new(T), @@ -440,7 +444,7 @@ func TestWithDir(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -454,22 +458,22 @@ func TestWithDir(t *testing.T) { test.checkFunc = defaultCheckFunc } - got := WithDir(test.args.dir) + got := WithTicker(test.args.enabled) obj := new(T) if err := test.checkFunc(test.want, obj, got(obj)); err != nil { tt.Errorf("error = %v", err) } */ - // Uncomment this block if the option returns an error, otherwise delete it + // Uncomment this block if the option do not return an error, otherwise delete it /* if test.checkFunc == nil { test.checkFunc = defaultCheckFunc } - got := WithDir(test.args.dir) + got := WithTicker(test.args.enabled) obj := new(T) got(obj) - if err := test.checkFunc(tt.want, obj); err != nil { + if err := test.checkFunc(test.want, obj); err != nil { tt.Errorf("error = %v", err) } */ @@ -477,10 +481,11 @@ func TestWithDir(t *testing.T) { } } -func TestWithBlobStorage(t *testing.T) { +func TestWithErrGroup(t *testing.T) { + // Change interface type to the type of object you are testing type T = interface{} type args struct { - storage storage.Storage + eg errgroup.Group } type want struct { obj *T @@ -515,7 +520,7 @@ func TestWithBlobStorage(t *testing.T) { /* defaultCheckFunc := func(w want, obj *T) error { if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.c) + return errors.Errorf("got = %v, want %v", obj, w.obj) } return nil } @@ -527,7 +532,7 @@ func TestWithBlobStorage(t *testing.T) { { name: "test_case_1", args: args { - storage: nil, + eg: nil, }, want: want { obj: new(T), @@ -541,7 +546,7 @@ func TestWithBlobStorage(t *testing.T) { return test { name: "test_case_2", args: args { - storage: nil, + eg: nil, }, want: want { obj: new(T), @@ -553,7 +558,7 @@ func TestWithBlobStorage(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -567,22 +572,22 @@ func TestWithBlobStorage(t *testing.T) { test.checkFunc = defaultCheckFunc } - got := WithBlobStorage(test.args.storage) + got := WithErrGroup(test.args.eg) obj := new(T) if err := test.checkFunc(test.want, obj, got(obj)); err != nil { tt.Errorf("error = %v", err) } */ - // Uncomment this block if the option returns an error, otherwise delete it + // Uncomment this block if the option do not return an error, otherwise delete it /* if test.checkFunc == nil { test.checkFunc = defaultCheckFunc } - got := WithBlobStorage(test.args.storage) + got := WithErrGroup(test.args.eg) obj := new(T) got(obj) - if err := test.checkFunc(tt.want, obj); err != nil { + if err := test.checkFunc(test.want, obj); err != nil { tt.Errorf("error = %v", err) } */ @@ -590,10 +595,11 @@ func TestWithBlobStorage(t *testing.T) { } } -func TestWithPostStopTimeout(t *testing.T) { +func TestWithDir(t *testing.T) { + // Change interface type to the type of object you are testing type T = interface{} type args struct { - dur string + dir string } type want struct { obj *T @@ -628,7 +634,7 @@ func TestWithPostStopTimeout(t *testing.T) { /* defaultCheckFunc := func(w want, obj *T) error { if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.c) + return errors.Errorf("got = %v, want %v", obj, w.obj) } return nil } @@ -640,7 +646,7 @@ func TestWithPostStopTimeout(t *testing.T) { { name: "test_case_1", args: args { - dur: "", + dir: "", }, want: want { obj: new(T), @@ -654,7 +660,7 @@ func TestWithPostStopTimeout(t *testing.T) { return test { name: "test_case_2", args: args { - dur: "", + dir: "", }, want: want { obj: new(T), @@ -666,7 +672,7 @@ func TestWithPostStopTimeout(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(t) + defer goleak.VerifyNone(tt) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -680,22 +686,136 @@ func TestWithPostStopTimeout(t *testing.T) { test.checkFunc = defaultCheckFunc } - got := WithPostStopTimeout(test.args.dur) + got := WithDir(test.args.dir) obj := new(T) if err := test.checkFunc(test.want, obj, got(obj)); err != nil { tt.Errorf("error = %v", err) } */ + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithDir(test.args.dir) + obj := new(T) + got(obj) + if err := test.checkFunc(test.want, obj); err != nil { + tt.Errorf("error = %v", err) + } + */ + }) + } +} + +func TestWithBlobStorage(t *testing.T) { + // Change interface type to the type of object you are testing + type T = interface{} + type args struct { + storage storage.Storage + } + type want struct { + obj *T + // Uncomment this line if the option returns an error, otherwise delete it + // err error + } + type test struct { + name string + args args + want want + // Use the first line if the option returns an error. otherwise use the second line + // checkFunc func(want, *T, error) error + // checkFunc func(want, *T) error + beforeFunc func(args) + afterFunc func(args) + } + + // Uncomment this block if the option returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + // Uncomment this block if the option do not returns an error, otherwise delete it + /* + defaultCheckFunc := func(w want, obj *T) error { + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + */ + + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + storage: nil, + }, + want: want { + obj: new(T), + }, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + storage: nil, + }, + want: want { + obj: new(T), + }, + } + }(), + */ + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + // Uncomment this block if the option returns an error, otherwise delete it /* if test.checkFunc == nil { test.checkFunc = defaultCheckFunc } - got := WithPostStopTimeout(test.args.dur) + + got := WithBlobStorage(test.args.storage) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + */ + + // Uncomment this block if the option do not return an error, otherwise delete it + /* + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + got := WithBlobStorage(test.args.storage) obj := new(T) got(obj) - if err := test.checkFunc(tt.want, obj); err != nil { + if err := test.checkFunc(test.want, obj); err != nil { tt.Errorf("error = %v", err) } */ diff --git a/pkg/agent/sidecar/service/restorer/restorer.go b/pkg/agent/sidecar/service/restorer/restorer.go index 791243d76a..7f84ee4326 100644 --- a/pkg/agent/sidecar/service/restorer/restorer.go +++ b/pkg/agent/sidecar/service/restorer/restorer.go @@ -29,6 +29,7 @@ import ( "github.com/vdaas/vald/internal/backoff" "github.com/vdaas/vald/internal/errgroup" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/file" ctxio "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/observability/trace" @@ -166,10 +167,16 @@ func (r *restorer) restore(ctx context.Context) (err error) { return err } - sr, err = ctxio.NewReaderWithContext(ctx, sr) + sr, err = ctxio.NewReadCloserWithContext(ctx, sr) if err != nil { return err } + defer func() { + e := sr.Close() + if e != nil { + log.Errorf("error on closing blob-storage reader: %s", e) + } + }() _, err = io.Copy(pw, sr) if err != nil { @@ -211,7 +218,14 @@ func (r *restorer) restore(ctx context.Context) (err error) { } } case tar.TypeReg: - f, err := os.OpenFile( + if _, err := os.Stat(target); err == nil { + log.Warn(errors.ErrFileAlreadyExists(target)) + return nil + } else if !os.IsNotExist(err) { + return err + } + + f, err := file.Open( target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode), @@ -227,7 +241,7 @@ func (r *restorer) restore(ctx context.Context) (err error) { _, err = io.Copy(fw, tr) if err != nil { - return err + return errors.Wrap(f.Close(), err.Error()) } err = f.Close() diff --git a/pkg/agent/sidecar/service/storage/storage.go b/pkg/agent/sidecar/service/storage/storage.go index d8dbb5347f..f4b1461eeb 100644 --- a/pkg/agent/sidecar/service/storage/storage.go +++ b/pkg/agent/sidecar/service/storage/storage.go @@ -33,7 +33,7 @@ import ( type Storage interface { Start(ctx context.Context) (<-chan error, error) - Reader(ctx context.Context) (io.Reader, error) + Reader(ctx context.Context) (io.ReadCloser, error) Writer(ctx context.Context) (io.WriteCloser, error) } @@ -135,7 +135,7 @@ func (b *bs) Start(ctx context.Context) (<-chan error, error) { return ech, nil } -func (b *bs) Reader(ctx context.Context) (r io.Reader, err error) { +func (b *bs) Reader(ctx context.Context) (r io.ReadCloser, err error) { r, err = b.bucket.Reader(ctx, b.filename+b.suffix) if err != nil { return nil, err diff --git a/pkg/agent/sidecar/service/storage/storage_test.go b/pkg/agent/sidecar/service/storage/storage_test.go index 4d05834999..66ceb86114 100644 --- a/pkg/agent/sidecar/service/storage/storage_test.go +++ b/pkg/agent/sidecar/service/storage/storage_test.go @@ -481,7 +481,7 @@ func Test_bs_Reader(t *testing.T) { compressor compress.Compressor } type want struct { - wantR io.Reader + wantR io.ReadCloser err error } type test struct { @@ -489,11 +489,11 @@ func Test_bs_Reader(t *testing.T) { args args fields fields want want - checkFunc func(want, io.Reader, error) error + checkFunc func(want, io.ReadCloser, error) error beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, gotR io.Reader, err error) error { + defaultCheckFunc := func(w want, gotR io.ReadCloser, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got error = %v, want %v", err, w.err) } diff --git a/pkg/agent/sidecar/usecase/sidecar/sidecar.go b/pkg/agent/sidecar/usecase/sidecar/sidecar.go index b133fd1436..3a1f90bb23 100644 --- a/pkg/agent/sidecar/usecase/sidecar/sidecar.go +++ b/pkg/agent/sidecar/usecase/sidecar/sidecar.go @@ -133,6 +133,8 @@ func New(cfg *config.Data) (r runner.Runner, err error) { so, err = observer.New( observer.WithErrGroup(eg), + observer.WithWatch(cfg.AgentSidecar.WatchEnabled), + observer.WithTicker(cfg.AgentSidecar.AutoBackupEnabled), observer.WithBackupDuration(cfg.AgentSidecar.AutoBackupDuration), observer.WithPostStopTimeout(cfg.AgentSidecar.PostStopTimeout), observer.WithDir(cfg.AgentSidecar.WatchDir),