Skip to content

Commit

Permalink
remove vcache for vald agent due to vcache delete timing control fail…
Browse files Browse the repository at this point in the history 
…ure and time ordered concurrent vector queue called vqueue

Signed-off-by: kpango <kpango@vdaas.org>
  • Loading branch information
@kpango
kpango committed Feb 19, 2021
1 parent 0fa80d1 commit 2d94bcb
Show file tree
Hide file tree
Showing 13 changed files with 2,446 additions and 2,385 deletions.
2 changes: 1 addition & 1 deletion cmd/agent/core/ngt/sample.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ ngt:
bulk_insert_chunk_size: 10
creation_edge_size: 20
default_epsilon: 0.01
default_pool_size: 10000
default_pool_size: 100
default_radius: -1
dimension: 6
distance_type: l2
Expand Down
8 changes: 4 additions & 4 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.16
replace (
cloud.google.com/go => cloud.google.com/go v0.77.0
github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0+incompatible
github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.37.14
github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.37.15
github.com/boltdb/bolt => github.com/boltdb/bolt v1.3.1
github.com/chzyer/logex => github.com/chzyer/logex v1.1.11-0.20170329064859-445be9e134b2
github.com/coreos/etcd => go.etcd.io/etcd v3.3.25+incompatible
Expand All @@ -21,7 +21,7 @@ replace (
github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.15.0
github.com/gorilla/websocket => github.com/gorilla/websocket v1.4.2
github.com/hailocab/go-hostpool => github.com/monzo/go-hostpool v0.0.0-20200724120130-287edbb29340
github.com/klauspost/compress => github.com/klauspost/compress v1.11.8-0.20210205131735-b5cd74455ec9
github.com/klauspost/compress => github.com/klauspost/compress v1.11.8-0.20210219140231-0afc4af7e6ba
github.com/tensorflow/tensorflow => github.com/tensorflow/tensorflow v2.1.2+incompatible
golang.org/x/crypto => golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df
google.golang.org/grpc => google.golang.org/grpc v1.35.0
Expand Down Expand Up @@ -75,11 +75,11 @@ require (
go.uber.org/zap v1.16.0
golang.org/x/net v0.0.0-20210119194325-5f4716e94777
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b
golang.org/x/sys v0.0.0-20210219172841-57ea560cfca1
gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614
gonum.org/v1/plot v0.8.1
google.golang.org/api v0.40.0
google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06
google.golang.org/genproto v0.0.0-20210219173056-d891e3cb3b5b
google.golang.org/grpc v1.35.0
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.20.4
Expand Down
16 changes: 8 additions & 8 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.37.14 h1:thuR1hd1doCvsaMDYDMhqCGSmw39bSvZaw+DPGhMm5w=
github.com/aws/aws-sdk-go v1.37.14/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.37.15 h1:W7l7gLLMcYRlg6a+uvf3Zz4jYwdqYzhe5ymqwWoOhp4=
github.com/aws/aws-sdk-go v1.37.15/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down Expand Up @@ -296,8 +296,8 @@ github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5Pt
github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.11.8-0.20210205131735-b5cd74455ec9 h1:OAhCmjDqWViQo/ZB+YeKShJ4DFrkmVRWRUGgzzJLs28=
github.com/klauspost/compress v1.11.8-0.20210205131735-b5cd74455ec9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.8-0.20210219140231-0afc4af7e6ba h1:QdsKSRMQoB/H6WIbEwhTV/SeafLuYELLmbxoKEf1En4=
github.com/klauspost/compress v1.11.8-0.20210219140231-0afc4af7e6ba/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
Expand Down Expand Up @@ -685,8 +685,8 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b h1:lAZ0/chPUDWwjqosYR0X4M490zQhMsiJ4K3DbA7o+3g=
golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210219172841-57ea560cfca1 h1:mDSj8NPponP6fRpRDblAGl5bpSHjPulHtk5lGl0gLSY=
golang.org/x/sys v0.0.0-20210219172841-57ea560cfca1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down Expand Up @@ -813,8 +813,8 @@ google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210212180131-e7f2df4ecc2d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06 h1:Px6YyLaNKEo5eoniIBAv6Es0jbvyEmSYqOac64iS2Rs=
google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210219173056-d891e3cb3b5b h1:zTeTu5p/EXQSqNJboHUw32wdNFYQTT9vSc+ibSpCoLk=
google.golang.org/genproto v0.0.0-20210219173056-d891e3cb3b5b/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
Expand Down
4 changes: 2 additions & 2 deletions internal/config/net_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ func TestDialer_Bind(t *testing.T) {
}
}

func TestTCP_Bind(t *testing.T) {
func TestNet_Bind(t *testing.T) {
type fields struct {
DNS *DNS
Dialer *Dialer
Expand Down Expand Up @@ -267,7 +267,7 @@ func TestTCP_Bind(t *testing.T) {
}
}

func TestTCP_Opts(t *testing.T) {
func TestNet_Opts(t *testing.T) {
type fields struct {
DNS *DNS
Dialer *Dialer
Expand Down
14 changes: 6 additions & 8 deletions internal/core/algorithm/ngt/ngt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,16 +499,14 @@ func (n *ngt) Remove(id uint) error {
}

// BulkRemove removes multiple index from NGT index.
func (n *ngt) BulkRemove(ids ...uint) error {
n.mu.Lock()
defer n.mu.Unlock()
for _, id := range ids {
if C.ngt_remove_index(n.index, C.ObjectID(id), n.ebuf) == ErrorCode {
ne := n.ebuf
return n.newGoError(ne)
func (n *ngt) BulkRemove(ids ...uint) (errs error) {
for i, id := range ids {
err := n.Remove(id)
if err != nil {
errs = errors.Wrapf(errs, "bulkremove error detected index number: %d,\tid: %d\terr: %v", i, id, err)
}
}
return nil
return errs
}

// GetVector returns vector stored in NGT index.
Expand Down
20 changes: 20 additions & 0 deletions internal/errors/vqueue.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
//
// Copyright (C) 2019-2021 vdaas.org vald team <vald@vdaas.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// Package errors provides error types and function
package errors

var ErrVQueueFinalizing = New("error vector queue is now finalizing...")
42 changes: 21 additions & 21 deletions pkg/agent/core/ngt/handler/grpc/handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -431,8 +431,8 @@ func Test_server_Search(t *testing.T) {
streamConcurrency int
}
type want struct {
want *payload.Search_Response
err error
wantRes *payload.Search_Response
err error
}
type test struct {
name string
Expand All @@ -443,12 +443,12 @@ func Test_server_Search(t *testing.T) {
beforeFunc func(args)
afterFunc func(args)
}
defaultCheckFunc := func(w want, got *payload.Search_Response, err error) error {
defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error {
if !errors.Is(err, w.err) {
return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
}
if !reflect.DeepEqual(got, w.want) {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
if !reflect.DeepEqual(gotRes, w.wantRes) {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes)
}
return nil
}
Expand Down Expand Up @@ -518,8 +518,8 @@ func Test_server_Search(t *testing.T) {
streamConcurrency: test.fields.streamConcurrency,
}

got, err := s.Search(test.args.ctx, test.args.req)
if err := test.checkFunc(test.want, got, err); err != nil {
gotRes, err := s.Search(test.args.ctx, test.args.req)
if err := test.checkFunc(test.want, gotRes, err); err != nil {
tt.Errorf("error = %v", err)
}
})
Expand All @@ -540,8 +540,8 @@ func Test_server_SearchByID(t *testing.T) {
streamConcurrency int
}
type want struct {
want *payload.Search_Response
err error
wantRes *payload.Search_Response
err error
}
type test struct {
name string
Expand All @@ -552,12 +552,12 @@ func Test_server_SearchByID(t *testing.T) {
beforeFunc func(args)
afterFunc func(args)
}
defaultCheckFunc := func(w want, got *payload.Search_Response, err error) error {
defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error {
if !errors.Is(err, w.err) {
return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
}
if !reflect.DeepEqual(got, w.want) {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
if !reflect.DeepEqual(gotRes, w.wantRes) {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes)
}
return nil
}
Expand Down Expand Up @@ -627,8 +627,8 @@ func Test_server_SearchByID(t *testing.T) {
streamConcurrency: test.fields.streamConcurrency,
}

got, err := s.SearchByID(test.args.ctx, test.args.req)
if err := test.checkFunc(test.want, got, err); err != nil {
gotRes, err := s.SearchByID(test.args.ctx, test.args.req)
if err := test.checkFunc(test.want, gotRes, err); err != nil {
tt.Errorf("error = %v", err)
}
})
Expand Down Expand Up @@ -1791,8 +1791,8 @@ func Test_server_Upsert(t *testing.T) {
streamConcurrency int
}
type want struct {
want *payload.Object_Location
err error
wantLoc *payload.Object_Location
err error
}
type test struct {
name string
Expand All @@ -1803,12 +1803,12 @@ func Test_server_Upsert(t *testing.T) {
beforeFunc func(args)
afterFunc func(args)
}
defaultCheckFunc := func(w want, got *payload.Object_Location, err error) error {
defaultCheckFunc := func(w want, gotLoc *payload.Object_Location, err error) error {
if !errors.Is(err, w.err) {
return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
}
if !reflect.DeepEqual(got, w.want) {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
if !reflect.DeepEqual(gotLoc, w.wantLoc) {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotLoc, w.wantLoc)
}
return nil
}
Expand Down Expand Up @@ -1878,8 +1878,8 @@ func Test_server_Upsert(t *testing.T) {
streamConcurrency: test.fields.streamConcurrency,
}

got, err := s.Upsert(test.args.ctx, test.args.req)
if err := test.checkFunc(test.want, got, err); err != nil {
gotLoc, err := s.Upsert(test.args.ctx, test.args.req)
if err := test.checkFunc(test.want, gotLoc, err); err != nil {
tt.Errorf("error = %v", err)
}
})
Expand Down
Loading

0 comments on commit 2d94bcb

Please sign in to comment.