Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[bugfix] change exists cheking for agent vqueue #1256

Merged
merged 3 commits into from
May 25, 2021
Merged

[bugfix] change exists cheking for agent vqueue #1256

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e43c479
[bugfix] change exists cheking for agent vqueue
kpango May 25, 2021
7791728
Merge branch 'master' into bugfix/agent-vqueue/change-existing-checking
kpango May 25, 2021
0c02c0f
fix
kpango May 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ module github.com/vdaas/vald
go 1.16

replace (
cloud.google.com/go => cloud.google.com/go v0.82.1-0.20210520180002-258980a039db
cloud.google.com/go/storage => cloud.google.com/go/storage v1.15.1-0.20210520180002-258980a039db
cloud.google.com/go => cloud.google.com/go v0.82.1-0.20210525054938-8bd87cbfee00
cloud.google.com/go/storage => cloud.google.com/go/storage v1.15.1-0.20210525054938-8bd87cbfee00
github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.1-0.20210210161804-c7f947c0610d+incompatible
github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.38.45
github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.38.46
github.com/boltdb/bolt => github.com/boltdb/bolt v1.3.1
github.com/chzyer/logex => github.com/chzyer/logex v1.1.11-0.20170329064859-445be9e134b2
github.com/coreos/etcd => go.etcd.io/etcd v3.3.25+incompatible
Expand All @@ -16,7 +16,7 @@ replace (
github.com/gocql/gocql => github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556
github.com/gogo/googleapis => github.com/gogo/googleapis v1.4.1
github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
github.com/google/go-cmp => github.com/google/go-cmp v0.5.5
github.com/google/go-cmp => github.com/google/go-cmp v0.5.6
github.com/google/pprof => github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a
github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5
github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.17.0
Expand All @@ -38,9 +38,9 @@ replace (
)

require (
cloud.google.com/go v0.81.0
cloud.google.com/go v0.82.0
cloud.google.com/go/storage v1.15.0
code.cloudfoundry.org/bytefmt v0.0.0-20210519212014-d85d569ae74a
code.cloudfoundry.org/bytefmt v0.0.0-20210524144015-27119551aaea
contrib.go.opencensus.io/exporter/jaeger v0.2.1
contrib.go.opencensus.io/exporter/prometheus v0.3.0
contrib.go.opencensus.io/exporter/stackdriver v0.13.6
Expand Down Expand Up @@ -77,15 +77,15 @@ require (
go.uber.org/goleak v1.1.10
go.uber.org/zap v1.16.0
gocloud.dev v0.23.0
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
golang.org/x/net v0.0.0-20210525063256-abc453219eb5
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1
gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614
gonum.org/v1/plot v0.9.0
google.golang.org/api v0.47.0
google.golang.org/genproto v0.0.0-20210521181308-5ccab8a35a9a
google.golang.org/grpc v1.37.1
google.golang.org/genproto v0.0.0-20210524171403-669157292da3
google.golang.org/grpc v1.38.0
gopkg.in/yaml.v2 v2.4.0
inet.af/netaddr v0.0.0-20210523191804-d57edf19c517
k8s.io/api v0.20.7
Expand Down
35 changes: 18 additions & 17 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
cloud.google.com/go v0.82.1-0.20210520180002-258980a039db h1:xCAzr79wv6gXNNRCXWIOazkakY+8svIdVzM+CZHlfKo=
cloud.google.com/go v0.82.1-0.20210520180002-258980a039db/go.mod h1:aaOxpQIjleHbu7ACFxPFX7rinCeayDh1cmtx3jT56ZA=
cloud.google.com/go v0.82.1-0.20210525054938-8bd87cbfee00 h1:E5Koz1/pqPbvNCpl3AW0+YlXCEwSF6JlaGIPQvfWjSw=
cloud.google.com/go v0.82.1-0.20210525054938-8bd87cbfee00/go.mod h1:PdTdtzcD5plR6MgV+H51AuJDxqPujrfxTa/VNmJDtcU=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo=
cloud.google.com/go/pubsub v1.10.3/go.mod h1:FUcc28GpGxxACoklPsE1sCtbkY4Ix+ro7yvw+h82Jn4=
cloud.google.com/go/storage v1.15.1-0.20210520180002-258980a039db h1:fUSm1Ouq5tE3LxSiwWoDJTj/VxVXpCY34XcV/Uk7RUs=
cloud.google.com/go/storage v1.15.1-0.20210520180002-258980a039db/go.mod h1:j8Rfn7cnrx4q7ChDAkHuiXmv1BIuI9b/QPOvLRvgyT4=
cloud.google.com/go/storage v1.15.1-0.20210525054938-8bd87cbfee00 h1:bxP85K3iE7FN93Li7ZwdwOJHPSLB9ZGkBhhUKt1QS1k=
cloud.google.com/go/storage v1.15.1-0.20210525054938-8bd87cbfee00/go.mod h1:QPQeqgdRA3A61g4itMoco3whFotU5WswsDH2jMyPjGU=
code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
code.cloudfoundry.org/bytefmt v0.0.0-20210519212014-d85d569ae74a h1:Kpv1Rq9O/4Pq2Hrj/MYKEf1doUmWOD/jzNw/+++3EOU=
code.cloudfoundry.org/bytefmt v0.0.0-20210519212014-d85d569ae74a/go.mod h1:yABoJ2k296jrOM76JpOlcsbJuC/4d+WtP+MEAQuuQOk=
code.cloudfoundry.org/bytefmt v0.0.0-20210524144015-27119551aaea h1:sgtgFDWrhV2cML3UbS2EZu+AfHLP96N0DJ1IHqI1rXA=
code.cloudfoundry.org/bytefmt v0.0.0-20210524144015-27119551aaea/go.mod h1:yABoJ2k296jrOM76JpOlcsbJuC/4d+WtP+MEAQuuQOk=
contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI=
contrib.go.opencensus.io/exporter/jaeger v0.2.1/go.mod h1:Y8IsLgdxqh1QxYxPC5IgXVmBaeLUeQFfBeBi9PbeZd0=
Expand Down Expand Up @@ -86,8 +86,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.38.45 h1:pQmv1vT/voRAjENnPsT4WobFBgLwnODDFogrt2kXc7M=
github.com/aws/aws-sdk-go v1.38.45/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.38.46 h1:voiwaKmwU1K6Y0dfjqTSiy5xOG4LPyr5sHD92cj+g2c=
github.com/aws/aws-sdk-go v1.38.46/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
Expand Down Expand Up @@ -272,8 +272,8 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-replayers/grpcreplay v1.0.0 h1:B5kVOzJ1hBgnevTgIWhSTatQ3608yu/2NnU0Ta1d0kY=
github.com/google/go-replayers/grpcreplay v1.0.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
github.com/google/go-replayers/httpreplay v0.1.2 h1:HCfx+dQzwN9XbGTHF8qJ+67WN8glL9FTWV5rraCJ/jU=
Expand All @@ -285,8 +285,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=
github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ=
github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a h1:jmAp/2PZAScNd62lTD3Mcb0Ey9FvIIJtLohPhtxZJ+Q=
github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
Expand Down Expand Up @@ -779,8 +779,8 @@ golang.org/x/net v0.0.0-20210420210106-798c2154c571/go.mod h1:72T/g9IO56b78aLF+1
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -971,11 +971,12 @@ google.golang.org/genproto v0.0.0-20210423144448-3a41ef94ed2b/go.mod h1:P3QM42oQ
google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
google.golang.org/genproto v0.0.0-20210506142907-4a47615972c2/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
google.golang.org/genproto v0.0.0-20210518161634-ec7691c0a37d/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
google.golang.org/genproto v0.0.0-20210521181308-5ccab8a35a9a h1:FaCiYXNZoBH/gnmVjMAHgOgdmpVVROBYOA+qCOHh6Hc=
google.golang.org/genproto v0.0.0-20210521181308-5ccab8a35a9a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
google.golang.org/genproto v0.0.0-20210524142926-3e3a6030be83/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
google.golang.org/genproto v0.0.0-20210524171403-669157292da3 h1:xFyh6GBb+NO1L0xqb978I3sBPQpk6FrKO0jJGRvdj/0=
google.golang.org/genproto v0.0.0-20210524171403-669157292da3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
Expand Down
2 changes: 1 addition & 1 deletion internal/net/grpc/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ func (g *gRPCClient) RangeConcurrent(ctx context.Context,
}
mu.Unlock()
}
return err
return nil
}
}))
return true
Expand Down
4 changes: 3 additions & 1 deletion pkg/agent/core/ngt/service/ngt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,7 +695,9 @@ func (n *ngt) CreateAndSaveIndex(ctx context.Context, poolSize uint32) (err erro
func (n *ngt) Exists(uuid string) (oid uint32, ok bool) {
oid, ok = n.kvs.Get(uuid)
if !ok {
_, ok = n.vq.GetVector(uuid)
ok = n.vq.IVExists(uuid)
} else {
ok = !n.vq.DVExists(uuid)
}
return oid, ok
}
Expand Down
17 changes: 16 additions & 1 deletion pkg/agent/core/ngt/service/vqueue/queue.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,22 @@ func (v *vqueue) GetVector(uuid string) ([]float32, bool) {
v.imu.Lock()
vec, ok := v.uiim[uuid]
v.imu.Unlock()
return vec.vector, ok
if !ok {
// data not in the insert queue then return not exists(false)
return nil, false
}
v.dmu.Lock()
di, ok := v.udim[uuid]
v.dmu.Unlock()
if !ok {
// data not in the delete queue but exists in insert queue then return exists(true)
return vec.vector, true
}
// data exists both queue, compare data timestamp if insert queue timestamp is newer than delete one, this function returns exists(true)
if di < vec.date {
return vec.vector, true
}
return nil, false
}

func (v *vqueue) IVExists(uuid string) bool {
Expand Down