From e43c4796770ac3e331e0da7e131af615491e9c72 Mon Sep 17 00:00:00 2001 From: kpango Date: Tue, 25 May 2021 18:11:22 +0900 Subject: [PATCH 1/2] [bugfix] change exists cheking for agent vqueue Signed-off-by: kpango --- internal/net/grpc/client.go | 2 +- pkg/agent/core/ngt/service/ngt.go | 4 +++- pkg/agent/core/ngt/service/vqueue/queue.go | 17 ++++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/internal/net/grpc/client.go b/internal/net/grpc/client.go index 5e783f6ed5..ab84d148a2 100644 --- a/internal/net/grpc/client.go +++ b/internal/net/grpc/client.go @@ -362,7 +362,7 @@ func (g *gRPCClient) RangeConcurrent(ctx context.Context, } mu.Unlock() } - return err + return nil } })) return true diff --git a/pkg/agent/core/ngt/service/ngt.go b/pkg/agent/core/ngt/service/ngt.go index 30043aafaa..b78228592f 100644 --- a/pkg/agent/core/ngt/service/ngt.go +++ b/pkg/agent/core/ngt/service/ngt.go @@ -695,7 +695,9 @@ func (n *ngt) CreateAndSaveIndex(ctx context.Context, poolSize uint32) (err erro func (n *ngt) Exists(uuid string) (oid uint32, ok bool) { oid, ok = n.kvs.Get(uuid) if !ok { - _, ok = n.vq.GetVector(uuid) + ok = n.vq.IVExists(uuid) + } else { + ok = !n.vq.DVExists(uuid) } return oid, ok } diff --git a/pkg/agent/core/ngt/service/vqueue/queue.go b/pkg/agent/core/ngt/service/vqueue/queue.go index ebf3114fdc..652ea1e05d 100644 --- a/pkg/agent/core/ngt/service/vqueue/queue.go +++ b/pkg/agent/core/ngt/service/vqueue/queue.go @@ -233,7 +233,22 @@ func (v *vqueue) GetVector(uuid string) ([]float32, bool) { v.imu.Lock() vec, ok := v.uiim[uuid] v.imu.Unlock() - return vec.vector, ok + if !ok { + // data not in the insert queue then return not exists(false) + return nil, false + } + v.dmu.Lock() + di, ok := v.udim[uuid] + v.dmu.Unlock() + if !ok { + // data not in the delete queue but exists in insert queue then return exists(true) + return vec.vector, true + } + // data exists both queue, compare data timestamp if insert queue timestamp is newer than delete one, this function returns exists(true) + if di < vec.date { + return vec.vector, true + } + return nil, false } func (v *vqueue) IVExists(uuid string) bool { From 0c02c0fc3ddc8407c28246fa37d1629e1f276335 Mon Sep 17 00:00:00 2001 From: kpango Date: Tue, 25 May 2021 18:44:38 +0900 Subject: [PATCH 2/2] fix Signed-off-by: kpango --- go.mod | 18 +++++++++--------- go.sum | 35 ++++++++++++++++++----------------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index ce70b1c6fe..71fe49410a 100755 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module github.com/vdaas/vald go 1.16 replace ( - cloud.google.com/go => cloud.google.com/go v0.82.1-0.20210520180002-258980a039db - cloud.google.com/go/storage => cloud.google.com/go/storage v1.15.1-0.20210520180002-258980a039db + cloud.google.com/go => cloud.google.com/go v0.82.1-0.20210525054938-8bd87cbfee00 + cloud.google.com/go/storage => cloud.google.com/go/storage v1.15.1-0.20210525054938-8bd87cbfee00 github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.1-0.20210210161804-c7f947c0610d+incompatible - github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.38.45 + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.38.46 github.com/boltdb/bolt => github.com/boltdb/bolt v1.3.1 github.com/chzyer/logex => github.com/chzyer/logex v1.1.11-0.20170329064859-445be9e134b2 github.com/coreos/etcd => go.etcd.io/etcd v3.3.25+incompatible @@ -16,7 +16,7 @@ replace ( github.com/gocql/gocql => github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556 github.com/gogo/googleapis => github.com/gogo/googleapis v1.4.1 github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2 - github.com/google/go-cmp => github.com/google/go-cmp v0.5.5 + github.com/google/go-cmp => github.com/google/go-cmp v0.5.6 github.com/google/pprof => github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.17.0 @@ -38,9 +38,9 @@ replace ( ) require ( - cloud.google.com/go v0.81.0 + cloud.google.com/go v0.82.0 cloud.google.com/go/storage v1.15.0 - code.cloudfoundry.org/bytefmt v0.0.0-20210519212014-d85d569ae74a + code.cloudfoundry.org/bytefmt v0.0.0-20210524144015-27119551aaea contrib.go.opencensus.io/exporter/jaeger v0.2.1 contrib.go.opencensus.io/exporter/prometheus v0.3.0 contrib.go.opencensus.io/exporter/stackdriver v0.13.6 @@ -77,15 +77,15 @@ require ( go.uber.org/goleak v1.1.10 go.uber.org/zap v1.16.0 gocloud.dev v0.23.0 - golang.org/x/net v0.0.0-20210521195947-fe42d452be8f + golang.org/x/net v0.0.0-20210525063256-abc453219eb5 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614 gonum.org/v1/plot v0.9.0 google.golang.org/api v0.47.0 - google.golang.org/genproto v0.0.0-20210521181308-5ccab8a35a9a - google.golang.org/grpc v1.37.1 + google.golang.org/genproto v0.0.0-20210524171403-669157292da3 + google.golang.org/grpc v1.38.0 gopkg.in/yaml.v2 v2.4.0 inet.af/netaddr v0.0.0-20210523191804-d57edf19c517 k8s.io/api v0.20.7 diff --git a/go.sum b/go.sum index 28d7ddfb95..b2d73364b1 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,14 @@ bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -cloud.google.com/go v0.82.1-0.20210520180002-258980a039db h1:xCAzr79wv6gXNNRCXWIOazkakY+8svIdVzM+CZHlfKo= -cloud.google.com/go v0.82.1-0.20210520180002-258980a039db/go.mod h1:aaOxpQIjleHbu7ACFxPFX7rinCeayDh1cmtx3jT56ZA= +cloud.google.com/go v0.82.1-0.20210525054938-8bd87cbfee00 h1:E5Koz1/pqPbvNCpl3AW0+YlXCEwSF6JlaGIPQvfWjSw= +cloud.google.com/go v0.82.1-0.20210525054938-8bd87cbfee00/go.mod h1:PdTdtzcD5plR6MgV+H51AuJDxqPujrfxTa/VNmJDtcU= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo= cloud.google.com/go/pubsub v1.10.3/go.mod h1:FUcc28GpGxxACoklPsE1sCtbkY4Ix+ro7yvw+h82Jn4= -cloud.google.com/go/storage v1.15.1-0.20210520180002-258980a039db h1:fUSm1Ouq5tE3LxSiwWoDJTj/VxVXpCY34XcV/Uk7RUs= -cloud.google.com/go/storage v1.15.1-0.20210520180002-258980a039db/go.mod h1:j8Rfn7cnrx4q7ChDAkHuiXmv1BIuI9b/QPOvLRvgyT4= +cloud.google.com/go/storage v1.15.1-0.20210525054938-8bd87cbfee00 h1:bxP85K3iE7FN93Li7ZwdwOJHPSLB9ZGkBhhUKt1QS1k= +cloud.google.com/go/storage v1.15.1-0.20210525054938-8bd87cbfee00/go.mod h1:QPQeqgdRA3A61g4itMoco3whFotU5WswsDH2jMyPjGU= code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= -code.cloudfoundry.org/bytefmt v0.0.0-20210519212014-d85d569ae74a h1:Kpv1Rq9O/4Pq2Hrj/MYKEf1doUmWOD/jzNw/+++3EOU= -code.cloudfoundry.org/bytefmt v0.0.0-20210519212014-d85d569ae74a/go.mod h1:yABoJ2k296jrOM76JpOlcsbJuC/4d+WtP+MEAQuuQOk= +code.cloudfoundry.org/bytefmt v0.0.0-20210524144015-27119551aaea h1:sgtgFDWrhV2cML3UbS2EZu+AfHLP96N0DJ1IHqI1rXA= +code.cloudfoundry.org/bytefmt v0.0.0-20210524144015-27119551aaea/go.mod h1:yABoJ2k296jrOM76JpOlcsbJuC/4d+WtP+MEAQuuQOk= contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI= contrib.go.opencensus.io/exporter/jaeger v0.2.1/go.mod h1:Y8IsLgdxqh1QxYxPC5IgXVmBaeLUeQFfBeBi9PbeZd0= @@ -86,8 +86,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.38.45 h1:pQmv1vT/voRAjENnPsT4WobFBgLwnODDFogrt2kXc7M= -github.com/aws/aws-sdk-go v1.38.45/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.38.46 h1:voiwaKmwU1K6Y0dfjqTSiy5xOG4LPyr5sHD92cj+g2c= +github.com/aws/aws-sdk-go v1.38.46/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -272,8 +272,8 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-replayers/grpcreplay v1.0.0 h1:B5kVOzJ1hBgnevTgIWhSTatQ3608yu/2NnU0Ta1d0kY= github.com/google/go-replayers/grpcreplay v1.0.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= github.com/google/go-replayers/httpreplay v0.1.2 h1:HCfx+dQzwN9XbGTHF8qJ+67WN8glL9FTWV5rraCJ/jU= @@ -285,8 +285,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a h1:jmAp/2PZAScNd62lTD3Mcb0Ey9FvIIJtLohPhtxZJ+Q= github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -779,8 +779,8 @@ golang.org/x/net v0.0.0-20210420210106-798c2154c571/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U= -golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -971,11 +971,12 @@ google.golang.org/genproto v0.0.0-20210423144448-3a41ef94ed2b/go.mod h1:P3QM42oQ google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210506142907-4a47615972c2/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210518161634-ec7691c0a37d/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210521181308-5ccab8a35a9a h1:FaCiYXNZoBH/gnmVjMAHgOgdmpVVROBYOA+qCOHh6Hc= -google.golang.org/genproto v0.0.0-20210521181308-5ccab8a35a9a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210524142926-3e3a6030be83/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210524171403-669157292da3 h1:xFyh6GBb+NO1L0xqb978I3sBPQpk6FrKO0jJGRvdj/0= +google.golang.org/genproto v0.0.0-20210524171403-669157292da3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=