From fdcf0648e345e0c78ea45492894f10c8b7d33dc4 Mon Sep 17 00:00:00 2001 From: kpango Date: Sun, 18 Oct 2020 00:25:55 +0900 Subject: [PATCH 1/6] [patch] add 3 new distance type support for agent-ngt Signed-off-by: kpango --- internal/core/ngt/ngt.go | 2 ++ internal/core/ngt/option.go | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/internal/core/ngt/ngt.go b/internal/core/ngt/ngt.go index 68945e43fa..250185b551 100644 --- a/internal/core/ngt/ngt.go +++ b/internal/core/ngt/ngt.go @@ -124,6 +124,8 @@ const ( NormalizedAngle // NormalizedCosine is cosine distance with normalization NormalizedCosine + // Jaccard is jaccard distance + Jaccard // ErrorCode is false ErrorCode = C._Bool(false) diff --git a/internal/core/ngt/option.go b/internal/core/ngt/option.go index f675bcae47..f667582891 100644 --- a/internal/core/ngt/option.go +++ b/internal/core/ngt/option.go @@ -104,6 +104,8 @@ func WithDistanceTypeByString(dt string) Option { d = NormalizedAngle case "normalizedcosine": d = NormalizedCosine + case "jaccard": + d = Jaccard } return WithDistanceType(d) } @@ -132,11 +134,17 @@ func WithDistanceType(t distanceType) Option { return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "Cosine") } case NormalizedAngle: - // TODO: not implemented in C API - return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedAngle") + if C.ngt_set_property_distance_type_normalized_angle(n.prop, n.ebuf) == ErrorCode{ + return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedAngle") + } case NormalizedCosine: - // TODO: not implemented in C API - return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedCosine") + if C.ngt_set_property_distance_type_normalized_cosine(n.prop, n.ebuf) == ErrorCode{ + return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedCosine") + } + case Jaccard: + if C.ngt_set_property_distance_type_jaccard(n.prop, n.ebuf) == ErrorCode{ + return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "Jaccard") + } default: return errors.ErrUnsupportedDistanceType } From d262d394f43feaa44818f56275db2a2a477b7e0a Mon Sep 17 00:00:00 2001 From: kpango Date: Sun, 18 Oct 2020 00:35:47 +0900 Subject: [PATCH 2/6] fix Signed-off-by: kpango --- internal/core/ngt/option.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/core/ngt/option.go b/internal/core/ngt/option.go index f667582891..57f59ad263 100644 --- a/internal/core/ngt/option.go +++ b/internal/core/ngt/option.go @@ -94,17 +94,17 @@ func WithDistanceTypeByString(dt string) Option { d = L1 case "l2": d = L2 - case "angle": + case "angle", "ang": d = Angle - case "hamming": + case "hamming", "ham": d = Hamming case "cosine", "cos": d = Cosine - case "normalizedangle": + case "normalizedangle", "nang", "nangle": d = NormalizedAngle - case "normalizedcosine": + case "normalizedcosine", "nham", "nhamming": d = NormalizedCosine - case "jaccard": + case "jaccard", "jac": d = Jaccard } return WithDistanceType(d) @@ -134,15 +134,15 @@ func WithDistanceType(t distanceType) Option { return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "Cosine") } case NormalizedAngle: - if C.ngt_set_property_distance_type_normalized_angle(n.prop, n.ebuf) == ErrorCode{ + if C.ngt_set_property_distance_type_normalized_angle(n.prop, n.ebuf) == ErrorCode { return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedAngle") } case NormalizedCosine: - if C.ngt_set_property_distance_type_normalized_cosine(n.prop, n.ebuf) == ErrorCode{ + if C.ngt_set_property_distance_type_normalized_cosine(n.prop, n.ebuf) == ErrorCode { return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedCosine") } case Jaccard: - if C.ngt_set_property_distance_type_jaccard(n.prop, n.ebuf) == ErrorCode{ + if C.ngt_set_property_distance_type_jaccard(n.prop, n.ebuf) == ErrorCode { return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "Jaccard") } default: From be32b60ddcc1ab4fb2d48e6532de65fe7f5557c8 Mon Sep 17 00:00:00 2001 From: kpango Date: Mon, 19 Oct 2020 10:54:13 +0900 Subject: [PATCH 3/6] fix Signed-off-by: kpango --- charts/vald/values.yaml | 7 ++++--- internal/core/ngt/ngt.go | 10 ++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index 5c2d3db26b..38959f87a4 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -1247,10 +1247,11 @@ agent: # @schema {"name": "agent.ngt.bulk_insert_chunk_size", "type": "integer"} # agent.ngt.bulk_insert_chunk_size -- bulk insert chunk size bulk_insert_chunk_size: 10 - # @schema {"name": "agent.ngt.distance_type", "type": "string", "enum": ["l1", "l2", "angle", "hamming", "cos", "cosine", "normalizedangle", "normalizedcosine"]} + # @schema {"name": "agent.ngt.distance_type", "type": "string", "enum": ["l1", "l2", "angle", "hamming", "cos", "cosine", "normalizedangle", "normalizedcosine", "jaccard"]} # agent.ngt.distance_type -- distance type. - # it should be `l1`, `l2`, `angle`, `hamming`, `cosine`, `normalizedangle` or `normalizedcosine`. - # for further details: https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference + # it should be `l1`, `l2`, `angle`, `hamming`, `cosine`, `normalizedangle`, `normalizedcosine` or `jaccard`. + # for further details about NGT libraries supported distance is https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference + # and vald agent's supported NGT distance type is https://pkg.go.dev/github.com/vdaas/vald/internal/core/ngt#pkg-constants distance_type: l2 # @schema {"name": "agent.ngt.object_type", "type": "string", "enum": ["float", "uint8"]} # agent.ngt.object_type -- object type. diff --git a/internal/core/ngt/ngt.go b/internal/core/ngt/ngt.go index 250185b551..d75ee67d63 100644 --- a/internal/core/ngt/ngt.go +++ b/internal/core/ngt/ngt.go @@ -101,13 +101,21 @@ type objectType int type distanceType int const ( + // ------------------------------------------------------------- + // Object Type Definition + // ------------------------------------------------------------- // ObjectNone is unknown object type ObjectNone objectType = iota // Uint8 is 8bit unsigned integer Uint8 // Float is 32bit floating point number Float + // ------------------------------------------------------------- + + // ------------------------------------------------------------- + // Distance Type Definition + // ------------------------------------------------------------- // DistanceNone is unknown distance type DistanceNone distanceType = iota - 1 // L1 is l1 norm @@ -126,6 +134,8 @@ const ( NormalizedCosine // Jaccard is jaccard distance Jaccard + // ------------------------------------------------------------- + // ErrorCode is false ErrorCode = C._Bool(false) From ea55cbe3ced72fac0ee0cb0d0c8c4e9846597eed Mon Sep 17 00:00:00 2001 From: kpango Date: Mon, 19 Oct 2020 10:57:09 +0900 Subject: [PATCH 4/6] fix Signed-off-by: kpango --- docs/tutorial/agent-on-docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial/agent-on-docker.md b/docs/tutorial/agent-on-docker.md index 26831aceda..6b3f90adc4 100644 --- a/docs/tutorial/agent-on-docker.md +++ b/docs/tutorial/agent-on-docker.md @@ -82,7 +82,7 @@ This chapter will use NGT for the core engine of Vald Agent. dimension: 784 # bulk insert chunk size bulk_insert_chunk_size: 10 - # distance_type, which should be "l1", "l2" "angle", "hamming", "cosine", "normalizedangle" or "nomralizedcosine" + # distance_type, which should be "l1", "l2" "angle", "hamming", "cosine", "normalizedangle", "nomralizedcosine" or "jaccard" distance_type: l2 # object_type, which should be "float" or "uint8" object_type: float From bb6a7ceb9577b05f07d59c2162b51e2e9cfb7d6e Mon Sep 17 00:00:00 2001 From: Yusuke Kato Date: Mon, 19 Oct 2020 11:02:27 +0900 Subject: [PATCH 5/6] Update docs/tutorial/agent-on-docker.md Co-authored-by: Rintaro Okamura --- docs/tutorial/agent-on-docker.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/tutorial/agent-on-docker.md b/docs/tutorial/agent-on-docker.md index 6b3f90adc4..f1cf488e5c 100644 --- a/docs/tutorial/agent-on-docker.md +++ b/docs/tutorial/agent-on-docker.md @@ -82,7 +82,7 @@ This chapter will use NGT for the core engine of Vald Agent. dimension: 784 # bulk insert chunk size bulk_insert_chunk_size: 10 - # distance_type, which should be "l1", "l2" "angle", "hamming", "cosine", "normalizedangle", "nomralizedcosine" or "jaccard" + # distance_type, which should be "l1", "l2" "angle", "hamming", "cosine", "normalizedangle", "normalizedcosine" or "jaccard" distance_type: l2 # object_type, which should be "float" or "uint8" object_type: float @@ -156,4 +156,3 @@ This chapter will use NGT for the core engine of Vald Agent. 1. Clean Up Stop the Vald Agent docker container via `Ctrl+C`. - From 09f08c38375f5002ce9b4f3c4c816369e7494f84 Mon Sep 17 00:00:00 2001 From: kpango Date: Mon, 19 Oct 2020 11:20:27 +0900 Subject: [PATCH 6/6] fix Signed-off-by: kpango --- internal/core/ngt/option.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/core/ngt/option.go b/internal/core/ngt/option.go index 57f59ad263..239ae77970 100644 --- a/internal/core/ngt/option.go +++ b/internal/core/ngt/option.go @@ -100,9 +100,9 @@ func WithDistanceTypeByString(dt string) Option { d = Hamming case "cosine", "cos": d = Cosine - case "normalizedangle", "nang", "nangle": + case "normalizedangle", "normalized angle", "normalized ang", "nang", "nangle": d = NormalizedAngle - case "normalizedcosine", "nham", "nhamming": + case "normalizedcosine","normalized cosine", "normalized cos", "ncos", "ncosine": d = NormalizedCosine case "jaccard", "jac": d = Jaccard