[patch] add 3 new distance type support for agent-ngt (#780)

* [patch] add 3 new distance type support for agent-ngt Signed-off-by: kpango <i.can.feel.gravity@gmail.com> * fix Signed-off-by: kpango <i.can.feel.gravity@gmail.com> * fix Signed-off-by: kpango <i.can.feel.gravity@gmail.com> * fix Signed-off-by: kpango <i.can.feel.gravity@gmail.com> * Update docs/tutorial/agent-on-docker.md Co-authored-by: Rintaro Okamura <rintaro.okamura@gmail.com> * fix Signed-off-by: kpango <i.can.feel.gravity@gmail.com> Co-authored-by: Rintaro Okamura <rintaro.okamura@gmail.com>
vdaas · Oct 19, 2020 · e0d562b · e0d562b
1 parent 3b9325b
commit e0d562b
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 13 deletions.
diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml
@@ -1247,10 +1247,11 @@ agent:
     # @schema {"name": "agent.ngt.bulk_insert_chunk_size", "type": "integer"}
     # agent.ngt.bulk_insert_chunk_size -- bulk insert chunk size
     bulk_insert_chunk_size: 10
-    # @schema {"name": "agent.ngt.distance_type", "type": "string", "enum": ["l1", "l2", "angle", "hamming", "cos", "cosine", "normalizedangle", "normalizedcosine"]}
+    # @schema {"name": "agent.ngt.distance_type", "type": "string", "enum": ["l1", "l2", "angle", "hamming", "cos", "cosine", "normalizedangle", "normalizedcosine", "jaccard"]}
     # agent.ngt.distance_type -- distance type.
-    # it should be `l1`, `l2`, `angle`, `hamming`, `cosine`, `normalizedangle` or `normalizedcosine`.
-    # for further details: https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference
+    # it should be `l1`, `l2`, `angle`, `hamming`, `cosine`, `normalizedangle`, `normalizedcosine` or `jaccard`.
+    # for further details about NGT libraries supported distance is https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference
+    # and vald agent's supported NGT distance type is https://pkg.go.dev/github.com/vdaas/vald/internal/core/ngt#pkg-constants
     distance_type: l2
     # @schema {"name": "agent.ngt.object_type", "type": "string", "enum": ["float", "uint8"]}
     # agent.ngt.object_type -- object type.

diff --git a/docs/tutorial/agent-on-docker.md b/docs/tutorial/agent-on-docker.md
@@ -82,7 +82,7 @@ This chapter will use NGT for the core engine of Vald Agent.
       dimension: 784
       # bulk insert chunk size
       bulk_insert_chunk_size: 10
-      # distance_type, which should be "l1", "l2" "angle", "hamming", "cosine", "normalizedangle" or "nomralizedcosine"
+      # distance_type, which should be "l1", "l2" "angle", "hamming", "cosine", "normalizedangle", "normalizedcosine" or "jaccard"
       distance_type: l2
       # object_type, which should be "float" or "uint8"
       object_type: float
@@ -156,4 +156,3 @@ This chapter will use NGT for the core engine of Vald Agent.
 1. Clean Up
 
     Stop the Vald Agent docker container via `Ctrl+C`.
-
diff --git a/internal/core/ngt/ngt.go b/internal/core/ngt/ngt.go
@@ -101,13 +101,21 @@ type objectType int
 type distanceType int
 
 const (
+	// -------------------------------------------------------------
+	// Object Type Definition
+	// -------------------------------------------------------------
 	// ObjectNone is unknown object type
 	ObjectNone objectType = iota
 	// Uint8 is 8bit unsigned integer
 	Uint8
 	// Float is 32bit floating point number
 	Float
+	// -------------------------------------------------------------
 
+
+	// -------------------------------------------------------------
+	// Distance Type Definition
+	// -------------------------------------------------------------
 	// DistanceNone is unknown distance type
 	DistanceNone distanceType = iota - 1
 	// L1 is l1 norm
@@ -124,6 +132,10 @@ const (
 	NormalizedAngle
 	// NormalizedCosine is cosine distance with normalization
 	NormalizedCosine
+	// Jaccard is jaccard distance
+	Jaccard
+	// -------------------------------------------------------------
+
 
 	// ErrorCode is false
 	ErrorCode = C._Bool(false)

diff --git a/internal/core/ngt/option.go b/internal/core/ngt/option.go
@@ -94,16 +94,18 @@ func WithDistanceTypeByString(dt string) Option {
 		d = L1
 	case "l2":
 		d = L2
-	case "angle":
+	case "angle", "ang":
 		d = Angle
-	case "hamming":
+	case "hamming", "ham":
 		d = Hamming
 	case "cosine", "cos":
 		d = Cosine
-	case "normalizedangle":
+	case "normalizedangle", "normalized angle", "normalized ang", "nang", "nangle":
 		d = NormalizedAngle
-	case "normalizedcosine":
+	case "normalizedcosine","normalized cosine", "normalized cos", "ncos", "ncosine":
 		d = NormalizedCosine
+	case "jaccard", "jac":
+		d = Jaccard
 	}
 	return WithDistanceType(d)
 }
@@ -132,11 +134,17 @@ func WithDistanceType(t distanceType) Option {
 				return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "Cosine")
 			}
 		case NormalizedAngle:
-			// TODO: not implemented in C API
-			return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedAngle")
+			if C.ngt_set_property_distance_type_normalized_angle(n.prop, n.ebuf) == ErrorCode {
+				return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedAngle")
+			}
 		case NormalizedCosine:
-			// TODO: not implemented in C API
-			return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedCosine")
+			if C.ngt_set_property_distance_type_normalized_cosine(n.prop, n.ebuf) == ErrorCode {
+				return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedCosine")
+			}
+		case Jaccard:
+			if C.ngt_set_property_distance_type_jaccard(n.prop, n.ebuf) == ErrorCode {
+				return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "Jaccard")
+			}
 		default:
 			return errors.ErrUnsupportedDistanceType
 		}