Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[patch] add 3 new distance type support for agent-ngt #780

Merged
merged 7 commits into from
Oct 19, 2020
Merged

[patch] add 3 new distance type support for agent-ngt #780

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
fdcf064
[patch] add 3 new distance type support for agent-ngt
Oct 17, 2020
d262d39
fix
Oct 17, 2020
be32b60
fix
Oct 19, 2020
ea55cbe
fix
Oct 19, 2020
bb6a7ce
Update docs/tutorial/agent-on-docker.md
Oct 19, 2020
09f08c3
fix
Oct 19, 2020
e0a0fbb
Merge branch 'feature/agent/add-distance-type' of https://github.com/…
Oct 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions charts/vald/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1247,10 +1247,11 @@ agent:
# @schema {"name": "agent.ngt.bulk_insert_chunk_size", "type": "integer"}
# agent.ngt.bulk_insert_chunk_size -- bulk insert chunk size
bulk_insert_chunk_size: 10
# @schema {"name": "agent.ngt.distance_type", "type": "string", "enum": ["l1", "l2", "angle", "hamming", "cos", "cosine", "normalizedangle", "normalizedcosine"]}
# @schema {"name": "agent.ngt.distance_type", "type": "string", "enum": ["l1", "l2", "angle", "hamming", "cos", "cosine", "normalizedangle", "normalizedcosine", "jaccard"]}
# agent.ngt.distance_type -- distance type.
# it should be `l1`, `l2`, `angle`, `hamming`, `cosine`, `normalizedangle` or `normalizedcosine`.
# for further details: https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference
# it should be `l1`, `l2`, `angle`, `hamming`, `cosine`, `normalizedangle`, `normalizedcosine` or `jaccard`.
# for further details about NGT libraries supported distance is https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference
# and vald agent's supported NGT distance type is https://pkg.go.dev/github.com/vdaas/vald/internal/core/ngt#pkg-constants
distance_type: l2
# @schema {"name": "agent.ngt.object_type", "type": "string", "enum": ["float", "uint8"]}
# agent.ngt.object_type -- object type.
Expand Down
3 changes: 1 addition & 2 deletions docs/tutorial/agent-on-docker.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ This chapter will use NGT for the core engine of Vald Agent.
dimension: 784
# bulk insert chunk size
bulk_insert_chunk_size: 10
# distance_type, which should be "l1", "l2" "angle", "hamming", "cosine", "normalizedangle" or "nomralizedcosine"
# distance_type, which should be "l1", "l2" "angle", "hamming", "cosine", "normalizedangle", "normalizedcosine" or "jaccard"
distance_type: l2
# object_type, which should be "float" or "uint8"
object_type: float
Expand Down Expand Up @@ -156,4 +156,3 @@ This chapter will use NGT for the core engine of Vald Agent.
1. Clean Up

Stop the Vald Agent docker container via `Ctrl+C`.

12 changes: 12 additions & 0 deletions internal/core/ngt/ngt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,21 @@ type objectType int
type distanceType int

const (
// -------------------------------------------------------------
// Object Type Definition
// -------------------------------------------------------------
// ObjectNone is unknown object type
ObjectNone objectType = iota
// Uint8 is 8bit unsigned integer
Uint8
// Float is 32bit floating point number
Float
// -------------------------------------------------------------


// -------------------------------------------------------------
// Distance Type Definition
// -------------------------------------------------------------
// DistanceNone is unknown distance type
DistanceNone distanceType = iota - 1
// L1 is l1 norm
Expand All @@ -124,6 +132,10 @@ const (
NormalizedAngle
// NormalizedCosine is cosine distance with normalization
NormalizedCosine
// Jaccard is jaccard distance
Jaccard
// -------------------------------------------------------------


// ErrorCode is false
ErrorCode = C._Bool(false)
Expand Down
24 changes: 16 additions & 8 deletions internal/core/ngt/option.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,16 +94,18 @@ func WithDistanceTypeByString(dt string) Option {
d = L1
case "l2":
d = L2
case "angle":
case "angle", "ang":
d = Angle
case "hamming":
case "hamming", "ham":
d = Hamming
case "cosine", "cos":
d = Cosine
case "normalizedangle":
case "normalizedangle", "normalized angle", "normalized ang", "nang", "nangle":
d = NormalizedAngle
case "normalizedcosine":
case "normalizedcosine","normalized cosine", "normalized cos", "ncos", "ncosine":
d = NormalizedCosine
case "jaccard", "jac":
d = Jaccard
}
return WithDistanceType(d)
}
Expand Down Expand Up @@ -132,11 +134,17 @@ func WithDistanceType(t distanceType) Option {
return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "Cosine")
}
case NormalizedAngle:
// TODO: not implemented in C API
return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedAngle")
if C.ngt_set_property_distance_type_normalized_angle(n.prop, n.ebuf) == ErrorCode {
return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedAngle")
}
case NormalizedCosine:
// TODO: not implemented in C API
return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedCosine")
if C.ngt_set_property_distance_type_normalized_cosine(n.prop, n.ebuf) == ErrorCode {
return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "NormalizedCosine")
}
case Jaccard:
if C.ngt_set_property_distance_type_jaccard(n.prop, n.ebuf) == ErrorCode {
return errors.ErrFailedToSetDistanceType(n.newGoError(n.ebuf), "Jaccard")
}
default:
return errors.ErrUnsupportedDistanceType
}
Expand Down