create investigation test of max dim for NGT (#1633)

* ✅ create investigation of max dim for NGT Signed-off-by: vankichi <kyukawa315@gmail.com> * Apply suggestions from code review Co-authored-by: Kosuke Morimoto <ksk@vdaas.org> * ✅ ♻️ change ngtDimensionSizeLimt to global value Signed-off-by: vankichi <kyukawa315@gmail.com> * ✅ ♻️ fix constant value name Signed-off-by: vankichi <kyukawa315@gmail.com> * ✅ fix failed test Signed-off-by: vankichi <kyukawa315@gmail.com> * 🤖 Update license headers / Format go codes and yaml files Signed-off-by: Vdaas CI <vald@vdaas.org> Co-authored-by: Kosuke Morimoto <ksk@vdaas.org> Co-authored-by: Vdaas CI <vald@vdaas.org>
vdaas · May 12, 2022 · c10358f · c10358f
1 parent 87679e9
commit c10358f
Show file tree

Hide file tree

Showing 22 changed files with 344 additions and 39 deletions.
diff --git a/internal/core/algorithm/ngt/ngt.go b/internal/core/algorithm/ngt/ngt.go
@@ -156,8 +156,8 @@ const (
 	// -------------------------------------------------------------
 	// dimension constraints
 	// -------------------------------------------------------------.
-	ngtVectorDimensionSizeLimit = 1 << 16
-	minimumDimensionSize        = algorithm.MinimumVectorDimensionSize
+	VectorDimensionSizeLimit = 1<<32 - 1
+	minimumDimensionSize     = algorithm.MinimumVectorDimensionSize
 	// -------------------------------------------------------------.
 )
 
@@ -685,8 +685,8 @@ func (n *ngt) GetVector(id uint) ([]float32, error) {
 		if results == nil {
 			return nil, n.newGoError(ebuf)
 		}
-		ret = (*[ngtVectorDimensionSizeLimit]float32)(unsafe.Pointer(results))[:dimension:dimension]
-		// for _, elem := range (*[ngtVectorDimensionSizeLimit]C.float)(unsafe.Pointer(results))[:dimension:dimension]{
+		ret = (*[VectorDimensionSizeLimit]float32)(unsafe.Pointer(results))[:dimension:dimension]
+		// for _, elem := range (*[VectorDimensionSizeLimit]C.float)(unsafe.Pointer(results))[:dimension:dimension]{
 		// 	ret = append(ret, float32(elem))
 		// }
 	case Uint8:
@@ -697,7 +697,7 @@ func (n *ngt) GetVector(id uint) ([]float32, error) {
 			return nil, n.newGoError(ebuf)
 		}
 		ret = make([]float32, 0, dimension)
-		for _, elem := range (*[ngtVectorDimensionSizeLimit]C.uint8_t)(unsafe.Pointer(results))[:dimension:dimension] {
+		for _, elem := range (*[VectorDimensionSizeLimit]C.uint8_t)(unsafe.Pointer(results))[:dimension:dimension] {
 			ret = append(ret, float32(elem))
 		}
 	default:

diff --git a/internal/core/algorithm/ngt/ngt_test.go b/internal/core/algorithm/ngt/ngt_test.go
@@ -183,7 +183,7 @@ func TestNew(t *testing.T) {
 				},
 			},
 			want: want{
-				err: errors.NewErrCriticalOption("dimension", 1, errors.ErrInvalidDimensionSize(1, ngtVectorDimensionSizeLimit)),
+				err: errors.NewErrCriticalOption("dimension", 1, errors.ErrInvalidDimensionSize(1, VectorDimensionSizeLimit)),
 			},
 		},
 	}
@@ -761,7 +761,7 @@ func Test_gen(t *testing.T) {
 				},
 			},
 			want: want{
-				err: errors.NewErrCriticalOption("dimension", 1, errors.ErrInvalidDimensionSize(1, ngtVectorDimensionSizeLimit)),
+				err: errors.NewErrCriticalOption("dimension", 1, errors.ErrInvalidDimensionSize(1, VectorDimensionSizeLimit)),
 			},
 		},
 	}

diff --git a/internal/core/algorithm/ngt/option.go b/internal/core/algorithm/ngt/option.go
@@ -86,8 +86,8 @@ func WithBulkInsertChunkSize(size int) Option {
 // WithDimension represents the option to set the dimension for NGT.
 func WithDimension(size int) Option {
 	return func(n *ngt) error {
-		if size > ngtVectorDimensionSizeLimit || size < minimumDimensionSize {
-			err := errors.ErrInvalidDimensionSize(size, ngtVectorDimensionSizeLimit)
+		if size > VectorDimensionSizeLimit || size < minimumDimensionSize {
+			err := errors.ErrInvalidDimensionSize(size, VectorDimensionSizeLimit)
 			return errors.NewErrCriticalOption("dimension", size, err)
 		}
 

diff --git a/internal/core/algorithm/ngt/option_test.go b/internal/core/algorithm/ngt/option_test.go
@@ -20,6 +20,7 @@ package ngt
 import (
 	"math"
 	"reflect"
+	"strconv"
 	"testing"
 
 	"github.com/vdaas/vald/internal/errors"
@@ -329,13 +330,15 @@ func TestWithDimension(t *testing.T) {
 			},
 		},
 		{
-			name: "set success when the size is 0",
+			name: "return error when the size is 0",
 			args: args{
 				size: 0,
 			},
 			want: want{
 				obj: &T{},
-				err: errors.New("invalid critical option, name: dimension, val: 0: dimension size 0 is invalid, the supporting dimension size must be between 2 ~ 65536"),
+				err: errors.New(
+					"invalid critical option, name: dimension, val: 0: dimension size 0 is invalid, the supporting dimension size must be between 2 ~ " + strconv.Itoa(VectorDimensionSizeLimit),
+				),
 			},
 		},
 		{
@@ -348,9 +351,9 @@ func TestWithDimension(t *testing.T) {
 			},
 		},
 		{
-			name: "set success when the size is 65536",
+			name: "set success when the size is VectorDimensionSizeLimit",
 			args: args{
-				size: 65536,
+				size: VectorDimensionSizeLimit,
 			},
 			want: want{
 				obj: &T{},
@@ -363,7 +366,9 @@ func TestWithDimension(t *testing.T) {
 			},
 			want: want{
 				obj: &T{},
-				err: errors.New("invalid critical option, name: dimension, val: 1: dimension size 1 is invalid, the supporting dimension size must be between 2 ~ 65536"),
+				err: errors.New(
+					"invalid critical option, name: dimension, val: 1: dimension size 1 is invalid, the supporting dimension size must be between 2 ~ " + strconv.Itoa(VectorDimensionSizeLimit),
+				),
 			},
 		},
 		{
@@ -373,27 +378,40 @@ func TestWithDimension(t *testing.T) {
 			},
 			want: want{
 				obj: &T{},
-				err: errors.New("invalid critical option, name: dimension, val: -100: dimension size -100 is invalid, the supporting dimension size must be between 2 ~ 65536"),
+				err: errors.New(
+					"invalid critical option, name: dimension, val: -100: dimension size -100 is invalid, the supporting dimension size must be between 2 ~ " + strconv.Itoa(VectorDimensionSizeLimit),
+				),
 			},
 		},
 		{
-			name: "return error when the size is 65537",
+			name: "return error when the size is larger than VectorDimensionSizeLimit",
 			args: args{
-				size: 65537,
+				size: VectorDimensionSizeLimit + 1,
 			},
 			want: want{
 				obj: &T{},
-				err: errors.New("invalid critical option, name: dimension, val: 65537: dimension size 65537 is invalid, the supporting dimension size must be between 2 ~ 65536"),
+				err: errors.New(
+					"invalid critical option, name: dimension, val: 4294967296: dimension size 4294967296 is invalid, the supporting dimension size must be between 2 ~ " + strconv.Itoa(
+						VectorDimensionSizeLimit,
+					),
+				),
 			},
 		},
 		{
-			name: "return error when the size is MaxInt32",
+			name: "set success when the size is MaxInt32",
 			args: args{
 				size: math.MaxInt32,
 			},
 			want: want{
-				obj: &T{},
-				err: errors.New("invalid critical option, name: dimension, val: 2147483647: dimension size 2147483647 is invalid, the supporting dimension size must be between 2 ~ 65536"),
+				obj: func() *T {
+					t := &T{
+						dimension: math.MaxInt32,
+					}
+					if err := t.setup(); err != nil {
+						return nil
+					}
+					return t
+				}(),
 			},
 		},
 		{
@@ -403,7 +421,11 @@ func TestWithDimension(t *testing.T) {
 			},
 			want: want{
 				obj: &T{},
-				err: errors.New("invalid critical option, name: dimension, val: -2147483648: dimension size -2147483648 is invalid, the supporting dimension size must be between 2 ~ 65536"),
+				err: errors.New(
+					"invalid critical option, name: dimension, val: -2147483648: dimension size -2147483648 is invalid, the supporting dimension size must be between 2 ~ " + strconv.Itoa(
+						VectorDimensionSizeLimit,
+					),
+				),
 			},
 		},
 	}

diff --git a/internal/net/control/control_darwin.go b/internal/net/control/control_darwin.go
@@ -1,4 +1,3 @@
-//go:build darwin && !linux && !windows && !wasm && !js
 // +build darwin,!linux,!windows,!wasm,!js
 
 //

diff --git a/internal/net/control/control_other.go b/internal/net/control/control_other.go
@@ -1,4 +1,3 @@
-//go:build wasm && js && !windows && !linux && !darwin
 // +build wasm,js,!windows,!linux,!darwin
 
 //

diff --git a/internal/net/control/control_unix.go b/internal/net/control/control_unix.go
@@ -1,4 +1,3 @@
-//go:build linux && !windows && !wasm && !js && !darwin
 // +build linux,!windows,!wasm,!js,!darwin
 
 //

diff --git a/internal/net/control/control_windows.go b/internal/net/control/control_windows.go
@@ -1,4 +1,3 @@
-//go:build windows
 // +build windows
 
 //

diff --git a/internal/runner/runner_race_test.go b/internal/runner/runner_race_test.go
@@ -1,4 +1,3 @@
-//go:build !race
 // +build !race
 
 //

diff --git a/internal/singleflight/singleflight_test.go b/internal/singleflight/singleflight_test.go
@@ -1,4 +1,3 @@
-//go:build !race
 // +build !race
 
 //

diff --git a/internal/timeutil/location/set_test.go b/internal/timeutil/location/set_test.go
@@ -1,4 +1,3 @@
-//go:build !race
 // +build !race
 
 //

diff --git a/tests/e2e/crud/crud_test.go b/tests/e2e/crud/crud_test.go
@@ -1,4 +1,3 @@
-//go:build e2e
 // +build e2e
 
 //

diff --git a/tests/e2e/hdf5/hdf5.go b/tests/e2e/hdf5/hdf5.go
@@ -1,4 +1,3 @@
-//go:build e2e
 // +build e2e
 
 //

diff --git a/tests/e2e/kubernetes/client/client.go b/tests/e2e/kubernetes/client/client.go
@@ -1,4 +1,3 @@
-//go:build e2e
 // +build e2e
 
 //

diff --git a/tests/e2e/kubernetes/portforward/portforward.go b/tests/e2e/kubernetes/portforward/portforward.go
@@ -1,4 +1,3 @@
-//go:build e2e
 // +build e2e
 
 //

diff --git a/tests/e2e/multiapis/multiapis_test.go b/tests/e2e/multiapis/multiapis_test.go
@@ -1,4 +1,3 @@
-//go:build e2e
 // +build e2e
 
 //

diff --git a/tests/e2e/operation/doc.go b/tests/e2e/operation/doc.go
@@ -1,4 +1,3 @@
-//go:build e2e
 // +build e2e
 
 //

diff --git a/tests/e2e/operation/multi.go b/tests/e2e/operation/multi.go
@@ -1,4 +1,3 @@
-//go:build e2e
 // +build e2e
 
 //

diff --git a/tests/e2e/operation/operation.go b/tests/e2e/operation/operation.go
@@ -1,4 +1,3 @@
-//go:build e2e
 // +build e2e
 
 //

diff --git a/tests/e2e/operation/stream.go b/tests/e2e/operation/stream.go
@@ -1,4 +1,3 @@
-//go:build e2e
 // +build e2e
 
 //

diff --git a/tests/e2e/sidecar/sidecar_test.go b/tests/e2e/sidecar/sidecar_test.go
@@ -1,4 +1,3 @@
-//go:build e2e
 // +build e2e
 
 //