feat(vertexai): support shard_size in google_vertex_ai_index (Goo…

…gleCloudPlatform#8293)
hao-nan-li · Jul 27, 2023 · 74cd743 · 74cd743
1 parent bcf53a6
commit 74cd743
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 0 deletions.
diff --git a/mmv1/products/vertexai/Index.yaml b/mmv1/products/vertexai/Index.yaml
@@ -131,6 +131,16 @@ properties:
               performed. Exact reordering is a procedure where results returned by an
               approximate search algorithm are reordered via a more expensive distance computation.
               Required if tree-AH algorithm is used.
+          - !ruby/object:Api::Type::String
+            name: 'shardSize'
+            description: |-
+              Index data is split into equal parts to be processed. These are called "shards".
+              The shard size must be specified when creating an index. The value must be one of the followings:
+              * SHARD_SIZE_SMALL: Small (2GB)
+              * SHARD_SIZE_MEDIUM: Medium (20GB)
+              * SHARD_SIZE_LARGE: Large (50GB)
+            immutable: true
+            default_from_api: true
           - !ruby/object:Api::Type::String
             name: 'distanceMeasureType'
             description: |-

diff --git a/mmv1/templates/terraform/examples/vertex_ai_index.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_index.tf.erb
@@ -27,6 +27,7 @@ resource "google_vertex_ai_index" "index" {
     config {
       dimensions = 2
       approximate_neighbors_count = 150
+      shard_size = "SHARD_SIZE_SMALL"
       distance_measure_type = "DOT_PRODUCT_DISTANCE"
       algorithm_config {
         tree_ah_config {

diff --git a/mmv1/templates/terraform/examples/vertex_ai_index_streaming.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_index_streaming.tf.erb
@@ -26,6 +26,7 @@ resource "google_vertex_ai_index" "index" {
     contents_delta_uri = "gs://${google_storage_bucket.bucket.name}/contents"
     config {
       dimensions = 2
+      shard_size = "SHARD_SIZE_LARGE"
       distance_measure_type = "COSINE_DISTANCE"
       feature_norm_type = "UNIT_L2_NORM"
       algorithm_config {

diff --git a/mmv1/third_party/terraform/tests/resource_vertex_ai_index_test.go b/mmv1/third_party/terraform/tests/resource_vertex_ai_index_test.go
@@ -89,6 +89,7 @@ resource "google_vertex_ai_index" "index" {
     config {
       dimensions = 2
       approximate_neighbors_count = 150
+      shard_size = "SHARD_SIZE_SMALL"
       distance_measure_type = "DOT_PRODUCT_DISTANCE"
       algorithm_config {
         tree_ah_config {
@@ -145,6 +146,7 @@ resource "google_vertex_ai_index" "index" {
     config {
       dimensions = 2
       approximate_neighbors_count = 150
+      shard_size = "SHARD_SIZE_SMALL"
       distance_measure_type = "DOT_PRODUCT_DISTANCE"
       algorithm_config {
         tree_ah_config {