From c5681a57ff2dedd2c2ca1fe6c91948b894bfc884 Mon Sep 17 00:00:00 2001 From: Shotaro Kohama Date: Fri, 7 Jul 2023 16:26:13 -0700 Subject: [PATCH 1/4] feat: support shard-size in google_vertex_ai_index --- mmv1/products/vertexai/Index.yaml | 9 +++++++++ mmv1/templates/terraform/examples/vertex_ai_index.tf.erb | 1 + .../terraform/examples/vertex_ai_index_streaming.tf.erb | 1 + .../terraform/tests/resource_vertex_ai_index_test.go | 2 ++ 4 files changed, 13 insertions(+) diff --git a/mmv1/products/vertexai/Index.yaml b/mmv1/products/vertexai/Index.yaml index f288a3b3c458..e76555c831a7 100644 --- a/mmv1/products/vertexai/Index.yaml +++ b/mmv1/products/vertexai/Index.yaml @@ -131,6 +131,15 @@ properties: performed. Exact reordering is a procedure where results returned by an approximate search algorithm are reordered via a more expensive distance computation. Required if tree-AH algorithm is used. + - !ruby/object:Api::Type::String + name: 'shardSize' + description: |- + Index data is split into equal parts to be processed. These are called "shards". + The shard size must be specified when creating an index. The value must be one of the followings: + * SHARD_SIZE_SMALL: Small (2GB) + * SHARD_SIZE_MEDIUM: Medium (20GB) + * SHARD_SIZE_LARGE: Large (50GB) + default_value: 'SHARD_SIZE_MEDIUM' - !ruby/object:Api::Type::String name: 'distanceMeasureType' description: |- diff --git a/mmv1/templates/terraform/examples/vertex_ai_index.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_index.tf.erb index ccdfdad0126f..1a1e90b2c329 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_index.tf.erb +++ b/mmv1/templates/terraform/examples/vertex_ai_index.tf.erb @@ -27,6 +27,7 @@ resource "google_vertex_ai_index" "index" { config { dimensions = 2 approximate_neighbors_count = 150 + shard_size = "SHARD_SIZE_SMALL" distance_measure_type = "DOT_PRODUCT_DISTANCE" algorithm_config { tree_ah_config { diff --git a/mmv1/templates/terraform/examples/vertex_ai_index_streaming.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_index_streaming.tf.erb index 181f3f14691a..acaf268b8254 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_index_streaming.tf.erb +++ b/mmv1/templates/terraform/examples/vertex_ai_index_streaming.tf.erb @@ -26,6 +26,7 @@ resource "google_vertex_ai_index" "index" { contents_delta_uri = "gs://${google_storage_bucket.bucket.name}/contents" config { dimensions = 2 + shard_size = "SHARD_SIZE_LARGE" distance_measure_type = "COSINE_DISTANCE" feature_norm_type = "UNIT_L2_NORM" algorithm_config { diff --git a/mmv1/third_party/terraform/tests/resource_vertex_ai_index_test.go b/mmv1/third_party/terraform/tests/resource_vertex_ai_index_test.go index 8710a56c4bad..6b3bee88eee3 100644 --- a/mmv1/third_party/terraform/tests/resource_vertex_ai_index_test.go +++ b/mmv1/third_party/terraform/tests/resource_vertex_ai_index_test.go @@ -88,6 +88,7 @@ resource "google_vertex_ai_index" "index" { config { dimensions = 2 approximate_neighbors_count = 150 + shard_size = "SHARD_SIZE_SMALL" distance_measure_type = "DOT_PRODUCT_DISTANCE" algorithm_config { tree_ah_config { @@ -144,6 +145,7 @@ resource "google_vertex_ai_index" "index" { config { dimensions = 2 approximate_neighbors_count = 150 + shard_size = "SHARD_SIZE_SMALL" distance_measure_type = "DOT_PRODUCT_DISTANCE" algorithm_config { tree_ah_config { From d2f1319e3f53adca2660c3b17f076c1c93801821 Mon Sep 17 00:00:00 2001 From: Shotaro Kohama Date: Sat, 8 Jul 2023 23:04:07 -0700 Subject: [PATCH 2/4] feat: add `immutable: true` to shard_size --- mmv1/products/vertexai/Index.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/mmv1/products/vertexai/Index.yaml b/mmv1/products/vertexai/Index.yaml index e76555c831a7..27d256c9d844 100644 --- a/mmv1/products/vertexai/Index.yaml +++ b/mmv1/products/vertexai/Index.yaml @@ -139,6 +139,7 @@ properties: * SHARD_SIZE_SMALL: Small (2GB) * SHARD_SIZE_MEDIUM: Medium (20GB) * SHARD_SIZE_LARGE: Large (50GB) + immutable: true default_value: 'SHARD_SIZE_MEDIUM' - !ruby/object:Api::Type::String name: 'distanceMeasureType' From 9178158721e91bbd16079a35a09a99fe7119abb8 Mon Sep 17 00:00:00 2001 From: Shotaro Kohama Date: Mon, 10 Jul 2023 14:03:49 -0700 Subject: [PATCH 3/4] fix: use default_from_api intead of default_value and immutable --- mmv1/products/vertexai/Index.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mmv1/products/vertexai/Index.yaml b/mmv1/products/vertexai/Index.yaml index 27d256c9d844..b7843939be94 100644 --- a/mmv1/products/vertexai/Index.yaml +++ b/mmv1/products/vertexai/Index.yaml @@ -139,8 +139,7 @@ properties: * SHARD_SIZE_SMALL: Small (2GB) * SHARD_SIZE_MEDIUM: Medium (20GB) * SHARD_SIZE_LARGE: Large (50GB) - immutable: true - default_value: 'SHARD_SIZE_MEDIUM' + default_from_api: true - !ruby/object:Api::Type::String name: 'distanceMeasureType' description: |- From b36bed69838e3b86eef510e85e10b72085f6ec00 Mon Sep 17 00:00:00 2001 From: Shotaro Kohama Date: Mon, 10 Jul 2023 14:08:41 -0700 Subject: [PATCH 4/4] fix: restore immutable: true --- mmv1/products/vertexai/Index.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/mmv1/products/vertexai/Index.yaml b/mmv1/products/vertexai/Index.yaml index b7843939be94..ac878a8c5ad2 100644 --- a/mmv1/products/vertexai/Index.yaml +++ b/mmv1/products/vertexai/Index.yaml @@ -139,6 +139,7 @@ properties: * SHARD_SIZE_SMALL: Small (2GB) * SHARD_SIZE_MEDIUM: Medium (20GB) * SHARD_SIZE_LARGE: Large (50GB) + immutable: true default_from_api: true - !ruby/object:Api::Type::String name: 'distanceMeasureType'