move param files (#210)

Signed-off-by: Vijayan Balasubramanian <balasvij@amazon.com> (cherry picked from commit ea6eba5) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Feb 28, 2024 · 8db9742 · 8db9742
1 parent 89227b8
commit 8db9742
Show file tree

Hide file tree

Showing 7 changed files with 95 additions and 9 deletions.
diff --git a/vectorsearch/params/corpus/10million/faiss-cohere-768-dp.json b/vectorsearch/params/corpus/10million/faiss-cohere-768-dp.json
@@ -0,0 +1,28 @@
+{
+    "target_index_name": "target_index",
+    "target_field_name": "target_field",
+    "target_index_body": "indices/faiss-index.json",
+    "target_index_primary_shards": 18,
+    "target_index_dimension": 768,
+    "target_index_space_type": "innerproduct",
+
+    "target_index_bulk_size": 100,
+    "target_index_bulk_index_data_set_format": "hdf5",
+    "target_index_bulk_index_data_set_corpus": "cohere-10m",
+    "target_index_bulk_indexing_clients": 10,
+
+    "target_index_max_num_segments": 10,
+    "target_index_force_merge_timeout": 100,
+    "hnsw_ef_search": 100,
+    "hnsw_ef_construction": 100,
+
+    "query_k": 100,
+    "query_body": {
+         "docvalue_fields" : ["_id"],
+         "stored_fields" : "_none_"
+    },
+
+    "query_data_set_format": "hdf5",
+    "query_data_set_corpus": "cohere-10m",
+    "query_count": 10000
+  }
diff --git a/vectorsearch/params/corpus/10million/lucene-cohere-768-dp.json b/vectorsearch/params/corpus/10million/lucene-cohere-768-dp.json
@@ -0,0 +1,28 @@
+{
+    "target_index_name": "target_index",
+    "target_field_name": "target_field",
+    "target_index_body": "indices/lucene-index.json",
+    "target_index_primary_shards": 18,
+    "target_index_dimension": 768,
+    "target_index_space_type": "innerproduct",
+
+    "target_index_bulk_size": 100,
+    "target_index_bulk_index_data_set_format": "hdf5",
+    "target_index_bulk_index_data_set_corpus": "cohere-10m",
+    "target_index_bulk_indexing_clients": 10,
+
+    "target_index_max_num_segments": 10,
+    "target_index_force_merge_timeout": 100,
+    "hnsw_ef_search": 100,
+    "hnsw_ef_construction": 100,
+
+    "query_k": 100,
+    "query_body": {
+         "docvalue_fields" : ["_id"],
+         "stored_fields" : "_none_"
+    },
+
+    "query_data_set_format": "hdf5",
+    "query_data_set_corpus": "cohere-10m",
+    "query_count": 10000
+  }
diff --git a/vectorsearch/params/corpus/10million/nmslib-cohere-768-dp.json b/vectorsearch/params/corpus/10million/nmslib-cohere-768-dp.json
@@ -0,0 +1,30 @@
+{
+    "target_index_name": "target_index",
+    "target_field_name": "target_field",
+    "target_index_body": "indices/nmslib-index.json",
+    "target_index_primary_shards": 18,
+    "target_index_dimension": 768,
+    "target_index_space_type": "innerproduct",
+
+    "target_index_bulk_size": 100,
+    "target_index_bulk_index_data_set_format": "hdf5",
+    "target_index_bulk_index_data_set_corpus": "cohere-10m",
+    "target_index_bulk_indexing_clients": 10,
+
+    "target_index_max_num_segments": 10,
+    "target_index_force_merge_timeout": 100,
+    "hnsw_ef_search": 100,
+    "hnsw_ef_construction": 100,
+
+    "query_k": 100,
+    "query_body": {
+         "docvalue_fields" : ["_id"],
+         "stored_fields" : "_none_"
+    },
+
+    "query_data_set_format": "hdf5",
+    "query_data_set_corpus": "cohere-1m",
+    "neighbors_data_set_corpus": "cohere-10m",
+    "neighbors_data_set_format": "hdf5",
+    "query_count": 10000
+  }
diff --git a/...ch/params/corpus/faiss-cohere-768-dp.json → .../corpus/1million/faiss-cohere-768-dp.json b/...ch/params/corpus/faiss-cohere-768-dp.json → .../corpus/1million/faiss-cohere-768-dp.json
@@ -8,7 +8,7 @@
 
     "target_index_bulk_size": 100,
     "target_index_bulk_index_data_set_format": "hdf5",
-    "target_index_bulk_index_data_set_corpus": "cohere-100k",
+    "target_index_bulk_index_data_set_corpus": "cohere-1m",
     "target_index_bulk_indexing_clients": 10,
 
     "target_index_max_num_segments": 10,
@@ -23,6 +23,6 @@
     },
 
     "query_data_set_format": "hdf5",
-    "query_data_set_corpus":"cohere-100k",
+    "query_data_set_corpus": "cohere-1m",
     "query_count": 10000
   }
diff --git a/...h/params/corpus/lucene-cohere-768-dp.json → ...corpus/1million/lucene-cohere-768-dp.json b/...h/params/corpus/lucene-cohere-768-dp.json → ...corpus/1million/lucene-cohere-768-dp.json
@@ -8,7 +8,7 @@
 
     "target_index_bulk_size": 100,
     "target_index_bulk_index_data_set_format": "hdf5",
-    "target_index_bulk_index_data_set_corpus": "cohere",
+    "target_index_bulk_index_data_set_corpus": "cohere-1m",
     "target_index_bulk_indexing_clients": 10,
 
     "target_index_max_num_segments": 10,
@@ -23,6 +23,6 @@
     },
 
     "query_data_set_format": "hdf5",
-    "query_data_set_corpus": "cohere",
-    "query_count": 1000
+    "query_data_set_corpus": "cohere-1m",
+    "query_count": 10000
   }
diff --git a/...h/params/corpus/nmslib-cohere-768-dp.json → ...corpus/1million/nmslib-cohere-768-dp.json b/...h/params/corpus/nmslib-cohere-768-dp.json → ...corpus/1million/nmslib-cohere-768-dp.json
diff --git a/vectorsearch/workload.json b/vectorsearch/workload.json
@@ -16,7 +16,7 @@
         {
           "source-file": "documents-1k.hdf5.bz2",
           "source-format": "hdf5",
-          "document-count": 1000,
+          "document-count": 1000
         }
       ]
     },
@@ -27,7 +27,7 @@
         {
           "source-file": "documents-100k.hdf5.bz2",
           "source-format": "hdf5",
-          "document-count": 100000,
+          "document-count": 100000
         }
       ]
     },
@@ -38,7 +38,7 @@
         {
           "source-file": "documents-1m.hdf5.bz2",
           "source-format": "hdf5",
-          "document-count": 1000000,
+          "document-count": 1000000
         }
       ]
     },
@@ -49,7 +49,7 @@
         {
           "source-file": "documents-10m.hdf5.bz2",
           "source-format": "hdf5",
-          "document-count": 10000000,
+          "document-count": 10000000
         }
       ]
     }