diff --git a/vectorsearch/params/corpus/10million/faiss-cohere-768-dp.json b/vectorsearch/params/corpus/10million/faiss-cohere-768-dp.json new file mode 100644 index 00000000..07f95de8 --- /dev/null +++ b/vectorsearch/params/corpus/10million/faiss-cohere-768-dp.json @@ -0,0 +1,28 @@ +{ + "target_index_name": "target_index", + "target_field_name": "target_field", + "target_index_body": "indices/faiss-index.json", + "target_index_primary_shards": 18, + "target_index_dimension": 768, + "target_index_space_type": "innerproduct", + + "target_index_bulk_size": 100, + "target_index_bulk_index_data_set_format": "hdf5", + "target_index_bulk_index_data_set_corpus": "cohere-10m", + "target_index_bulk_indexing_clients": 10, + + "target_index_max_num_segments": 10, + "target_index_force_merge_timeout": 100, + "hnsw_ef_search": 100, + "hnsw_ef_construction": 100, + + "query_k": 100, + "query_body": { + "docvalue_fields" : ["_id"], + "stored_fields" : "_none_" + }, + + "query_data_set_format": "hdf5", + "query_data_set_corpus": "cohere-10m", + "query_count": 10000 + } diff --git a/vectorsearch/params/corpus/10million/lucene-cohere-768-dp.json b/vectorsearch/params/corpus/10million/lucene-cohere-768-dp.json new file mode 100644 index 00000000..2f725aa0 --- /dev/null +++ b/vectorsearch/params/corpus/10million/lucene-cohere-768-dp.json @@ -0,0 +1,28 @@ +{ + "target_index_name": "target_index", + "target_field_name": "target_field", + "target_index_body": "indices/lucene-index.json", + "target_index_primary_shards": 18, + "target_index_dimension": 768, + "target_index_space_type": "innerproduct", + + "target_index_bulk_size": 100, + "target_index_bulk_index_data_set_format": "hdf5", + "target_index_bulk_index_data_set_corpus": "cohere-10m", + "target_index_bulk_indexing_clients": 10, + + "target_index_max_num_segments": 10, + "target_index_force_merge_timeout": 100, + "hnsw_ef_search": 100, + "hnsw_ef_construction": 100, + + "query_k": 100, + "query_body": { + "docvalue_fields" : ["_id"], + "stored_fields" : "_none_" + }, + + "query_data_set_format": "hdf5", + "query_data_set_corpus": "cohere-10m", + "query_count": 10000 + } diff --git a/vectorsearch/params/corpus/10million/nmslib-cohere-768-dp.json b/vectorsearch/params/corpus/10million/nmslib-cohere-768-dp.json new file mode 100644 index 00000000..6fdee69f --- /dev/null +++ b/vectorsearch/params/corpus/10million/nmslib-cohere-768-dp.json @@ -0,0 +1,30 @@ +{ + "target_index_name": "target_index", + "target_field_name": "target_field", + "target_index_body": "indices/nmslib-index.json", + "target_index_primary_shards": 18, + "target_index_dimension": 768, + "target_index_space_type": "innerproduct", + + "target_index_bulk_size": 100, + "target_index_bulk_index_data_set_format": "hdf5", + "target_index_bulk_index_data_set_corpus": "cohere-10m", + "target_index_bulk_indexing_clients": 10, + + "target_index_max_num_segments": 10, + "target_index_force_merge_timeout": 100, + "hnsw_ef_search": 100, + "hnsw_ef_construction": 100, + + "query_k": 100, + "query_body": { + "docvalue_fields" : ["_id"], + "stored_fields" : "_none_" + }, + + "query_data_set_format": "hdf5", + "query_data_set_corpus": "cohere-1m", + "neighbors_data_set_corpus": "cohere-10m", + "neighbors_data_set_format": "hdf5", + "query_count": 10000 + } diff --git a/vectorsearch/params/corpus/faiss-cohere-768-dp.json b/vectorsearch/params/corpus/1million/faiss-cohere-768-dp.json similarity index 87% rename from vectorsearch/params/corpus/faiss-cohere-768-dp.json rename to vectorsearch/params/corpus/1million/faiss-cohere-768-dp.json index 1625addb..3266edd4 100644 --- a/vectorsearch/params/corpus/faiss-cohere-768-dp.json +++ b/vectorsearch/params/corpus/1million/faiss-cohere-768-dp.json @@ -8,7 +8,7 @@ "target_index_bulk_size": 100, "target_index_bulk_index_data_set_format": "hdf5", - "target_index_bulk_index_data_set_corpus": "cohere-100k", + "target_index_bulk_index_data_set_corpus": "cohere-1m", "target_index_bulk_indexing_clients": 10, "target_index_max_num_segments": 10, @@ -23,6 +23,6 @@ }, "query_data_set_format": "hdf5", - "query_data_set_corpus":"cohere-100k", + "query_data_set_corpus": "cohere-1m", "query_count": 10000 } diff --git a/vectorsearch/params/corpus/lucene-cohere-768-dp.json b/vectorsearch/params/corpus/1million/lucene-cohere-768-dp.json similarity index 84% rename from vectorsearch/params/corpus/lucene-cohere-768-dp.json rename to vectorsearch/params/corpus/1million/lucene-cohere-768-dp.json index 4d2c05c7..5b5fc2cc 100644 --- a/vectorsearch/params/corpus/lucene-cohere-768-dp.json +++ b/vectorsearch/params/corpus/1million/lucene-cohere-768-dp.json @@ -8,7 +8,7 @@ "target_index_bulk_size": 100, "target_index_bulk_index_data_set_format": "hdf5", - "target_index_bulk_index_data_set_corpus": "cohere", + "target_index_bulk_index_data_set_corpus": "cohere-1m", "target_index_bulk_indexing_clients": 10, "target_index_max_num_segments": 10, @@ -23,6 +23,6 @@ }, "query_data_set_format": "hdf5", - "query_data_set_corpus": "cohere", - "query_count": 1000 + "query_data_set_corpus": "cohere-1m", + "query_count": 10000 } diff --git a/vectorsearch/params/corpus/nmslib-cohere-768-dp.json b/vectorsearch/params/corpus/1million/nmslib-cohere-768-dp.json similarity index 100% rename from vectorsearch/params/corpus/nmslib-cohere-768-dp.json rename to vectorsearch/params/corpus/1million/nmslib-cohere-768-dp.json diff --git a/vectorsearch/workload.json b/vectorsearch/workload.json index fb2b498f..157ce142 100644 --- a/vectorsearch/workload.json +++ b/vectorsearch/workload.json @@ -16,7 +16,7 @@ { "source-file": "documents-1k.hdf5.bz2", "source-format": "hdf5", - "document-count": 1000, + "document-count": 1000 } ] }, @@ -27,7 +27,7 @@ { "source-file": "documents-100k.hdf5.bz2", "source-format": "hdf5", - "document-count": 100000, + "document-count": 100000 } ] }, @@ -38,7 +38,7 @@ { "source-file": "documents-1m.hdf5.bz2", "source-format": "hdf5", - "document-count": 1000000, + "document-count": 1000000 } ] }, @@ -49,7 +49,7 @@ { "source-file": "documents-10m.hdf5.bz2", "source-format": "hdf5", - "document-count": 10000000, + "document-count": 10000000 } ] }