From 875cad1f30ddd7edda8de0700e7b53d160fa67d0 Mon Sep 17 00:00:00 2001 From: Sai Medhini Reddy Maryada <117196660+saimedhi@users.noreply.github.com> Date: Thu, 7 Mar 2024 06:15:42 -0800 Subject: [PATCH] Added K-NN APIs get_model, search_model, delete_model, train_model (#192) * Added Knn APIs get_model, search_model, delete_model, train_model Signed-off-by: saimedhi * Added Knn APIs get_model, search_model, delete_model, train_model Signed-off-by: saimedhi * Added spec for k-NN apis Signed-off-by: saimedhi * Added spec for k-NN apis Signed-off-by: saimedhi --------- Signed-off-by: saimedhi --- model/common_strings.smithy | 7 + model/knn/delete_model/operations.smithy | 23 ++++ model/knn/delete_model/structures.smithy | 18 +++ model/knn/get_model/operations.smithy | 23 ++++ model/knn/get_model/structures.smithy | 18 +++ model/knn/search_model/operations.smithy | 33 +++++ model/knn/search_model/structures.smithy | 167 +++++++++++++++++++++++ model/knn/train_model/operations.smithy | 32 +++++ model/knn/train_model/structures.smithy | 55 ++++++++ model/opensearch.smithy | 6 + 10 files changed, 382 insertions(+) create mode 100644 model/knn/delete_model/operations.smithy create mode 100644 model/knn/delete_model/structures.smithy create mode 100644 model/knn/get_model/operations.smithy create mode 100644 model/knn/get_model/structures.smithy create mode 100644 model/knn/search_model/operations.smithy create mode 100644 model/knn/search_model/structures.smithy create mode 100644 model/knn/train_model/operations.smithy create mode 100644 model/knn/train_model/structures.smithy diff --git a/model/common_strings.smithy b/model/common_strings.smithy index b6a2c928c..a2021587e 100644 --- a/model/common_strings.smithy +++ b/model/common_strings.smithy @@ -247,6 +247,10 @@ string PathTaskId @documentation("The name of the template.") string PathTemplateName +@pattern("^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$") +@documentation("The id of the model.") +string PathModelId + @documentation("The analyzer to use for the query string.") string Analyzer @@ -281,6 +285,9 @@ string Lang @documentation("Comma-separated list of the persistent ids of the nodes to exclude from the voting configuration. If specified, you may not also specify ?node_names.") string NodeIds +@documentation("Preferred node to execute training.") +string NodeId + @documentation("Comma-separated list of the names of the nodes to exclude from the voting configuration. If specified, you may not also specify ?node_ids.") string NodeNames diff --git a/model/knn/delete_model/operations.smithy b/model/knn/delete_model/operations.smithy new file mode 100644 index 000000000..7bc4459f2 --- /dev/null +++ b/model/knn/delete_model/operations.smithy @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@externalDocumentation( + "API Reference": "https://opensearch.org/docs/latest/search-plugins/knn/api/#delete-model" +) + +@xOperationGroup("knn.delete_model") +@xVersionAdded("1.0") +@suppress(["HttpMethodSemantics.UnexpectedPayload"]) +@suppress(["HttpUriConflict"]) +@http(method: "DELETE", uri: "/_plugins/_knn/models/{model_id}") +@documentation("Used to delete a particular model in the cluster.") +operation KNNDeleteModel { + input: KNNDeleteModel_Input, + output: KNNDeleteModel_Output +} diff --git a/model/knn/delete_model/structures.smithy b/model/knn/delete_model/structures.smithy new file mode 100644 index 000000000..2f9fe30e0 --- /dev/null +++ b/model/knn/delete_model/structures.smithy @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@input +structure KNNDeleteModel_Input { + @required + @httpLabel + model_id: PathModelId, +} + +// TODO: Fill in Output Structure +structure KNNDeleteModel_Output {} diff --git a/model/knn/get_model/operations.smithy b/model/knn/get_model/operations.smithy new file mode 100644 index 000000000..557678686 --- /dev/null +++ b/model/knn/get_model/operations.smithy @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@externalDocumentation( + "API Reference": "https://opensearch.org/docs/latest/search-plugins/knn/api/#get-model" +) + +@xOperationGroup("knn.get_model") +@xVersionAdded("1.0") +@readonly +@suppress(["HttpUriConflict"]) +@http(method: "GET", uri: "/_plugins/_knn/models/{model_id}") +@documentation("Used to retrieve information about models present in the cluster.") +operation KNNGetModel { + input: KNNGetModel_Input, + output: KNNGetModel_Output +} diff --git a/model/knn/get_model/structures.smithy b/model/knn/get_model/structures.smithy new file mode 100644 index 000000000..ad0bafd49 --- /dev/null +++ b/model/knn/get_model/structures.smithy @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@input +structure KNNGetModel_Input { + @required + @httpLabel + model_id: PathModelId, +} + +// TODO: Fill in Output Structure +structure KNNGetModel_Output {} diff --git a/model/knn/search_model/operations.smithy b/model/knn/search_model/operations.smithy new file mode 100644 index 000000000..a6becc931 --- /dev/null +++ b/model/knn/search_model/operations.smithy @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@externalDocumentation( + "API Reference": "https://opensearch.org/docs/latest/search-plugins/knn/api/#search-model" +) + +@xOperationGroup("knn.search_models") +@xVersionAdded("1.0") +@readonly +@suppress(["HttpUriConflict"]) +@http(method: "GET", uri: "/_plugins/_knn/models/_search") +@documentation("Use an OpenSearch query to search for models in the index.") +operation KNNSearchModels_Get { + input: KNNSearchModels_Get_Input, + output: KNNSearchModels_Output +} + +@xOperationGroup("knn.search_models") +@xVersionAdded("1.0") +@suppress(["HttpUriConflict"]) +@http(method: "POST", uri: "/_plugins/_knn/models/_search") +@documentation("Use an OpenSearch query to search for models in the index.") +operation KNNSearchModels_Post { + input: KNNSearchModels_Post_Input, + output: KNNSearchModels_Output +} diff --git a/model/knn/search_model/structures.smithy b/model/knn/search_model/structures.smithy new file mode 100644 index 000000000..c21a945f4 --- /dev/null +++ b/model/knn/search_model/structures.smithy @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@mixin +structure KNNSearchModels_QueryParams { + @httpQuery("analyzer") + analyzer: Analyzer, + + @httpQuery("analyze_wildcard") + @default(false) + analyze_wildcard: AnalyzeWildcard, + + @httpQuery("ccs_minimize_roundtrips") + @default(true) + ccs_minimize_roundtrips: CcsMinimizeRoundtrips, + + @httpQuery("default_operator") + @default("OR") + default_operator: DefaultOperator, + + @httpQuery("df") + df: Df, + + @httpQuery("explain") + @documentation("Specify whether to return detailed information about score computation as part of a hit.") + explain: Explain, + + @httpQuery("stored_fields") + stored_fields: StoredFields, + + @httpQuery("docvalue_fields") + docvalue_fields: DocvalueFields, + + @httpQuery("from") + @default(0) + from: From, + + @httpQuery("ignore_unavailable") + ignore_unavailable: IgnoreUnavailable, + + @httpQuery("ignore_throttled") + ignore_throttled: IgnoreThrottled, + + @httpQuery("allow_no_indices") + allow_no_indices: AllowNoIndices, + + @httpQuery("expand_wildcards") + @default("open") + expand_wildcards: ExpandWildcards, + + @httpQuery("lenient") + lenient: Lenient, + + @httpQuery("preference") + @default("random") + preference: Preference, + + @httpQuery("q") + q: Q, + + @httpQuery("routing") + routing: Routings, + + @httpQuery("scroll") + scroll: Scroll, + + @httpQuery("search_type") + search_type: SearchType, + + @httpQuery("size") + @documentation("Number of hits to return.") + @default(10) + size: Size, + + @httpQuery("sort") + sort: Sort, + + @httpQuery("_source") + _source: Source, + + @httpQuery("_source_excludes") + _source_excludes: SourceExcludes, + + @httpQuery("_source_includes") + _source_includes: SourceIncludes, + + @httpQuery("terminate_after") + terminate_after: TerminateAfter, + + @httpQuery("stats") + stats: Stats, + + @httpQuery("suggest_field") + suggest_field: SuggestField, + + @httpQuery("suggest_mode") + @default("missing") + suggest_mode: SuggestMode, + + @httpQuery("suggest_size") + suggest_size: SuggestSize, + + @httpQuery("suggest_text") + suggest_text: SuggestText, + + @httpQuery("timeout") + timeout: Timeout, + + @httpQuery("track_scores") + track_scores: TrackScores, + + @httpQuery("track_total_hits") + track_total_hits: TrackTotalHits, + + @httpQuery("allow_partial_search_results") + @default(true) + allow_partial_search_results: AllowPartialSearchResults, + + @httpQuery("typed_keys") + typed_keys: TypedKeys, + + @httpQuery("version") + version: WithVersion, + + @httpQuery("seq_no_primary_term") + seq_no_primary_term: SeqNoPrimaryTerm, + + @httpQuery("request_cache") + request_cache: RequestCache, + + @httpQuery("batched_reduce_size") + @default(512) + batched_reduce_size: BatchedReduceSize, + + @httpQuery("max_concurrent_shard_requests") + @documentation("The number of concurrent shard requests per node this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests.") + @default(5) + max_concurrent_shard_requests: MaxConcurrentShardRequests, + + @httpQuery("pre_filter_shard_size") + pre_filter_shard_size: PreFilterShardSize, + + @httpQuery("rest_total_hits_as_int") + @default(false) + rest_total_hits_as_int: RestTotalHitsAsInt, +} + +// TODO: Fill in Body Parameters +structure KNNSearchModels_BodyParams {} + +@input +structure KNNSearchModels_Get_Input with [KNNSearchModels_QueryParams] {} + +@input +structure KNNSearchModels_Post_Input with [KNNSearchModels_QueryParams] { + @httpPayload + content: KNNSearchModels_BodyParams, +} + +// TODO: Fill in Output Structure +structure KNNSearchModels_Output {} diff --git a/model/knn/train_model/operations.smithy b/model/knn/train_model/operations.smithy new file mode 100644 index 000000000..a65032765 --- /dev/null +++ b/model/knn/train_model/operations.smithy @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@externalDocumentation( + "API Reference": "https://opensearch.org/docs/latest/search-plugins/knn/api/#train-model" +) + +@xOperationGroup("knn.train_model") +@xVersionAdded("1.0") +@suppress(["HttpUriConflict"]) +@http(method: "POST", uri: "/_plugins/_knn/models/_train") +@documentation("Create and train a model that can be used for initializing k-NN native library indexes during indexing.") +operation KNNTrainModel { + input: KNNTrainModel_Input, + output: KNNTrainModel_Output +} + +@xOperationGroup("knn.train_model") +@xVersionAdded("1.0") +@suppress(["HttpUriConflict"]) +@http(method: "POST", uri: "/_plugins/_knn/models/{model_id}/_train") +@documentation("Create and train a model that can be used for initializing k-NN native library indexes during indexing.") +operation KNNTrainModel_WithModelId { + input: KNNTrainModel_WithModelId_Input, + output: KNNTrainModel_Output +} diff --git a/model/knn/train_model/structures.smithy b/model/knn/train_model/structures.smithy new file mode 100644 index 000000000..096667250 --- /dev/null +++ b/model/knn/train_model/structures.smithy @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@mixin +structure KNNTrainModel_QueryParams { + @httpQuery("preference") + preference: NodeId, +} + +structure KNNTrainModel_BodyParams { + @required + training_index: String, + + @required + training_field: String, + + @required + dimension: Integer, + + max_training_vector_count: Integer, + + search_size: Integer, + + description: String, + + @required + method: String, +} + +@input +structure KNNTrainModel_Input with [KNNTrainModel_QueryParams] { + @required + @httpPayload + content: KNNTrainModel_BodyParams, +} + +@input +structure KNNTrainModel_WithModelId_Input with [KNNTrainModel_QueryParams] { + @required + @httpPayload + content: KNNTrainModel_BodyParams, + + @required + @httpLabel + model_id: PathModelId, +} + +// TODO: Fill in Output Structure +structure KNNTrainModel_Output {} diff --git a/model/opensearch.smithy b/model/opensearch.smithy index ccb6b3805..e8a1b55bc 100644 --- a/model/opensearch.smithy +++ b/model/opensearch.smithy @@ -256,6 +256,12 @@ service OpenSearch { IngestSimulate_Get_WithId, IngestSimulate_Post, IngestSimulate_Post_WithId, + KNNDeleteModel, + KNNGetModel, + KNNSearchModels_Get, + KNNSearchModels_Post, + KNNTrainModel, + KNNTrainModel_WithModelId, Mget_Get, Mget_Get_WithIndex, Mget_Post,