Merge branch 'main' into add/knn.warmup

Signed-off-by: Sai Medhini Reddy Maryada <117196660+saimedhi@users.noreply.github.com>

.github/workflows/coverage.yml

@@ -0,0 +1,42 @@
+name: API Coverage
+
+on: [push, pull_request]
+
+env:
+  JAVA_VERSION: 11
+  OPENSEARCH_INITIAL_ADMIN_PASSWORD: BobgG7YrtsdKf9M
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+      - name: Build and Run Docker Container
+        run: |
+          docker build coverage --tag opensearch-with-api-plugin
+          docker run -d -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e OPENSEARCH_INITIAL_ADMIN_PASSWORD="$OPENSEARCH_INITIAL_ADMIN_PASSWORD" opensearch-with-api-plugin
+          sleep 15
+      - name: Display OpenSearch Info
+        run: |
+          curl   -ks -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" https://localhost:9200/ | jq
+      - name: Dump and Compare API
+        run: |
+          curl -ks -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" https://localhost:9200/_plugins/api | jq > OpenSearch.auto.openapi.json
+          docker run --rm --mount type=bind,source=.,target=/specs openapitools/openapi-diff:latest /specs/OpenSearch.openapi.json /specs/OpenSearch.auto.openapi.json --json /specs/diff.json
+      - name: Show Diff
+        run: |
+          echo "-------- Missing APIs"
+          jq -r '.newEndpoints | group_by(.pathUrl)[] | "\(.[0].pathUrl): \([.[].method])"' diff.json
+          echo "-------- Legacy APIs"
+          jq -r '.missingEndpoints | group_by(.pathUrl)[] | "\(.[0].pathUrl): \([.[].method])"' diff.json
+      - name: Gather Coverage
+        id: coverage
+        shell: bash
+        run: |
+          current=`jq -r '.paths | keys | length' OpenSearch.openapi.json`
+          total=`jq -r '.paths | keys | length' OpenSearch.auto.openapi.json`
+          percent=$((current * 100 / total))
+          echo "API specs implemented for $current/$total ($percent%) APIs."

coverage/Dockerfile

@@ -0,0 +1,7 @@
+ARG OPENSEARCH_VERSION=2.12.0
+FROM opensearchproject/opensearch:${OPENSEARCH_VERSION}
+ARG OPENSEARCH_VERSION
+RUN /usr/share/opensearch/bin/opensearch-plugin \
+    install \
+    --batch \
+    https://github.com/dblock/opensearch-api/releases/download/v${OPENSEARCH_VERSION}/opensearch-api-${OPENSEARCH_VERSION}.0.zip

coverage/README.md

@@ -0,0 +1,7 @@
+### API Coverage
+
+Uses the [opensearch-api plugin](https://github.com/dblock/opensearch-api), and [openapi-diff](https://github.com/OpenAPITools/openapi-diff) to show the difference between OpenSearch APIs, and the [OpenAPI spec checked into this repo](../OpenSearch.openapi.json).
+
+API coverage is run on all pull requests via the [coverage workflow](../.github/workflows/coverage.yml).
+
+

model/common_strings.smithy

@@ -247,6 +247,10 @@ string PathTaskId
 @documentation("The name of the template.")
 string PathTemplateName
 
+@pattern("^(?!_|template|query|field|point|clear|usage|stats|hot|reload|painless).+$")
+@documentation("The id of the model.")
+string PathModelId
+
 @documentation("The analyzer to use for the query string.")
 string Analyzer
 
@@ -281,6 +285,9 @@ string Lang
 @documentation("Comma-separated list of the persistent ids of the nodes to exclude from the voting configuration. If specified, you may not also specify ?node_names.")
 string NodeIds
 
+@documentation("Preferred node to execute training.")
+string NodeId
+
 @documentation("Comma-separated list of the names of the nodes to exclude from the voting configuration. If specified, you may not also specify ?node_ids.")
 string NodeNames
 

model/knn/delete_model/operations.smithy

@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+//  The OpenSearch Contributors require contributions made to
+//  this file be licensed under the Apache-2.0 license or a
+//  compatible open source license.
+
+$version: "2"
+namespace OpenSearch
+
+@externalDocumentation(
+    "API Reference": "https://opensearch.org/docs/latest/search-plugins/knn/api/#delete-model"
+)
+
+@xOperationGroup("knn.delete_model")
+@xVersionAdded("1.0")
+@suppress(["HttpMethodSemantics.UnexpectedPayload"])
+@suppress(["HttpUriConflict"])
+@http(method: "DELETE", uri: "/_plugins/_knn/models/{model_id}")
+@documentation("Used to delete a particular model in the cluster.")
+operation KNNDeleteModel {
+    input: KNNDeleteModel_Input,
+    output: KNNDeleteModel_Output
+}

model/knn/delete_model/structures.smithy

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+//  The OpenSearch Contributors require contributions made to
+//  this file be licensed under the Apache-2.0 license or a
+//  compatible open source license.
+
+$version: "2"
+namespace OpenSearch
+
+@input
+structure KNNDeleteModel_Input {
+    @required
+    @httpLabel
+    model_id: PathModelId,
+}
+
+// TODO: Fill in Output Structure
+structure KNNDeleteModel_Output {}

model/knn/get_model/operations.smithy

@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+//  The OpenSearch Contributors require contributions made to
+//  this file be licensed under the Apache-2.0 license or a
+//  compatible open source license.
+
+$version: "2"
+namespace OpenSearch
+
+@externalDocumentation(
+    "API Reference": "https://opensearch.org/docs/latest/search-plugins/knn/api/#get-model"
+)
+
+@xOperationGroup("knn.get_model")
+@xVersionAdded("1.0")
+@readonly
+@suppress(["HttpUriConflict"])
+@http(method: "GET", uri: "/_plugins/_knn/models/{model_id}")
+@documentation("Used to retrieve information about models present in the cluster.")
+operation KNNGetModel {
+    input: KNNGetModel_Input,
+    output: KNNGetModel_Output
+}

model/knn/get_model/structures.smithy

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+//  The OpenSearch Contributors require contributions made to
+//  this file be licensed under the Apache-2.0 license or a
+//  compatible open source license.
+
+$version: "2"
+namespace OpenSearch
+
+@input
+structure KNNGetModel_Input {
+    @required
+    @httpLabel
+    model_id: PathModelId,
+}
+
+// TODO: Fill in Output Structure
+structure KNNGetModel_Output {}

model/knn/search_model/operations.smithy

@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+//  The OpenSearch Contributors require contributions made to
+//  this file be licensed under the Apache-2.0 license or a
+//  compatible open source license.
+
+$version: "2"
+namespace OpenSearch
+
+@externalDocumentation(
+    "API Reference": "https://opensearch.org/docs/latest/search-plugins/knn/api/#search-model"
+)
+
+@xOperationGroup("knn.search_models")
+@xVersionAdded("1.0")
+@readonly
+@suppress(["HttpUriConflict"])
+@http(method: "GET", uri: "/_plugins/_knn/models/_search")
+@documentation("Use an OpenSearch query to search for models in the index.")
+operation KNNSearchModels_Get {
+    input: KNNSearchModels_Get_Input,
+    output: KNNSearchModels_Output
+}
+
+@xOperationGroup("knn.search_models")
+@xVersionAdded("1.0")
+@suppress(["HttpUriConflict"])
+@http(method: "POST", uri: "/_plugins/_knn/models/_search")
+@documentation("Use an OpenSearch query to search for models in the index.")
+operation KNNSearchModels_Post {
+    input: KNNSearchModels_Post_Input,
+    output: KNNSearchModels_Output
+}

model/knn/search_model/structures.smithy

@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+//  The OpenSearch Contributors require contributions made to
+//  this file be licensed under the Apache-2.0 license or a
+//  compatible open source license.
+
+$version: "2"
+namespace OpenSearch
+
+@mixin
+structure KNNSearchModels_QueryParams {
+    @httpQuery("analyzer")
+    analyzer: Analyzer,
+
+    @httpQuery("analyze_wildcard")
+    @default(false)
+    analyze_wildcard: AnalyzeWildcard,
+
+    @httpQuery("ccs_minimize_roundtrips")
+    @default(true)
+    ccs_minimize_roundtrips: CcsMinimizeRoundtrips,
+
+    @httpQuery("default_operator")
+    @default("OR")
+    default_operator: DefaultOperator,
+
+    @httpQuery("df")
+    df: Df,
+
+    @httpQuery("explain")
+    @documentation("Specify whether to return detailed information about score computation as part of a hit.")
+    explain: Explain,
+
+    @httpQuery("stored_fields")
+    stored_fields: StoredFields,
+
+    @httpQuery("docvalue_fields")
+    docvalue_fields: DocvalueFields,
+
+    @httpQuery("from")
+    @default(0)
+    from: From,
+
+    @httpQuery("ignore_unavailable")
+    ignore_unavailable: IgnoreUnavailable,
+
+    @httpQuery("ignore_throttled")
+    ignore_throttled: IgnoreThrottled,
+
+    @httpQuery("allow_no_indices")
+    allow_no_indices: AllowNoIndices,
+
+    @httpQuery("expand_wildcards")
+    @default("open")
+    expand_wildcards: ExpandWildcards,
+
+    @httpQuery("lenient")
+    lenient: Lenient,
+
+    @httpQuery("preference")
+    @default("random")
+    preference: Preference,
+
+    @httpQuery("q")
+    q: Q,
+
+    @httpQuery("routing")
+    routing: Routings,
+
+    @httpQuery("scroll")
+    scroll: Scroll,
+
+    @httpQuery("search_type")
+    search_type: SearchType,
+
+    @httpQuery("size")
+    @documentation("Number of hits to return.")
+    @default(10)
+    size: Size,
+
+    @httpQuery("sort")
+    sort: Sort,
+
+    @httpQuery("_source")
+    _source: Source,
+
+    @httpQuery("_source_excludes")
+    _source_excludes: SourceExcludes,
+
+    @httpQuery("_source_includes")
+    _source_includes: SourceIncludes,
+
+    @httpQuery("terminate_after")
+    terminate_after: TerminateAfter,
+
+    @httpQuery("stats")
+    stats: Stats,
+
+    @httpQuery("suggest_field")
+    suggest_field: SuggestField,
+
+    @httpQuery("suggest_mode")
+    @default("missing")
+    suggest_mode: SuggestMode,
+
+    @httpQuery("suggest_size")
+    suggest_size: SuggestSize,
+
+    @httpQuery("suggest_text")
+    suggest_text: SuggestText,
+
+    @httpQuery("timeout")
+    timeout: Timeout,
+
+    @httpQuery("track_scores")
+    track_scores: TrackScores,
+
+    @httpQuery("track_total_hits")
+    track_total_hits: TrackTotalHits,
+
+    @httpQuery("allow_partial_search_results")
+    @default(true)
+    allow_partial_search_results: AllowPartialSearchResults,
+
+    @httpQuery("typed_keys")
+    typed_keys: TypedKeys,
+
+    @httpQuery("version")
+    version: WithVersion,
+
+    @httpQuery("seq_no_primary_term")
+    seq_no_primary_term: SeqNoPrimaryTerm,
+
+    @httpQuery("request_cache")
+    request_cache: RequestCache,
+
+    @httpQuery("batched_reduce_size")
+    @default(512)
+    batched_reduce_size: BatchedReduceSize,
+
+    @httpQuery("max_concurrent_shard_requests")
+    @documentation("The number of concurrent shard requests per node this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests.")
+    @default(5)
+    max_concurrent_shard_requests: MaxConcurrentShardRequests,
+
+    @httpQuery("pre_filter_shard_size")
+    pre_filter_shard_size: PreFilterShardSize,
+
+    @httpQuery("rest_total_hits_as_int")
+    @default(false)
+    rest_total_hits_as_int: RestTotalHitsAsInt,
+}
+
+// TODO: Fill in Body Parameters
+structure KNNSearchModels_BodyParams {}
+
+@input
+structure KNNSearchModels_Get_Input with [KNNSearchModels_QueryParams] {}
+
+@input
+structure KNNSearchModels_Post_Input with [KNNSearchModels_QueryParams] {
+    @httpPayload
+    content: KNNSearchModels_BodyParams,
+}
+
+// TODO: Fill in Output Structure
+structure KNNSearchModels_Output {}