diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index e2511438e7f95..b16621aaaa471 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.TestUtil /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one @@ -29,6 +30,7 @@ tasks.named("javadoc").configure { enabled = false } configurations { expression painless + nativeLib } dependencies { @@ -45,6 +47,7 @@ dependencies { implementation project(path: ':libs:elasticsearch-simdvec') expression(project(path: ':modules:lang-expression', configuration: 'zip')) painless(project(path: ':modules:lang-painless', configuration: 'zip')) + nativeLib(project(':libs:elasticsearch-native')) api "org.openjdk.jmh:jmh-core:$versions.jmh" annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh" // Dependencies of JMH @@ -76,17 +79,8 @@ tasks.register("copyPainless", Copy) { tasks.named("run").configure { executable = "${BuildParams.runtimeJavaHome}/bin/java" args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index" - dependsOn "copyExpression", "copyPainless" - systemProperty 'es.nativelibs.path', file("../libs/native/libraries/build/platform/${platformName()}-${os.arch}") -} - -String platformName() { - String name = System.getProperty("os.name"); - if (name.startsWith("Mac")) { - return "darwin"; - } else { - return name.toLowerCase(Locale.ROOT); - } + dependsOn "copyExpression", "copyPainless", configurations.nativeLib + systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString()) } spotless { diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 32f35b05015b9..2a2a77a6df820 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -22,7 +22,7 @@ <% if (docker_base == 'iron_bank') { %> ARG BASE_REGISTRY=registry1.dso.mil ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9 -ARG BASE_TAG=9.3 +ARG BASE_TAG=9.4 <% } %> ################################################################################ diff --git a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml index 38ce16a413af2..f4364c5008c09 100644 --- a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml +++ b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml @@ -14,7 +14,7 @@ tags: # Build args passed to Dockerfile ARGs args: BASE_IMAGE: "redhat/ubi/ubi9" - BASE_TAG: "9.3" + BASE_TAG: "9.4" # Docker image labels labels: diff --git a/docs/changelog/112066.yaml b/docs/changelog/112066.yaml new file mode 100644 index 0000000000000..5dd846766bc8e --- /dev/null +++ b/docs/changelog/112066.yaml @@ -0,0 +1,6 @@ +pr: 112066 +summary: Do not treat replica as unassigned if primary recently created and unassigned + time is below a threshold +area: Health +type: enhancement +issues: [] diff --git a/docs/changelog/112178.yaml b/docs/changelog/112178.yaml new file mode 100644 index 0000000000000..f1011291542b8 --- /dev/null +++ b/docs/changelog/112178.yaml @@ -0,0 +1,6 @@ +pr: 112178 +summary: Avoid wrapping rejection exception in exchange +area: ES|QL +type: bug +issues: + - 112106 diff --git a/docs/changelog/112242.yaml b/docs/changelog/112242.yaml new file mode 100644 index 0000000000000..7292a00166de2 --- /dev/null +++ b/docs/changelog/112242.yaml @@ -0,0 +1,5 @@ +pr: 112242 +summary: Fix toReleaseVersion() when called on the current version id +area: Infra/Core +type: bug +issues: [111900] diff --git a/docs/changelog/112260.yaml b/docs/changelog/112260.yaml new file mode 100644 index 0000000000000..3f5642188a367 --- /dev/null +++ b/docs/changelog/112260.yaml @@ -0,0 +1,6 @@ +pr: 112260 +summary: Fix DLS over Runtime Fields +area: "Authorization" +type: bug +issues: + - 111637 diff --git a/docs/changelog/112273.yaml b/docs/changelog/112273.yaml new file mode 100644 index 0000000000000..3182a1884a145 --- /dev/null +++ b/docs/changelog/112273.yaml @@ -0,0 +1,5 @@ +pr: 111181 +summary: "[Inference API] Add Docs for AlibabaCloud AI Search Support for the Inference API" +area: Machine Learning +type: enhancement +issues: [ ] diff --git a/docs/changelog/112277.yaml b/docs/changelog/112277.yaml new file mode 100644 index 0000000000000..eac474555999a --- /dev/null +++ b/docs/changelog/112277.yaml @@ -0,0 +1,5 @@ +pr: 112277 +summary: Upgrade `repository-azure` dependencies +area: Snapshot/Restore +type: upgrade +issues: [] diff --git a/docs/changelog/112320.yaml b/docs/changelog/112320.yaml new file mode 100644 index 0000000000000..d35a08dfa4e91 --- /dev/null +++ b/docs/changelog/112320.yaml @@ -0,0 +1,5 @@ +pr: 112320 +summary: Upgrade xcontent to Jackson 2.17.2 +area: Infra/Core +type: upgrade +issues: [] diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index 33db148755d8e..8fdf8aecc2ae5 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -39,6 +39,7 @@ include::delete-inference.asciidoc[] include::get-inference.asciidoc[] include::post-inference.asciidoc[] include::put-inference.asciidoc[] +include::service-alibabacloud-ai-search.asciidoc[] include::service-amazon-bedrock.asciidoc[] include::service-anthropic.asciidoc[] include::service-azure-ai-studio.asciidoc[] diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index 57485e0720cca..ba26a563541fc 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -39,6 +39,7 @@ The create {infer} API enables you to create an {infer} endpoint and configure a The following services are available through the {infer} API, click the links to review the configuration details of the services: +* <> * <> * <> * <> diff --git a/docs/reference/inference/service-alibabacloud-ai-search.asciidoc b/docs/reference/inference/service-alibabacloud-ai-search.asciidoc new file mode 100644 index 0000000000000..df5220573d9e4 --- /dev/null +++ b/docs/reference/inference/service-alibabacloud-ai-search.asciidoc @@ -0,0 +1,184 @@ +[[infer-service-alibabacloud-ai-search]] +=== AlibabaCloud AI Search {infer} service + +Creates an {infer} endpoint to perform an {infer} task with the `alibabacloud-ai-search` service. + +[discrete] +[[infer-service-alibabacloud-ai-search-api-request]] +==== {api-request-title} + +`PUT /_inference//` + +[discrete] +[[infer-service-alibabacloud-ai-search-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=inference-id] + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=task-type] ++ +-- +Available task types: + +* `text_embedding`, +* `sparse_embedding`. +* `rerank`. +-- + +[discrete] +[[infer-service-alibabacloud-ai-search-api-request-body]] +==== {api-request-body-title} + +`service`:: +(Required, string) The type of service supported for the specified task type. +In this case, +`alibabacloud-ai-search`. + +`service_settings`:: +(Required, object) +include::inference-shared.asciidoc[tag=service-settings] ++ +-- +These settings are specific to the `alibabacloud-ai-search` service. +-- + +`api_key`::: +(Required, string) +A valid API key for the AlibabaCloud AI Search API. + +`service_id`::: +(Required, string) +The name of the model service to use for the {infer} task. ++ +-- +Available service_ids for the `text_embedding` task: + +* `ops-text-embedding-001` +* `ops-text-embedding-zh-001` +* `ops-text-embedding-en-001` +* `ops-text-embedding-002` + +For the supported `text_embedding` service_ids, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[documentation]. + +Available service_id for the `sparse_embedding` task: + +* `ops-text-sparse-embedding-001` + +For the supported `sparse_embedding` service_id, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-sparse-embedding-api-details[documentation]. + +Available service_id for the `rerank` task is: + +* `ops-bge-reranker-larger` + +For the supported `rerank` service_id, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/ranker-api-details[documentation]. +-- + +`host`::: +(Required, string) +The name of the host address used for the {infer} task. You can find the host address at https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] of the documentation. + +`workspace`::: +(Required, string) +The name of the workspace used for the {infer} task. + +`rate_limit`::: +(Optional, object) +By default, the `alibabacloud-ai-search` service sets the number of requests allowed per minute to `1000`. +This helps to minimize the number of rate limit errors returned from AlibabaCloud AI Search. +To modify this, set the `requests_per_minute` setting of this object in your service settings: ++ +-- +include::inference-shared.asciidoc[tag=request-per-minute-example] +-- + + +`task_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=task-settings] ++ +.`task_settings` for the `text_embedding` task type +[%collapsible%closed] +===== +`input_type`::: +(Optional, string) +Specifies the type of input passed to the model. +Valid values are: +* `ingest`: for storing document embeddings in a vector database. +* `search`: for storing embeddings of search queries run against a vector database to find relevant documents. +===== ++ +.`task_settings` for the `sparse_embedding` task type +[%collapsible%closed] +===== +`input_type`::: +(Optional, string) +Specifies the type of input passed to the model. +Valid values are: +* `ingest`: for storing document embeddings in a vector database. +* `search`: for storing embeddings of search queries run against a vector database to find relevant documents. + +`return_token`::: +(Optional, boolean) +If `true`, the token name will be returned in the response. Defaults to `false` which means only the token ID will be returned in the response. +===== + +[discrete] +[[inference-example-alibabacloud-ai-search]] +==== AlibabaCloud AI Search service examples + +The following example shows how to create an {infer} endpoint called `alibabacloud_ai_search_embeddings` to perform a `text_embedding` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/alibabacloud_ai_search_embeddings +{ + "service": "alibabacloud-ai-search", + "service_settings": { + "api_key": "", + "service_id": "ops-text-embedding-001", + "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com", + "workspace": "default" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The following example shows how to create an {infer} endpoint called +`alibabacloud_ai_search_sparse` to perform a `sparse_embedding` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/sparse_embedding/alibabacloud_ai_search_sparse +{ + "service": "alibabacloud-ai-search", + "service_settings": { + "api_key": "", + "service_id": "ops-text-sparse-embedding-001", + "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com", + "workspace": "default" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The next example shows how to create an {infer} endpoint called +`alibabacloud_ai_search_rerank` to perform a `rerank` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/rerank/alibabacloud_ai_search_rerank +{ + "service": "alibabacloud-ai-search", + "service_settings": { + "api_key": "", + "service_id": "ops-bge-reranker-larger", + "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com", + "workspace": "default" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] diff --git a/docs/reference/intro.asciidoc b/docs/reference/intro.asciidoc index 3fc23b44994a7..cd9c126e7b1fd 100644 --- a/docs/reference/intro.asciidoc +++ b/docs/reference/intro.asciidoc @@ -1,42 +1,70 @@ [[elasticsearch-intro]] == What is {es}? -_**You know, for search (and analysis)**_ - -{es} is the distributed search and analytics engine at the heart of -the {stack}. {ls} and {beats} facilitate collecting, aggregating, and -enriching your data and storing it in {es}. {kib} enables you to -interactively explore, visualize, and share insights into your data and manage -and monitor the stack. {es} is where the indexing, search, and analysis -magic happens. - -{es} provides near real-time search and analytics for all types of data. Whether you -have structured or unstructured text, numerical data, or geospatial data, -{es} can efficiently store and index it in a way that supports fast searches. -You can go far beyond simple data retrieval and aggregate information to discover -trends and patterns in your data. And as your data and query volume grows, the -distributed nature of {es} enables your deployment to grow seamlessly right -along with it. - -While not _every_ problem is a search problem, {es} offers speed and flexibility -to handle data in a wide variety of use cases: - -* Add a search box to an app or website -* Store and analyze logs, metrics, and security event data -* Use machine learning to automatically model the behavior of your data in real - time -* Use {es} as a vector database to create, store, and search vector embeddings -* Automate business workflows using {es} as a storage engine -* Manage, integrate, and analyze spatial information using {es} as a geographic - information system (GIS) -* Store and process genetic data using {es} as a bioinformatics research tool - -We’re continually amazed by the novel ways people use search. But whether -your use case is similar to one of these, or you're using {es} to tackle a new -problem, the way you work with your data, documents, and indices in {es} is -the same. + +{es-repo}[{es}] is a distributed search and analytics engine, scalable data store, and vector database built on Apache Lucene. +It's optimized for speed and relevance on production-scale workloads. +Use {es} to search, index, store, and analyze data of all shapes and sizes in near real time. + +[TIP] +==== +{es} has a lot of features. Explore the full list on the https://www.elastic.co/elasticsearch/features[product webpage^]. +==== + +{es} is the heart of the {estc-welcome-current}/stack-components.html[Elastic Stack] and powers the Elastic https://www.elastic.co/enterprise-search[Search], https://www.elastic.co/observability[Observability] and https://www.elastic.co/security[Security] solutions. + +{es} is used for a wide and growing range of use cases. Here are a few examples: + +* *Monitor log and event data*. Store logs, metrics, and event data for observability and security information and event management (SIEM). +* *Build search applications*. Add search capabilities to apps or websites, or build enterprise search engines over your organization's internal data sources. +* *Vector database*. Store and search vectorized data, and create vector embeddings with built-in and third-party natural language processing (NLP) models. +* *Retrieval augmented generation (RAG)*. Use {es} as a retrieval engine to augment Generative AI models. +* *Application and security monitoring*. Monitor and analyze application performance and security data effectively. +* *Machine learning*. Use {ml} to automatically model the behavior of your data in real-time. + +This is just a sample of search, observability, and security use cases enabled by {es}. +Refer to our https://www.elastic.co/customers/success-stories[customer success stories] for concrete examples across a range of industries. +// Link to demos, search labs chatbots + +[discrete] +[[elasticsearch-intro-elastic-stack]] +.What is the Elastic Stack? +******************************* +{es} is the core component of the Elastic Stack, a suite of products for collecting, storing, searching, and visualizing data. +https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current/stack-components.html[Learn more about the Elastic Stack]. +******************************* +// TODO: Remove once we've moved Stack Overview to a subpage? + +[discrete] +[[elasticsearch-intro-deploy]] +=== Deployment options + +To use {es}, you need a running instance of the {es} service. +You can deploy {es} in various ways: + +* <>. Get started quickly with a minimal local Docker setup. +* {cloud}/ec-getting-started-trial.html[*Elastic Cloud*]. {es} is available as part of our hosted Elastic Stack offering, deployed in the cloud with your provider of choice. Sign up for a https://cloud.elastic.co/registration[14 day free trial]. +* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless* (technical preview)]. Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14 day free trial]. + +**Advanced deployment options** + +* <>. Install, configure, and run {es} on your own premises. +* {ece-ref}/Elastic-Cloud-Enterprise-overview.html[*Elastic Cloud Enterprise*]. Deploy Elastic Cloud on public or private clouds, virtual machines, or your own premises. +* {eck-ref}/k8s-overview.html[*Elastic Cloud on Kubernetes*]. Deploy Elastic Cloud on Kubernetes. + +[discrete] +[[elasticsearch-next-steps]] +=== Learn more + +Here are some resources to help you get started: + +* <>. A beginner's guide to deploying your first {es} instance, indexing data, and running queries. +* https://elastic.co/webinars/getting-started-elasticsearch[Webinar: Introduction to {es}]. Register for our live webinars to learn directly from {es} experts. +* https://www.elastic.co/search-labs[Elastic Search Labs]. Tutorials and blogs that explore AI-powered search using the latest {es} features. +** Follow our tutorial https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[to build a hybrid search solution in Python]. +** Check out the https://github.com/elastic/elasticsearch-labs?tab=readme-ov-file#elasticsearch-examples--apps[`elasticsearch-labs` repository] for a range of Python notebooks and apps for various use cases. [[documents-indices]] -=== Data in: documents and indices +=== Documents and indices {es} is a distributed document store. Instead of storing information as rows of columnar data, {es} stores complex data structures that have been serialized @@ -65,8 +93,7 @@ behavior makes it easy to index and explore your data--just start indexing documents and {es} will detect and map booleans, floating point and integer values, dates, and strings to the appropriate {es} data types. -Ultimately, however, you know more about your data and how you want to use it -than {es} can. You can define rules to control dynamic mapping and explicitly +You can define rules to control dynamic mapping and explicitly define mappings to take full control of how fields are stored and indexed. Defining your own mappings enables you to: @@ -89,7 +116,7 @@ used at search time. When you query a full-text field, the query text undergoes the same analysis before the terms are looked up in the index. [[search-analyze]] -=== Information out: search and analyze +=== Search and analyze While you can use {es} as a document store and retrieve documents and their metadata, the real power comes from being able to easily access the full suite @@ -160,27 +187,8 @@ size 70 needles, you’re displaying a count of the size 70 needles that match your users' search criteria--for example, all size 70 _non-stick embroidery_ needles. -[discrete] -[[more-features]] -===== But wait, there’s more - -Want to automate the analysis of your time series data? You can use -{ml-docs}/ml-ad-overview.html[machine learning] features to create accurate -baselines of normal behavior in your data and identify anomalous patterns. With -machine learning, you can detect: - -* Anomalies related to temporal deviations in values, counts, or frequencies -* Statistical rarity -* Unusual behaviors for a member of a population - -And the best part? You can do this without having to specify algorithms, models, -or other data science-related configurations. - [[scalability]] -=== Scalability and resilience: clusters, nodes, and shards -++++ -Scalability and resilience -++++ +=== Scalability and resilience {es} is built to be always available and to scale with your needs. It does this by being distributed by nature. You can add servers (nodes) to a cluster to @@ -209,7 +217,7 @@ interrupting indexing or query operations. [discrete] [[it-depends]] -==== It depends... +==== Shard size and number of shards There are a number of performance considerations and trade offs with respect to shard size and the number of primary shards configured for an index. The more @@ -237,7 +245,7 @@ testing with your own data and queries]. [discrete] [[disaster-ccr]] -==== In case of disaster +==== Disaster recovery A cluster's nodes need good, reliable connections to each other. To provide better connections, you typically co-locate the nodes in the same data center or @@ -257,7 +265,7 @@ secondary clusters are read-only followers. [discrete] [[admin]] -==== Care and feeding +==== Security, management, and monitoring As with any enterprise system, you need tools to secure, manage, and monitor your {es} clusters. Security, monitoring, and administrative features @@ -265,3 +273,5 @@ that are integrated into {es} enable you to use {kibana-ref}/introduction.html[{ as a control center for managing a cluster. Features like <> and <> help you intelligently manage your data over time. + +Refer to <> for more information. \ No newline at end of file diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index 522a0c54c8aad..a006f288dc66d 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -7,8 +7,8 @@ beta[] -The `semantic_text` field type automatically generates embeddings for text -content using an inference endpoint. +The `semantic_text` field type automatically generates embeddings for text content using an inference endpoint. +Long passages are <> to smaller sections to enable the processing of larger corpuses of text. The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings. You can create the inference endpoint by using the <>. diff --git a/docs/reference/modules/discovery/fault-detection.asciidoc b/docs/reference/modules/discovery/fault-detection.asciidoc index 89c8a78eccbc6..d12985b70597c 100644 --- a/docs/reference/modules/discovery/fault-detection.asciidoc +++ b/docs/reference/modules/discovery/fault-detection.asciidoc @@ -300,7 +300,6 @@ To reconstruct the output, base64-decode the data and decompress it using ---- cat shardlock.log | sed -e 's/.*://' | base64 --decode | gzip --decompress ---- -//end::troubleshooting[] [discrete] ===== Diagnosing other network disconnections @@ -345,3 +344,4 @@ packet capture simultaneously from the nodes at both ends of an unstable connection and analyse it alongside the {es} logs from those nodes to determine if traffic between the nodes is being disrupted by another device on the network. +//end::troubleshooting[] diff --git a/docs/reference/search/search-your-data/near-real-time.asciidoc b/docs/reference/search/search-your-data/near-real-time.asciidoc index 46a996c237c38..47618ecd9fd7a 100644 --- a/docs/reference/search/search-your-data/near-real-time.asciidoc +++ b/docs/reference/search/search-your-data/near-real-time.asciidoc @@ -2,7 +2,7 @@ [[near-real-time]] === Near real-time search -The overview of <> indicates that when a document is stored in {es}, it is indexed and fully searchable in _near real-time_--within 1 second. What defines near real-time search? +When a document is stored in {es}, it is indexed and fully searchable in _near real-time_--within 1 second. What defines near real-time search? Lucene, the Java libraries on which {es} is based, introduced the concept of per-segment search. A _segment_ is similar to an inverted index, but the word _index_ in Lucene means "a collection of segments plus a commit point". After a commit, a new segment is added to the commit point and the buffer is cleared. diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc index f74bc65e31bf0..719aeb070fc7c 100644 --- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc @@ -17,6 +17,7 @@ Azure based examples use models available through https://ai.azure.com/explore/m or https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models[Azure OpenAI]. Mistral examples use the `mistral-embed` model from https://docs.mistral.ai/getting-started/models/[the Mistral API]. Amazon Bedrock examples use the `amazon.titan-embed-text-v1` model from https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[the Amazon Bedrock base models]. +AlibabaCloud AI Search examples use the `ops-text-embedding-zh-001` model from https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[the AlibabaCloud AI Search base models]. Click the name of the service you want to use on any of the widgets below to review the corresponding instructions. diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc index 997dbbe8a20e6..3a686e27cf580 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc @@ -49,6 +49,12 @@ id="infer-api-ingest-amazon-bedrock"> Amazon Bedrock +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc index 6adf3d2ebbf46..6678b60fabc40 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc @@ -216,3 +216,29 @@ PUT _ingest/pipeline/amazon_bedrock_embeddings and the `output_field` that will contain the {infer} results. // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] + +[source,console] +-------------------------------------------------- +PUT _ingest/pipeline/alibabacloud_ai_search_embeddings +{ + "processors": [ + { + "inference": { + "model_id": "alibabacloud_ai_search_embeddings", <1> + "input_output": { <2> + "input_field": "content", + "output_field": "content_embedding" + } + } + } + ] +} +-------------------------------------------------- +<1> The name of the inference endpoint you created by using the +<>, it's referred to as `inference_id` in that step. +<2> Configuration object that defines the `input_field` for the {infer} process +and the `output_field` that will contain the {infer} results. + +// end::alibabacloud-ai-search[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc index 4e3a453a7bbea..66b790bdd57a5 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc @@ -49,6 +49,12 @@ id="infer-api-mapping-amazon-bedrock"> Amazon Bedrock +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc index abeeb87f03e75..c86538ceb9c87 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc @@ -270,3 +270,35 @@ the {infer} pipeline configuration in the next step. <6> The field type which is text in this example. // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] + +[source,console] +-------------------------------------------------- +PUT alibabacloud-ai-search-embeddings +{ + "mappings": { + "properties": { + "content_embedding": { <1> + "type": "dense_vector", <2> + "dims": 1024, <3> + "element_type": "float" + }, + "content": { <4> + "type": "text" <5> + } + } + } +} +-------------------------------------------------- +<1> The name of the field to contain the generated tokens. It must be referenced +in the {infer} pipeline configuration in the next step. +<2> The field to contain the tokens is a `dense_vector` field. +<3> The output dimensions of the model. This value may be different depending on the underlying model used. +See the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[AlibabaCloud AI Search embedding model] documentation. +<4> The name of the field from which to create the dense vector representation. +In this example, the name of the field is `content`. It must be referenced in +the {infer} pipeline configuration in the next step. +<5> The field type which is text in this example. + +// end::alibabacloud-ai-search[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc index 45cb9fc51b9f1..86f52fee2063c 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc @@ -49,6 +49,12 @@ id="infer-api-reindex-amazon-bedrock"> Amazon Bedrock +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc index d961ec8bd39bd..25d4023c650c0 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc @@ -200,3 +200,26 @@ number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] + +[source,console] +---- +POST _reindex?wait_for_completion=false +{ + "source": { + "index": "test-data", + "size": 50 <1> + }, + "dest": { + "index": "alibabacloud-ai-search-embeddings", + "pipeline": "alibabacloud_ai_search_embeddings" + } +} +---- +// TEST[skip:TBD] +<1> The default batch size for reindexing is 1000. Reducing `size` to a smaller +number makes the update of the reindexing process quicker which enables you to +follow the progress closely and detect errors early. + +// end::alibabacloud-ai-search[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc index c867b39b88e3b..fb686a2d8be12 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc @@ -49,6 +49,12 @@ id="infer-api-requirements-amazon-bedrock"> Amazon Bedrock +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc index 603cd85a8f93d..c9e7ca8b80ba6 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc @@ -52,3 +52,9 @@ You can apply for access to Azure OpenAI by completing the form at https://aka.m * A pair of access and secret keys used to access Amazon Bedrock // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] +* An AlibabaCloud Account with https://console.aliyun.com[AlibabaCloud] access +* An API key generated for your account from the https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[API keys section] + +// end::alibabacloud-ai-search[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc index fa4a11c59a158..996148d80a4bd 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc @@ -49,6 +49,12 @@ id="infer-api-search-amazon-bedrock"> Amazon Bedrock +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc index f23ed1dfef05d..fe1f58b6bd1a9 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc @@ -531,3 +531,68 @@ query from the `amazon-bedrock-embeddings` index sorted by their proximity to th // NOTCONSOLE // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] + +[source,console] +-------------------------------------------------- +GET alibabacloud-ai-search-embeddings/_search +{ + "knn": { + "field": "content_embedding", + "query_vector_builder": { + "text_embedding": { + "model_id": "alibabacloud_ai_search_embeddings", + "model_text": "Calculate fuel cost" + } + }, + "k": 10, + "num_candidates": 100 + }, + "_source": [ + "id", + "content" + ] +} +-------------------------------------------------- +// TEST[skip:TBD] + +As a result, you receive the top 10 documents that are closest in meaning to the +query from the `alibabacloud-ai-search-embeddings` index sorted by their proximity to the query: + +[source,consol-result] +-------------------------------------------------- +"hits": [ + { + "_index": "alibabacloud-ai-search-embeddings", + "_id": "DDd5OowBHxQKHyc3TDSC", + "_score": 0.83704096, + "_source": { + "id": 862114, + "body": "How to calculate fuel cost for a road trip. By Tara Baukus Mello • Bankrate.com. Dear Driving for Dollars, My family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost.It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes.y family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost. It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes." + } + }, + { + "_index": "alibabacloud-ai-search-embeddings", + "_id": "ajd5OowBHxQKHyc3TDSC", + "_score": 0.8345704, + "_source": { + "id": 820622, + "body": "Home Heating Calculator. Typically, approximately 50% of the energy consumed in a home annually is for space heating. When deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important.This calculator can help you estimate the cost of fuel for different heating appliances.hen deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important. This calculator can help you estimate the cost of fuel for different heating appliances." + } + }, + { + "_index": "alibabacloud-ai-search-embeddings", + "_id": "Djd5OowBHxQKHyc3TDSC", + "_score": 0.8327426, + "_source": { + "id": 8202683, + "body": "Fuel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel.If you are paying $4 per gallon, the trip would cost you $200.Most boats have much larger gas tanks than cars.uel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel." + } + }, + (...) + ] +-------------------------------------------------- +// NOTCONSOLE + +// end::alibabacloud-ai-search[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc index f12be341d866d..1dfa6077553fe 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc @@ -49,6 +49,12 @@ id="infer-api-task-amazon-bedrock"> Amazon Bedrock +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc index b186b2c58ccc5..2b4aa1a200102 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc @@ -223,3 +223,32 @@ PUT _inference/text_embedding/amazon_bedrock_embeddings <1> <6> The model ID or ARN of the model to use. // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/alibabacloud_ai_search_embeddings <1> +{ + "service": "alibabacloud-ai-search", + "service_settings": { + "api_key": "", <2> + "service_id": "", <3> + "host": "", <4> + "workspace": "" <5> + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {infer} endpoint is `alibabacloud_ai_search_embeddings`. +<2> The API key for accessing the AlibabaCloud AI Search API. You can find your API keys in +your AlibabaCloud account under the +https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[API keys section]. You need to provide +your API key only once. The <> does not return your API +key. +<3> The AlibabaCloud AI Search embeddings model name, for example `ops-text-embedding-zh-001`. +<4> The name our your AlibabaCloud AI Search host address. +<5> The name our your AlibabaCloud AI Search workspace. + +// end::alibabacloud-ai-search[] + diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 261e210cdbe11..a27e2083a0849 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -119,44 +119,44 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -311,6 +311,11 @@ + + + + + @@ -346,6 +351,11 @@ + + + + + @@ -361,6 +371,11 @@ + + + + + @@ -381,6 +396,11 @@ + + + + + @@ -411,9 +431,9 @@ - - - + + + @@ -901,9 +921,9 @@ - - - + + + @@ -3394,6 +3414,11 @@ + + + + + diff --git a/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java b/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java new file mode 100644 index 0000000000000..0fe816bd3721d --- /dev/null +++ b/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.core; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to identify a block of code (a whole class, a method, or a field) that needs to be reviewed (for cleanup, remove or change) + * before releasing 10.0 + */ +@Retention(RetentionPolicy.SOURCE) +@Target({ ElementType.LOCAL_VARIABLE, ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE }) +public @interface UpdateForV10 { +} diff --git a/libs/x-content/impl/build.gradle b/libs/x-content/impl/build.gradle index 829b75524baeb..6cf278e826d4c 100644 --- a/libs/x-content/impl/build.gradle +++ b/libs/x-content/impl/build.gradle @@ -12,7 +12,7 @@ base { archivesName = "x-content-impl" } -String jacksonVersion = "2.17.0" +String jacksonVersion = "2.17.2" dependencies { compileOnly project(':libs:elasticsearch-core') diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java new file mode 100644 index 0000000000000..e7d26b0808a48 --- /dev/null +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.reindex; + +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.reindex.BulkIndexByScrollResponseMatcher; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.reindex.DeleteByQueryMetrics.DELETE_BY_QUERY_TIME_HISTOGRAM; +import static org.elasticsearch.reindex.ReindexMetrics.REINDEX_TIME_HISTOGRAM; +import static org.elasticsearch.reindex.UpdateByQueryMetrics.UPDATE_BY_QUERY_TIME_HISTOGRAM; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.equalTo; + +@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = ESIntegTestCase.Scope.TEST) +public class ReindexPluginMetricsIT extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return Arrays.asList(ReindexPlugin.class, TestTelemetryPlugin.class); + } + + protected ReindexRequestBuilder reindex() { + return new ReindexRequestBuilder(client()); + } + + protected UpdateByQueryRequestBuilder updateByQuery() { + return new UpdateByQueryRequestBuilder(client()); + } + + protected DeleteByQueryRequestBuilder deleteByQuery() { + return new DeleteByQueryRequestBuilder(client()); + } + + public static BulkIndexByScrollResponseMatcher matcher() { + return new BulkIndexByScrollResponseMatcher(); + } + + public void testReindexMetrics() throws Exception { + final String dataNodeName = internalCluster().startNode(); + + indexRandom( + true, + prepareIndex("source").setId("1").setSource("foo", "a"), + prepareIndex("source").setId("2").setSource("foo", "a"), + prepareIndex("source").setId("3").setSource("foo", "b"), + prepareIndex("source").setId("4").setSource("foo", "c") + ); + assertHitCount(prepareSearch("source").setSize(0), 4); + + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + + // Copy all the docs + reindex().source("source").destination("dest").get(); + // Use assertBusy to wait for all threads to complete so we get deterministic results + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(1)); + }); + + // Now none of them + createIndex("none"); + reindex().source("source").destination("none").filter(termQuery("foo", "no_match")).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(2)); + }); + + // Now half of them + reindex().source("source").destination("dest_half").filter(termQuery("foo", "a")).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(3)); + }); + + // Limit with maxDocs + reindex().source("source").destination("dest_size_one").maxDocs(1).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(4)); + }); + } + + public void testDeleteByQueryMetrics() throws Exception { + final String dataNodeName = internalCluster().startNode(); + + indexRandom( + true, + prepareIndex("test").setId("1").setSource("foo", "a"), + prepareIndex("test").setId("2").setSource("foo", "a"), + prepareIndex("test").setId("3").setSource("foo", "b"), + prepareIndex("test").setId("4").setSource("foo", "c"), + prepareIndex("test").setId("5").setSource("foo", "d"), + prepareIndex("test").setId("6").setSource("foo", "e"), + prepareIndex("test").setId("7").setSource("foo", "f") + ); + + assertHitCount(prepareSearch("test").setSize(0), 7); + + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + + // Deletes two docs that matches "foo:a" + deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(1)); + }); + + // Deletes the two first docs with limit by size + DeleteByQueryRequestBuilder request = deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).size(2).refresh(true); + request.source().addSort("foo.keyword", SortOrder.ASC); + request.get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(2)); + }); + + // Deletes but match no docs + deleteByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(3)); + }); + + // Deletes all remaining docs + deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(4)); + }); + } + + public void testUpdateByQueryMetrics() throws Exception { + final String dataNodeName = internalCluster().startNode(); + + indexRandom( + true, + prepareIndex("test").setId("1").setSource("foo", "a"), + prepareIndex("test").setId("2").setSource("foo", "a"), + prepareIndex("test").setId("3").setSource("foo", "b"), + prepareIndex("test").setId("4").setSource("foo", "c") + ); + assertHitCount(prepareSearch("test").setSize(0), 4); + assertEquals(1, client().prepareGet("test", "1").get().getVersion()); + assertEquals(1, client().prepareGet("test", "4").get().getVersion()); + + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + + // Reindex all the docs + updateByQuery().source("test").refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(1)); + }); + + // Now none of them + updateByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(2)); + }); + + // Now half of them + updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(3)); + }); + + // Limit with size + UpdateByQueryRequestBuilder request = updateByQuery().source("test").size(3).refresh(true); + request.source().addSort("foo.keyword", SortOrder.ASC); + request.get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(4)); + }); + } +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java new file mode 100644 index 0000000000000..2cedf0d5f5823 --- /dev/null +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.metric.LongHistogram; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +public class DeleteByQueryMetrics { + public static final String DELETE_BY_QUERY_TIME_HISTOGRAM = "es.delete_by_query.duration.histogram"; + + private final LongHistogram deleteByQueryTimeSecsHistogram; + + public DeleteByQueryMetrics(MeterRegistry meterRegistry) { + this( + meterRegistry.registerLongHistogram(DELETE_BY_QUERY_TIME_HISTOGRAM, "Time taken to execute Delete by Query request", "seconds") + ); + } + + private DeleteByQueryMetrics(LongHistogram deleteByQueryTimeSecsHistogram) { + this.deleteByQueryTimeSecsHistogram = deleteByQueryTimeSecsHistogram; + } + + public long recordTookTime(long tookTime) { + deleteByQueryTimeSecsHistogram.record(tookTime); + return tookTime; + } +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java new file mode 100644 index 0000000000000..3025357aa6538 --- /dev/null +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.metric.LongHistogram; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +public class ReindexMetrics { + + public static final String REINDEX_TIME_HISTOGRAM = "es.reindex.duration.histogram"; + + private final LongHistogram reindexTimeSecsHistogram; + + public ReindexMetrics(MeterRegistry meterRegistry) { + this(meterRegistry.registerLongHistogram(REINDEX_TIME_HISTOGRAM, "Time to reindex by search", "millis")); + } + + private ReindexMetrics(LongHistogram reindexTimeSecsHistogram) { + this.reindexTimeSecsHistogram = reindexTimeSecsHistogram; + } + + public long recordTookTime(long tookTime) { + reindexTimeSecsHistogram.record(tookTime); + return tookTime; + } +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java index 1a40f77250e5f..3169d4c4ee1fb 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.function.Predicate; import java.util.function.Supplier; @@ -85,8 +84,11 @@ public List getRestHandlers( @Override public Collection createComponents(PluginServices services) { - return Collections.singletonList( - new ReindexSslConfig(services.environment().settings(), services.environment(), services.resourceWatcherService()) + return List.of( + new ReindexSslConfig(services.environment().settings(), services.environment(), services.resourceWatcherService()), + new ReindexMetrics(services.telemetryProvider().getMeterRegistry()), + new UpdateByQueryMetrics(services.telemetryProvider().getMeterRegistry()), + new DeleteByQueryMetrics(services.telemetryProvider().getMeterRegistry()) ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index dbe1968bb076a..cb393a42f52a1 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; @@ -65,6 +66,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import java.util.function.LongSupplier; @@ -82,19 +84,22 @@ public class Reindexer { private final ThreadPool threadPool; private final ScriptService scriptService; private final ReindexSslConfig reindexSslConfig; + private final ReindexMetrics reindexMetrics; Reindexer( ClusterService clusterService, Client client, ThreadPool threadPool, ScriptService scriptService, - ReindexSslConfig reindexSslConfig + ReindexSslConfig reindexSslConfig, + @Nullable ReindexMetrics reindexMetrics ) { this.clusterService = clusterService; this.client = client; this.threadPool = threadPool; this.scriptService = scriptService; this.reindexSslConfig = reindexSslConfig; + this.reindexMetrics = reindexMetrics; } public void initTask(BulkByScrollTask task, ReindexRequest request, ActionListener listener) { @@ -102,6 +107,8 @@ public void initTask(BulkByScrollTask task, ReindexRequest request, ActionListen } public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkClient, ActionListener listener) { + long startTime = System.nanoTime(); + BulkByScrollParallelizationHelper.executeSlicedAction( task, request, @@ -122,7 +129,12 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl clusterService.state(), reindexSslConfig, request, - listener + ActionListener.runAfter(listener, () -> { + long elapsedTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime); + if (reindexMetrics != null) { + reindexMetrics.recordTookTime(elapsedTime); + } + }) ); searchAction.start(); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java index 755587feb47d3..53381c33d7f78 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.DeleteByQueryAction; @@ -25,12 +26,15 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.concurrent.TimeUnit; + public class TransportDeleteByQueryAction extends HandledTransportAction { private final ThreadPool threadPool; private final Client client; private final ScriptService scriptService; private final ClusterService clusterService; + private final DeleteByQueryMetrics deleteByQueryMetrics; @Inject public TransportDeleteByQueryAction( @@ -39,18 +43,21 @@ public TransportDeleteByQueryAction( Client client, TransportService transportService, ScriptService scriptService, - ClusterService clusterService + ClusterService clusterService, + @Nullable DeleteByQueryMetrics deleteByQueryMetrics ) { super(DeleteByQueryAction.NAME, transportService, actionFilters, DeleteByQueryRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.threadPool = threadPool; this.client = client; this.scriptService = scriptService; this.clusterService = clusterService; + this.deleteByQueryMetrics = deleteByQueryMetrics; } @Override public void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; + long startTime = System.nanoTime(); BulkByScrollParallelizationHelper.startSlicedAction( request, bulkByScrollTask, @@ -64,8 +71,20 @@ public void doExecute(Task task, DeleteByQueryRequest request, ActionListener { + long elapsedTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime); + if (deleteByQueryMetrics != null) { + deleteByQueryMetrics.recordTookTime(elapsedTime); + } + }) + ).start(); } ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java index a86af2ca2b83e..821a137ac7566 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.ReindexAction; @@ -53,7 +54,8 @@ public TransportReindexAction( AutoCreateIndex autoCreateIndex, Client client, TransportService transportService, - ReindexSslConfig sslConfig + ReindexSslConfig sslConfig, + @Nullable ReindexMetrics reindexMetrics ) { this( ReindexAction.NAME, @@ -66,7 +68,8 @@ public TransportReindexAction( autoCreateIndex, client, transportService, - sslConfig + sslConfig, + reindexMetrics ); } @@ -81,12 +84,13 @@ protected TransportReindexAction( AutoCreateIndex autoCreateIndex, Client client, TransportService transportService, - ReindexSslConfig sslConfig + ReindexSslConfig sslConfig, + @Nullable ReindexMetrics reindexMetrics ) { super(name, transportService, actionFilters, ReindexRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.client = client; this.reindexValidator = new ReindexValidator(settings, clusterService, indexNameExpressionResolver, autoCreateIndex); - this.reindexer = new Reindexer(clusterService, client, threadPool, scriptService, sslConfig); + this.reindexer = new Reindexer(clusterService, client, threadPool, scriptService, sslConfig, reindexMetrics); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java index fc0bfa3c8a214..997d4d32fe042 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.ScrollableHitSource; @@ -35,6 +36,7 @@ import org.elasticsearch.transport.TransportService; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import java.util.function.LongSupplier; @@ -44,6 +46,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction listener) { BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; + long startTime = System.nanoTime(); BulkByScrollParallelizationHelper.startSlicedAction( request, bulkByScrollTask, @@ -78,8 +84,21 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener clusterService.localNode(), bulkByScrollTask ); - new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, scriptService, request, state, listener) - .start(); + new AsyncIndexBySearchAction( + bulkByScrollTask, + logger, + assigningClient, + threadPool, + scriptService, + request, + state, + ActionListener.runAfter(listener, () -> { + long elapsedTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime); + if (updateByQueryMetrics != null) { + updateByQueryMetrics.recordTookTime(elapsedTime); + } + }) + ).start(); } ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java new file mode 100644 index 0000000000000..6ca52769a1ba9 --- /dev/null +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.metric.LongHistogram; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +public class UpdateByQueryMetrics { + public static final String UPDATE_BY_QUERY_TIME_HISTOGRAM = "es.update_by_query.duration.histogram"; + + private final LongHistogram updateByQueryTimeSecsHistogram; + + public UpdateByQueryMetrics(MeterRegistry meterRegistry) { + this( + meterRegistry.registerLongHistogram(UPDATE_BY_QUERY_TIME_HISTOGRAM, "Time taken to execute Update by Query request", "seconds") + ); + } + + private UpdateByQueryMetrics(LongHistogram updateByQueryTimeSecsHistogram) { + this.updateByQueryTimeSecsHistogram = updateByQueryTimeSecsHistogram; + } + + public long recordTookTime(long tookTime) { + updateByQueryTimeSecsHistogram.record(tookTime); + return tookTime; + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java new file mode 100644 index 0000000000000..58adc6aebaa9b --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.InstrumentType; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.RecordingMeterRegistry; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; + +import static org.elasticsearch.reindex.DeleteByQueryMetrics.DELETE_BY_QUERY_TIME_HISTOGRAM; + +public class DeleteByQueryMetricsTests extends ESTestCase { + private RecordingMeterRegistry recordingMeterRegistry; + private DeleteByQueryMetrics metrics; + + @Before + public void createMetrics() { + recordingMeterRegistry = new RecordingMeterRegistry(); + metrics = new DeleteByQueryMetrics(recordingMeterRegistry); + } + + public void testRecordTookTime() { + int secondsTaken = randomIntBetween(1, 50); + metrics.recordTookTime(secondsTaken); + List measurements = recordingMeterRegistry.getRecorder() + .getMeasurements(InstrumentType.LONG_HISTOGRAM, DELETE_BY_QUERY_TIME_HISTOGRAM); + assertEquals(measurements.size(), 1); + assertEquals(measurements.get(0).getLong(), secondsTaken); + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java new file mode 100644 index 0000000000000..4711530585817 --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.InstrumentType; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.RecordingMeterRegistry; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; + +import static org.elasticsearch.reindex.ReindexMetrics.REINDEX_TIME_HISTOGRAM; + +public class ReindexMetricsTests extends ESTestCase { + + private RecordingMeterRegistry recordingMeterRegistry; + private ReindexMetrics metrics; + + @Before + public void createMetrics() { + recordingMeterRegistry = new RecordingMeterRegistry(); + metrics = new ReindexMetrics(recordingMeterRegistry); + } + + public void testRecordTookTime() { + int secondsTaken = randomIntBetween(1, 50); + metrics.recordTookTime(secondsTaken); + List measurements = recordingMeterRegistry.getRecorder() + .getMeasurements(InstrumentType.LONG_HISTOGRAM, REINDEX_TIME_HISTOGRAM); + assertEquals(measurements.size(), 1); + assertEquals(measurements.get(0).getLong(), secondsTaken); + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java new file mode 100644 index 0000000000000..548d18d202984 --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.InstrumentType; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.RecordingMeterRegistry; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; + +import static org.elasticsearch.reindex.UpdateByQueryMetrics.UPDATE_BY_QUERY_TIME_HISTOGRAM; + +public class UpdateByQueryMetricsTests extends ESTestCase { + + private RecordingMeterRegistry recordingMeterRegistry; + private UpdateByQueryMetrics metrics; + + @Before + public void createMetrics() { + recordingMeterRegistry = new RecordingMeterRegistry(); + metrics = new UpdateByQueryMetrics(recordingMeterRegistry); + } + + public void testRecordTookTime() { + int secondsTaken = randomIntBetween(1, 50); + metrics.recordTookTime(secondsTaken); + List measurements = recordingMeterRegistry.getRecorder() + .getMeasurements(InstrumentType.LONG_HISTOGRAM, UPDATE_BY_QUERY_TIME_HISTOGRAM); + assertEquals(measurements.size(), 1); + assertEquals(measurements.get(0).getLong(), secondsTaken); + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java index 876ddefda161b..c4d591f804750 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java @@ -60,6 +60,7 @@ protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptSer null, transportService, scriptService, + null, null ); return new TransportUpdateByQueryAction.AsyncIndexBySearchAction( diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle index 9c63304e8267b..6334e5ae6a195 100644 --- a/modules/repository-azure/build.gradle +++ b/modules/repository-azure/build.gradle @@ -24,16 +24,16 @@ versions << [ dependencies { // Microsoft - api "com.azure:azure-core-http-netty:1.15.1" - api "com.azure:azure-core:1.50.0" - api "com.azure:azure-identity:1.13.1" - api "com.azure:azure-json:1.1.0" - api "com.azure:azure-storage-blob:12.26.1" - api "com.azure:azure-storage-common:12.26.0" - api "com.azure:azure-storage-internal-avro:12.11.1" - api "com.azure:azure-xml:1.0.0" + api "com.azure:azure-core-http-netty:1.15.3" + api "com.azure:azure-core:1.51.0" + api "com.azure:azure-identity:1.13.2" + api "com.azure:azure-json:1.2.0" + api "com.azure:azure-storage-blob:12.27.1" + api "com.azure:azure-storage-common:12.26.1" + api "com.azure:azure-storage-internal-avro:12.12.1" + api "com.azure:azure-xml:1.1.0" api "com.microsoft.azure:msal4j-persistence-extension:1.3.0" - api "com.microsoft.azure:msal4j:1.16.1" + api "com.microsoft.azure:msal4j:1.16.2" // Jackson api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" @@ -57,7 +57,7 @@ dependencies { api "org.reactivestreams:reactive-streams:1.0.4" // Others - api "com.fasterxml.woodstox:woodstox-core:6.4.0" + api "com.fasterxml.woodstox:woodstox-core:6.7.0" api "com.github.stephenc.jcip:jcip-annotations:1.0-1" api "com.nimbusds:content-type:2.3" api "com.nimbusds:lang-tag:1.7" @@ -69,7 +69,7 @@ dependencies { api "net.java.dev.jna:jna:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version api "net.minidev:accessors-smart:2.5.0" api "net.minidev:json-smart:2.5.0" - api "org.codehaus.woodstox:stax2-api:4.2.1" + api "org.codehaus.woodstox:stax2-api:4.2.2" api "org.ow2.asm:asm:9.3" runtimeOnly "com.google.crypto.tink:tink:1.14.0" diff --git a/muted-tests.yml b/muted-tests.yml index 5199221c25aaf..508403ee6238c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -137,9 +137,6 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testDeleteJobAfterMissingIndex issue: https://github.com/elastic/elasticsearch/issues/112088 -- class: org.elasticsearch.xpack.esql.EsqlAsyncSecurityIT - method: testLimitedPrivilege - issue: https://github.com/elastic/elasticsearch/issues/112110 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT method: test {stats.ByTwoCalculatedSecondOverwrites SYNC} issue: https://github.com/elastic/elasticsearch/issues/112117 @@ -160,9 +157,23 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testDeleteJobAsync issue: https://github.com/elastic/elasticsearch/issues/112212 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=indices.create/20_synthetic_source/stored field under object with store_array_source} - issue: https://github.com/elastic/elasticsearch/issues/112264 +- class: org.elasticsearch.search.query.ScriptScoreQueryTests + method: testScriptTermStatsAvailable + issue: https://github.com/elastic/elasticsearch/issues/112278 +- class: org.elasticsearch.search.query.ScriptScoreQueryTests + method: testScriptTermStatsNotAvailable + issue: https://github.com/elastic/elasticsearch/issues/112290 +- class: org.elasticsearch.search.retriever.rankdoc.RankDocsSortBuilderTests + method: testEqualsAndHashcode + issue: https://github.com/elastic/elasticsearch/issues/112312 +- class: org.elasticsearch.blobcache.shared.SharedBlobCacheServiceTests + method: testGetMultiThreaded + issue: https://github.com/elastic/elasticsearch/issues/112314 +- class: org.elasticsearch.search.retriever.RankDocRetrieverBuilderIT + method: testRankDocsRetrieverWithCollapse + issue: https://github.com/elastic/elasticsearch/issues/112254 +- class: org.elasticsearch.search.ccs.CCSUsageTelemetryIT + issue: https://github.com/elastic/elasticsearch/issues/112324 # Examples: # diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc index 5716afdd205c0..0ddac662e73ef 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc @@ -138,7 +138,7 @@ other test runners to skip tests if they do not support the capabilities API yet path: /_api parameters: [param1, param2] capabilities: [cap1, cap2] - test_runner_feature: [capabilities] + test_runner_features: [capabilities] reason: Capability required to run test - do: ... test definitions ... diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index a696f3b2b3224..fa08efe402b43 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -1342,8 +1342,8 @@ subobjects auto: # 112156 stored field under object with store_array_source: - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source + cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix"] + reason: requires bug fix to be implemented - do: indices.create: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java index 428e116ecd1ca..88d934973fc49 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.junit.annotations.TestIssueLogging; import java.util.ArrayList; import java.util.Arrays; @@ -41,10 +40,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; -@TestIssueLogging( - issueUrl = "https://github.com/elastic/elasticsearch/issues/109830", - value = "org.elasticsearch.action.search:TRACE," + "org.elasticsearch.search.SearchService:TRACE" -) public class SearchProgressActionListenerIT extends ESSingleNodeTestCase { private List shards; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java index 422696d6b61c6..4665dc486a904 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java @@ -8,7 +8,6 @@ package org.elasticsearch.repositories.blobstore; -import org.apache.lucene.tests.mockfile.ExtrasFS; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -23,18 +22,10 @@ import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; import org.elasticsearch.snapshots.SnapshotState; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.junit.Before; -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; -import java.util.Base64; import java.util.List; public class BlobStoreCorruptionIT extends AbstractSnapshotIntegTestCase { @@ -57,7 +48,7 @@ public void testCorruptionDetection() throws Exception { flushAndRefresh(indexName); createSnapshot(repositoryName, snapshotName, List.of(indexName)); - final var corruptedFile = corruptRandomFile(repositoryRootPath); + final var corruptedFile = BlobStoreCorruptionUtils.corruptRandomFile(repositoryRootPath); final var corruptedFileType = RepositoryFileType.getRepositoryFileType(repositoryRootPath, corruptedFile); final var corruptionDetectors = new ArrayList, ?>>(); @@ -126,61 +117,4 @@ public void testCorruptionDetection() throws Exception { logger.info(Strings.format("--> corrupted [%s] and caught exception", corruptedFile), exception); } } - - private static Path corruptRandomFile(Path repositoryRootPath) throws IOException { - final var corruptedFileType = getRandomCorruptibleFileType(); - final var corruptedFile = getRandomFileToCorrupt(repositoryRootPath, corruptedFileType); - if (randomBoolean()) { - logger.info("--> deleting [{}]", corruptedFile); - Files.delete(corruptedFile); - } else { - corruptFileContents(corruptedFile); - } - return corruptedFile; - } - - private static void corruptFileContents(Path fileToCorrupt) throws IOException { - final var oldFileContents = Files.readAllBytes(fileToCorrupt); - logger.info("--> contents of [{}] before corruption: [{}]", fileToCorrupt, Base64.getEncoder().encodeToString(oldFileContents)); - final byte[] newFileContents = new byte[randomBoolean() ? oldFileContents.length : between(0, oldFileContents.length)]; - System.arraycopy(oldFileContents, 0, newFileContents, 0, newFileContents.length); - if (newFileContents.length == oldFileContents.length) { - final var corruptionPosition = between(0, newFileContents.length - 1); - newFileContents[corruptionPosition] = randomValueOtherThan(oldFileContents[corruptionPosition], ESTestCase::randomByte); - logger.info( - "--> updating byte at position [{}] from [{}] to [{}]", - corruptionPosition, - oldFileContents[corruptionPosition], - newFileContents[corruptionPosition] - ); - } else { - logger.info("--> truncating file from length [{}] to length [{}]", oldFileContents.length, newFileContents.length); - } - Files.write(fileToCorrupt, newFileContents); - logger.info("--> contents of [{}] after corruption: [{}]", fileToCorrupt, Base64.getEncoder().encodeToString(newFileContents)); - } - - private static RepositoryFileType getRandomCorruptibleFileType() { - return randomValueOtherThanMany( - // these blob types do not have reliable corruption detection, so we must skip them - t -> t == RepositoryFileType.ROOT_INDEX_N || t == RepositoryFileType.ROOT_INDEX_LATEST, - () -> randomFrom(RepositoryFileType.values()) - ); - } - - private static Path getRandomFileToCorrupt(Path repositoryRootPath, RepositoryFileType corruptedFileType) throws IOException { - final var corruptibleFiles = new ArrayList(); - Files.walkFileTree(repositoryRootPath, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) throws IOException { - if (ExtrasFS.isExtra(filePath.getFileName().toString()) == false - && RepositoryFileType.getRepositoryFileType(repositoryRootPath, filePath) == corruptedFileType) { - corruptibleFiles.add(filePath); - } - return super.visitFile(filePath, attrs); - } - }); - return randomFrom(corruptibleFiles); - } - } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java new file mode 100644 index 0000000000000..40d98b2b5ea71 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java @@ -0,0 +1,708 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.ccs; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot; +import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result; +import org.elasticsearch.action.search.ClosePointInTimeRequest; +import org.elasticsearch.action.search.OpenPointInTimeRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.TransportClosePointInTimeAction; +import org.elasticsearch.action.search.TransportOpenPointInTimeAction; +import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.builder.PointInTimeBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.query.SlowRunningQueryBuilder; +import org.elasticsearch.search.query.ThrowingQueryBuilder; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.usage.UsageService; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.MRT_FEATURE; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.WILDCARD_FEATURE; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.equalTo; + +public class CCSUsageTelemetryIT extends AbstractMultiClustersTestCase { + private static final Logger LOGGER = LogManager.getLogger(CCSUsageTelemetryIT.class); + private static final String REMOTE1 = "cluster-a"; + private static final String REMOTE2 = "cluster-b"; + + @Override + protected boolean reuseClusters() { + return false; + } + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE1, REMOTE2); + } + + @Rule + public SkipUnavailableRule skipOverride = new SkipUnavailableRule(REMOTE1, REMOTE2); + + @Override + protected Map skipUnavailableForRemoteClusters() { + var map = skipOverride.getMap(); + LOGGER.info("Using skip_unavailable map: [{}]", map); + return map; + } + + @Override + protected Collection> nodePlugins(String clusterAlias) { + return CollectionUtils.appendToCopy(super.nodePlugins(clusterAlias), CrossClusterSearchIT.TestQueryBuilderPlugin.class); + } + + private SearchRequest makeSearchRequest(String... indices) { + SearchRequest searchRequest = new SearchRequest(indices); + searchRequest.allowPartialSearchResults(false); + searchRequest.setBatchedReduceSize(randomIntBetween(3, 20)); + searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); + if (randomBoolean()) { + searchRequest.setPreFilterShardSize(1); + } + searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(10)); + return searchRequest; + } + + /** + * Run search request and get telemetry from it + */ + private CCSTelemetrySnapshot getTelemetryFromSearch(SearchRequest searchRequest) throws ExecutionException, InterruptedException { + // We want to send search to a specific node (we don't care which one) so that we could + // collect the CCS telemetry from it later + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + // We don't care here too much about the response, we just want to trigger the telemetry collection. + // So we check it's not null and leave the rest to other tests. + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + return getTelemetrySnapshot(nodeName); + } + + private CCSTelemetrySnapshot getTelemetryFromFailedSearch(SearchRequest searchRequest) throws Exception { + // We want to send search to a specific node (we don't care which one) so that we could + // collect the CCS telemetry from it later + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + PlainActionFuture queryFuture = new PlainActionFuture<>(); + cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest, queryFuture); + assertBusy(() -> assertTrue(queryFuture.isDone())); + + // We expect failure, but we don't care too much which failure it is in this test + ExecutionException ee = expectThrows(ExecutionException.class, queryFuture::get); + assertNotNull(ee.getCause()); + + return getTelemetrySnapshot(nodeName); + } + + /** + * Create search request for indices and get telemetry from it + */ + private CCSTelemetrySnapshot getTelemetryFromSearch(String... indices) throws ExecutionException, InterruptedException { + return getTelemetryFromSearch(makeSearchRequest(indices)); + } + + /** + * Search on all remotes + */ + public void testAllRemotesSearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + boolean minimizeRoundtrips = TransportSearchAction.shouldMinimizeRoundtrips(searchRequest); + + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + assertResponse( + cluster(LOCAL_CLUSTER).client(nodeName) + .filterWithHeader(Map.of(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER, "kibana")) + .search(searchRequest), + Assert::assertNotNull + ); + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + assertThat(telemetry.getTookMrtTrue().count(), equalTo(minimizeRoundtrips ? 1L : 0L)); + assertThat(telemetry.getTookMrtFalse().count(), equalTo(minimizeRoundtrips ? 0L : 1L)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + assertThat(telemetry.getClientCounts().size(), equalTo(1)); + assertThat(telemetry.getClientCounts().get("kibana"), equalTo(1L)); + if (minimizeRoundtrips) { + assertThat(telemetry.getFeatureCounts().get(MRT_FEATURE), equalTo(1L)); + } else { + assertThat(telemetry.getFeatureCounts().get(MRT_FEATURE), equalTo(null)); + } + assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(null)); + + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String clusterAlias : remoteClusterAlias()) { + var clusterTelemetry = perCluster.get(clusterAlias); + assertThat(clusterTelemetry.getCount(), equalTo(1L)); + assertThat(clusterTelemetry.getSkippedCount(), equalTo(0L)); + assertThat(clusterTelemetry.getTook().count(), equalTo(1L)); + } + + // another search + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getSuccessCount(), equalTo(2L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + assertThat(telemetry.getTook().count(), equalTo(2L)); + assertThat(telemetry.getTookMrtTrue().count(), equalTo(minimizeRoundtrips ? 2L : 0L)); + assertThat(telemetry.getTookMrtFalse().count(), equalTo(minimizeRoundtrips ? 0L : 2L)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + assertThat(telemetry.getClientCounts().size(), equalTo(1)); + assertThat(telemetry.getClientCounts().get("kibana"), equalTo(1L)); + perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String clusterAlias : remoteClusterAlias()) { + var clusterTelemetry = perCluster.get(clusterAlias); + assertThat(clusterTelemetry.getCount(), equalTo(2L)); + assertThat(clusterTelemetry.getSkippedCount(), equalTo(0L)); + assertThat(clusterTelemetry.getTook().count(), equalTo(2L)); + } + } + + /** + * Search on a specific remote + */ + public void testOneRemoteSearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + // Make request to cluster a + SearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex); + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(2)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2), equalTo(null)); + assertThat(telemetry.getClientCounts().size(), equalTo(0)); + + // Make request to cluster b + searchRequest = makeSearchRequest(localIndex, REMOTE2 + ":" + remoteIndex); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getSuccessCount(), equalTo(2L)); + perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2).getTook().count(), equalTo(1L)); + } + + /** + * Local search should not produce any telemetry at all + */ + public void testLocalOnlySearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(localIndex); + assertThat(telemetry.getTotalCount(), equalTo(0L)); + } + + /** + * Search on remotes only, without local index + */ + public void testRemoteOnlySearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch("*:" + remoteIndex); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + assertThat(perCluster.size(), equalTo(2)); + assertThat(telemetry.getClientCounts().size(), equalTo(0)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2).getSkippedCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE2).getTook().count(), equalTo(1L)); + } + + /** + * Count wildcard searches. Only wildcards in index names (not in cluster names) are counted. + */ + public void testWildcardSearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(null)); + + searchRequest = makeSearchRequest("*", REMOTE1 + ":" + remoteIndex); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(1L)); + + searchRequest = makeSearchRequest(localIndex, REMOTE2 + ":*"); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(3L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(2L)); + + // Wildcards in cluster name do not count + searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(4L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(2L)); + + // Wildcard in the middle of the index name counts + searchRequest = makeSearchRequest(localIndex, REMOTE2 + ":rem*"); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(5L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(3L)); + + // Wildcard only counted once per search + searchRequest = makeSearchRequest("*", REMOTE1 + ":rem*", REMOTE2 + ":remote*"); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(6L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(4L)); + } + + /** + * Test complete search failure + */ + public void testFailedSearch() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + // shardId -1 means to throw the Exception on all shards, so should result in complete search failure + ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder(randomLong(), new IllegalStateException("index corrupted"), -1); + searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10)); + searchRequest.allowPartialSearchResults(true); + + CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + assertThat(telemetry.getTook().count(), equalTo(0L)); + assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L)); + assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L)); + Map expectedFailures = Map.of(Result.UNKNOWN.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailures)); + } + + /** + * Search when all the remotes failed and skipped + */ + public void testSkippedAllRemotesSearch() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + // throw Exception on all shards of remoteIndex, but not against localIndex + ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder( + randomLong(), + new IllegalStateException("index corrupted"), + remoteIndex + ); + searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10)); + searchRequest.allowPartialSearchResults(true); + + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + // Note that this counts how many searches had skipped remotes, not how many remotes are skipped + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + // Still count the remote that failed + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + // Each remote will have its skipped count bumped + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String remote : remoteClusterAlias()) { + assertThat(perCluster.get(remote).getCount(), equalTo(0L)); + assertThat(perCluster.get(remote).getSkippedCount(), equalTo(1L)); + assertThat(perCluster.get(remote).getTook().count(), equalTo(0L)); + } + } + + public void testSkippedOneRemoteSearch() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + // Remote1 will fail, Remote2 will just do nothing but it counts as success + SearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex, REMOTE2 + ":" + "nosuchindex*"); + // throw Exception on all shards of remoteIndex, but not against localIndex + ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder( + randomLong(), + new IllegalStateException("index corrupted"), + remoteIndex + ); + searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10)); + searchRequest.allowPartialSearchResults(true); + + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + // Note that this counts how many searches had skipped remotes, not how many remotes are skipped + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + // Still count the remote that failed + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + // Each remote will have its skipped count bumped + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + // This one is skipped + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(0L)); + // This one is OK + assertThat(perCluster.get(REMOTE2).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2).getSkippedCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE2).getTook().count(), equalTo(1L)); + } + + /** + * Test what happens if remote times out - it should be skipped + */ + public void testRemoteTimesOut() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex); + // This works only with minimize_roundtrips enabled, since otherwise timed out shards will be counted as + // partial failure, and we disable partial results.. + searchRequest.setCcsMinimizeRoundtrips(true); + + TimeValue searchTimeout = new TimeValue(200, TimeUnit.MILLISECONDS); + // query builder that will sleep for the specified amount of time in the query phase + SlowRunningQueryBuilder slowRunningQueryBuilder = new SlowRunningQueryBuilder(searchTimeout.millis() * 5, remoteIndex); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(slowRunningQueryBuilder).timeout(searchTimeout); + searchRequest.source(sourceBuilder); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L)); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(2)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(0L)); + assertThat(perCluster.get(REMOTE2), equalTo(null)); + } + + /** + * Test what happens if remote times out and there's no local - it should be skipped + */ + public void testRemoteOnlyTimesOut() throws Exception { + Map testClusterInfo = setupClusters(); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":" + remoteIndex); + // This works only with minimize_roundtrips enabled, since otherwise timed out shards will be counted as + // partial failure, and we disable partial results... + searchRequest.setCcsMinimizeRoundtrips(true); + + TimeValue searchTimeout = new TimeValue(100, TimeUnit.MILLISECONDS); + // query builder that will sleep for the specified amount of time in the query phase + SlowRunningQueryBuilder slowRunningQueryBuilder = new SlowRunningQueryBuilder(searchTimeout.millis() * 5, remoteIndex); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(slowRunningQueryBuilder).timeout(searchTimeout); + searchRequest.source(sourceBuilder); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L)); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(1)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(0L)); + assertThat(perCluster.get(REMOTE2), equalTo(null)); + } + + @SkipOverride(aliases = { REMOTE1 }) + public void testRemoteTimesOutFailure() throws Exception { + Map testClusterInfo = setupClusters(); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":" + remoteIndex); + + TimeValue searchTimeout = new TimeValue(100, TimeUnit.MILLISECONDS); + // query builder that will sleep for the specified amount of time in the query phase + SlowRunningQueryBuilder slowRunningQueryBuilder = new SlowRunningQueryBuilder(searchTimeout.millis() * 5, remoteIndex); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(slowRunningQueryBuilder).timeout(searchTimeout); + searchRequest.source(sourceBuilder); + + CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + // Failure is not skipping + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + // Still count the remote that failed + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L)); + assertThat(telemetry.getTook().count(), equalTo(0L)); + Map expectedFailure = Map.of(Result.TIMEOUT.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); + // No per-cluster data on total failure + assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); + } + + /** + * Search when all the remotes failed and not skipped + */ + @SkipOverride(aliases = { REMOTE1, REMOTE2 }) + public void testFailedAllRemotesSearch() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + // throw Exception on all shards of remoteIndex, but not against localIndex + ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder( + randomLong(), + new IllegalStateException("index corrupted"), + remoteIndex + ); + searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10)); + + CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + // Failure is not skipping + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + // Still count the remote that failed + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getTook().count(), equalTo(0L)); + Map expectedFailure = Map.of(Result.REMOTES_UNAVAILABLE.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); + // No per-cluster data on total failure + assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); + } + + /** + * Test that we're still counting remote search even if remote cluster has no such index + */ + public void testRemoteHasNoIndex() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(localIndex, REMOTE1 + ":" + "no_such_index*"); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(2)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2), equalTo(null)); + } + + /** + * Test that we're still counting remote search even if remote cluster has no such index + */ + @SkipOverride(aliases = { REMOTE1 }) + public void testRemoteHasNoIndexFailure() throws Exception { + SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":no_such_index"); + CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(0)); + Map expectedFailure = Map.of(Result.NOT_FOUND.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); + } + + public void testPITSearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + OpenPointInTimeRequest openPITRequest = new OpenPointInTimeRequest(localIndex, "*:" + remoteIndex).keepAlive( + TimeValue.timeValueMinutes(5) + ); + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + var client = cluster(LOCAL_CLUSTER).client(nodeName); + BytesReference pitID = client.execute(TransportOpenPointInTimeAction.TYPE, openPITRequest).actionGet().getPointInTimeId(); + SearchRequest searchRequest = new SearchRequest().source( + new SearchSourceBuilder().pointInTimeBuilder(new PointInTimeBuilder(pitID).setKeepAlive(TimeValue.timeValueMinutes(5))) + .sort("@timestamp") + .size(10) + ); + searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); + + assertResponse(client.search(searchRequest), Assert::assertNotNull); + // do it again + assertResponse(client.search(searchRequest), Assert::assertNotNull); + client.execute(TransportClosePointInTimeAction.TYPE, new ClosePointInTimeRequest(pitID)).actionGet(); + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getSuccessCount(), equalTo(2L)); + } + + private CCSTelemetrySnapshot getTelemetrySnapshot(String nodeName) { + var usage = cluster(LOCAL_CLUSTER).getInstance(UsageService.class, nodeName); + return usage.getCcsUsageHolder().getCCSTelemetrySnapshot(); + } + + private Map setupClusters() { + String localIndex = "demo"; + int numShardsLocal = randomIntBetween(2, 10); + Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build(); + assertAcked( + client(LOCAL_CLUSTER).admin() + .indices() + .prepareCreate(localIndex) + .setSettings(localSettings) + .setMapping("@timestamp", "type=date", "f", "type=text") + ); + indexDocs(client(LOCAL_CLUSTER), localIndex); + + String remoteIndex = "prod"; + int numShardsRemote = randomIntBetween(2, 10); + for (String clusterAlias : remoteClusterAlias()) { + final InternalTestCluster remoteCluster = cluster(clusterAlias); + remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(1, 3)); + assertAcked( + client(clusterAlias).admin() + .indices() + .prepareCreate(remoteIndex) + .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1))) + .setMapping("@timestamp", "type=date", "f", "type=text") + ); + assertFalse( + client(clusterAlias).admin() + .cluster() + .prepareHealth(remoteIndex) + .setWaitForYellowStatus() + .setTimeout(TimeValue.timeValueSeconds(10)) + .get() + .isTimedOut() + ); + indexDocs(client(clusterAlias), remoteIndex); + } + + Map clusterInfo = new HashMap<>(); + clusterInfo.put("local.index", localIndex); + clusterInfo.put("remote.index", remoteIndex); + return clusterInfo; + } + + private int indexDocs(Client client, String index) { + int numDocs = between(5, 20); + for (int i = 0; i < numDocs; i++) { + client.prepareIndex(index).setSource("f", "v", "@timestamp", randomNonNegativeLong()).get(); + } + client.admin().indices().prepareRefresh(index).get(); + return numDocs; + } + + /** + * Annotation to mark specific cluster in a test as not to be skipped when unavailable + */ + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.METHOD) + @interface SkipOverride { + String[] aliases(); + } + + /** + * Test rule to process skip annotations + */ + static class SkipUnavailableRule implements TestRule { + private final Map skipMap; + + SkipUnavailableRule(String... clusterAliases) { + this.skipMap = Arrays.stream(clusterAliases).collect(Collectors.toMap(Function.identity(), alias -> true)); + } + + public Map getMap() { + return skipMap; + } + + @Override + public Statement apply(Statement base, Description description) { + // Check for annotation named "SkipOverride" and set the overrides accordingly + var aliases = description.getAnnotation(SkipOverride.class); + if (aliases != null) { + for (String alias : aliases.aliases()) { + skipMap.put(alias, false); + } + } + return base; + } + + } +} diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index d412748ed4e57..086bfece87172 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -190,6 +190,7 @@ exports org.elasticsearch.common.file; exports org.elasticsearch.common.geo; exports org.elasticsearch.common.hash; + exports org.elasticsearch.injection.api; exports org.elasticsearch.injection.guice; exports org.elasticsearch.injection.guice.binder; exports org.elasticsearch.injection.guice.internal; diff --git a/server/src/main/java/org/elasticsearch/ReleaseVersions.java b/server/src/main/java/org/elasticsearch/ReleaseVersions.java index 7b5c8d1d42382..cacdca1c5b528 100644 --- a/server/src/main/java/org/elasticsearch/ReleaseVersions.java +++ b/server/src/main/java/org/elasticsearch/ReleaseVersions.java @@ -41,7 +41,7 @@ public class ReleaseVersions { private static final Pattern VERSION_LINE = Pattern.compile("(\\d+\\.\\d+\\.\\d+),(\\d+)"); - public static IntFunction generateVersionsLookup(Class versionContainer) { + public static IntFunction generateVersionsLookup(Class versionContainer, int current) { if (USES_VERSIONS == false) return Integer::toString; try { @@ -52,6 +52,9 @@ public static IntFunction generateVersionsLookup(Class versionContain } NavigableMap> versions = new TreeMap<>(); + // add the current version id, which won't be in the csv + versions.put(current, List.of(Version.CURRENT)); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(versionsFile, StandardCharsets.UTF_8))) { String line; while ((line = reader.readLine()) != null) { @@ -121,8 +124,8 @@ private static IntFunction lookupFunction(NavigableMap getAllVersions() { return VERSION_IDS.values(); } - static final IntFunction VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(TransportVersions.class); + static final IntFunction VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(TransportVersions.class, LATEST_DEFINED.id()); // no instance private TransportVersions() {} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java new file mode 100644 index 0000000000000..fe1da86dd54c7 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java @@ -0,0 +1,404 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.action.admin.cluster.stats.LongMetric.LongMetricValue; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Holds a snapshot of the CCS telemetry statistics from {@link CCSUsageTelemetry}. + * Used to hold the stats for a single node that's part of a {@link ClusterStatsNodeResponse}, as well as to + * accumulate stats for the entire cluster and return them as part of the {@link ClusterStatsResponse}. + *
+ * Theory of operation: + * - The snapshot is created on each particular node with the stats for the node, and is sent to the coordinating node + * - Coordinating node creates an empty snapshot and merges all the node snapshots into it using add() + *
+ * The snapshot contains {@link LongMetricValue}s for latencies, which currently contain full histograms (since you can't + * produce p90 from a set of node p90s, you need the full histogram for that). To avoid excessive copying (histogram weighs several KB), + * the snapshot is designed to be mutable, so that you can add multiple snapshots to it without copying the histograms all the time. + * It is not the intent to mutate the snapshot objects otherwise. + *
+ */ +public final class CCSTelemetrySnapshot implements Writeable, ToXContentFragment { + public static final String CCS_TELEMETRY_FIELD_NAME = "_search"; + private long totalCount; + private long successCount; + private final Map failureReasons; + + /** + * Latency metrics, overall. + */ + private final LongMetricValue took; + /** + * Latency metrics with minimize_roundtrips=true + */ + private final LongMetricValue tookMrtTrue; + /** + * Latency metrics with minimize_roundtrips=false + */ + private final LongMetricValue tookMrtFalse; + private long remotesPerSearchMax; + private double remotesPerSearchAvg; + private long skippedRemotes; + + private final Map featureCounts; + + private final Map clientCounts; + private final Map byRemoteCluster; + + /** + * Creates a new stats instance with the provided info. + */ + public CCSTelemetrySnapshot( + long totalCount, + long successCount, + Map failureReasons, + LongMetricValue took, + LongMetricValue tookMrtTrue, + LongMetricValue tookMrtFalse, + long remotesPerSearchMax, + double remotesPerSearchAvg, + long skippedRemotes, + Map featureCounts, + Map clientCounts, + Map byRemoteCluster + ) { + this.totalCount = totalCount; + this.successCount = successCount; + this.failureReasons = failureReasons; + this.took = took; + this.tookMrtTrue = tookMrtTrue; + this.tookMrtFalse = tookMrtFalse; + this.remotesPerSearchMax = remotesPerSearchMax; + this.remotesPerSearchAvg = remotesPerSearchAvg; + this.skippedRemotes = skippedRemotes; + this.featureCounts = featureCounts; + this.clientCounts = clientCounts; + this.byRemoteCluster = byRemoteCluster; + } + + /** + * Creates a new empty stats instance, that will get additional stats added through {@link #add(CCSTelemetrySnapshot)} + */ + public CCSTelemetrySnapshot() { + // Note this produces modifiable maps, so other snapshots can be merged into it + failureReasons = new HashMap<>(); + featureCounts = new HashMap<>(); + clientCounts = new HashMap<>(); + byRemoteCluster = new HashMap<>(); + took = new LongMetricValue(); + tookMrtTrue = new LongMetricValue(); + tookMrtFalse = new LongMetricValue(); + } + + public CCSTelemetrySnapshot(StreamInput in) throws IOException { + this.totalCount = in.readVLong(); + this.successCount = in.readVLong(); + this.failureReasons = in.readMap(StreamInput::readLong); + this.took = LongMetricValue.fromStream(in); + this.tookMrtTrue = LongMetricValue.fromStream(in); + this.tookMrtFalse = LongMetricValue.fromStream(in); + this.remotesPerSearchMax = in.readVLong(); + this.remotesPerSearchAvg = in.readDouble(); + this.skippedRemotes = in.readVLong(); + this.featureCounts = in.readMap(StreamInput::readLong); + this.clientCounts = in.readMap(StreamInput::readLong); + this.byRemoteCluster = in.readMap(PerClusterCCSTelemetry::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(totalCount); + out.writeVLong(successCount); + out.writeMap(failureReasons, StreamOutput::writeLong); + took.writeTo(out); + tookMrtTrue.writeTo(out); + tookMrtFalse.writeTo(out); + out.writeVLong(remotesPerSearchMax); + out.writeDouble(remotesPerSearchAvg); + out.writeVLong(skippedRemotes); + out.writeMap(featureCounts, StreamOutput::writeLong); + out.writeMap(clientCounts, StreamOutput::writeLong); + out.writeMap(byRemoteCluster, StreamOutput::writeWriteable); + } + + public long getTotalCount() { + return totalCount; + } + + public long getSuccessCount() { + return successCount; + } + + public Map getFailureReasons() { + return Collections.unmodifiableMap(failureReasons); + } + + public LongMetricValue getTook() { + return took; + } + + public LongMetricValue getTookMrtTrue() { + return tookMrtTrue; + } + + public LongMetricValue getTookMrtFalse() { + return tookMrtFalse; + } + + public long getRemotesPerSearchMax() { + return remotesPerSearchMax; + } + + public double getRemotesPerSearchAvg() { + return remotesPerSearchAvg; + } + + public long getSearchCountWithSkippedRemotes() { + return skippedRemotes; + } + + public Map getFeatureCounts() { + return Collections.unmodifiableMap(featureCounts); + } + + public Map getClientCounts() { + return Collections.unmodifiableMap(clientCounts); + } + + public Map getByRemoteCluster() { + return Collections.unmodifiableMap(byRemoteCluster); + } + + public static class PerClusterCCSTelemetry implements Writeable, ToXContentFragment { + private long count; + private long skippedCount; + private final LongMetricValue took; + + public PerClusterCCSTelemetry() { + took = new LongMetricValue(); + } + + public PerClusterCCSTelemetry(long count, long skippedCount, LongMetricValue took) { + this.took = took; + this.skippedCount = skippedCount; + this.count = count; + } + + public PerClusterCCSTelemetry(PerClusterCCSTelemetry other) { + this.count = other.count; + this.skippedCount = other.skippedCount; + this.took = new LongMetricValue(other.took); + } + + public PerClusterCCSTelemetry(StreamInput in) throws IOException { + this.count = in.readVLong(); + this.skippedCount = in.readVLong(); + this.took = LongMetricValue.fromStream(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(count); + out.writeVLong(skippedCount); + took.writeTo(out); + } + + public PerClusterCCSTelemetry add(PerClusterCCSTelemetry v) { + count += v.count; + skippedCount += v.skippedCount; + took.add(v.took); + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("total", count); + builder.field("skipped", skippedCount); + publishLatency(builder, "took", took); + builder.endObject(); + return builder; + } + + public long getCount() { + return count; + } + + public long getSkippedCount() { + return skippedCount; + } + + public LongMetricValue getTook() { + return took; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PerClusterCCSTelemetry that = (PerClusterCCSTelemetry) o; + return count == that.count && skippedCount == that.skippedCount && Objects.equals(took, that.took); + } + + @Override + public int hashCode() { + return Objects.hash(count, skippedCount, took); + } + } + + /** + * Add the provided stats to the ones held by the current instance, effectively merging the two. + * @param stats the other stats object to add to this one + */ + public void add(CCSTelemetrySnapshot stats) { + // This should be called in ClusterStatsResponse ctor only, so we don't need to worry about concurrency + if (stats.totalCount == 0) { + // Just ignore the empty stats. + // This could happen if the node is brand new or if the stats are not available, e.g. because it runs an old version. + return; + } + long oldCount = totalCount; + totalCount += stats.totalCount; + successCount += stats.successCount; + skippedRemotes += stats.skippedRemotes; + stats.failureReasons.forEach((k, v) -> failureReasons.merge(k, v, Long::sum)); + stats.featureCounts.forEach((k, v) -> featureCounts.merge(k, v, Long::sum)); + stats.clientCounts.forEach((k, v) -> clientCounts.merge(k, v, Long::sum)); + took.add(stats.took); + tookMrtTrue.add(stats.tookMrtTrue); + tookMrtFalse.add(stats.tookMrtFalse); + remotesPerSearchMax = Math.max(remotesPerSearchMax, stats.remotesPerSearchMax); + if (totalCount > 0 && oldCount > 0) { + // Weighted average + remotesPerSearchAvg = (remotesPerSearchAvg * oldCount + stats.remotesPerSearchAvg * stats.totalCount) / totalCount; + } else { + // If we didn't have any old value, we just take the new one + remotesPerSearchAvg = stats.remotesPerSearchAvg; + } + // we copy the object here since we'll be modifying it later on subsequent adds + // TODO: this may be sub-optimal, as we'll be copying histograms when adding first snapshot to an empty container, + // which we could have avoided probably. + stats.byRemoteCluster.forEach((r, v) -> byRemoteCluster.merge(r, new PerClusterCCSTelemetry(v), PerClusterCCSTelemetry::add)); + } + + /** + * Publishes the latency statistics to the provided {@link XContentBuilder}. + * Example: + * "took": { + * "max": 345032, + * "avg": 1620, + * "p90": 2570 + * } + */ + public static void publishLatency(XContentBuilder builder, String name, LongMetricValue took) throws IOException { + builder.startObject(name); + { + builder.field("max", took.max()); + builder.field("avg", took.avg()); + builder.field("p90", took.p90()); + } + builder.endObject(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(CCS_TELEMETRY_FIELD_NAME); + { + builder.field("total", totalCount); + builder.field("success", successCount); + builder.field("skipped", skippedRemotes); + publishLatency(builder, "took", took); + publishLatency(builder, "took_mrt_true", tookMrtTrue); + publishLatency(builder, "took_mrt_false", tookMrtFalse); + builder.field("remotes_per_search_max", remotesPerSearchMax); + builder.field("remotes_per_search_avg", remotesPerSearchAvg); + builder.field("failure_reasons", failureReasons); + builder.field("features", featureCounts); + builder.field("clients", clientCounts); + builder.startObject("clusters"); + { + for (var entry : byRemoteCluster.entrySet()) { + String remoteName = entry.getKey(); + if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(remoteName)) { + remoteName = SearchResponse.LOCAL_CLUSTER_NAME_REPRESENTATION; + } + builder.field(remoteName, entry.getValue()); + } + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CCSTelemetrySnapshot that = (CCSTelemetrySnapshot) o; + return totalCount == that.totalCount + && successCount == that.successCount + && skippedRemotes == that.skippedRemotes + && Objects.equals(failureReasons, that.failureReasons) + && Objects.equals(took, that.took) + && Objects.equals(tookMrtTrue, that.tookMrtTrue) + && Objects.equals(tookMrtFalse, that.tookMrtFalse) + && Objects.equals(remotesPerSearchMax, that.remotesPerSearchMax) + && Objects.equals(remotesPerSearchAvg, that.remotesPerSearchAvg) + && Objects.equals(featureCounts, that.featureCounts) + && Objects.equals(clientCounts, that.clientCounts) + && Objects.equals(byRemoteCluster, that.byRemoteCluster); + } + + @Override + public int hashCode() { + return Objects.hash( + totalCount, + successCount, + failureReasons, + took, + tookMrtTrue, + tookMrtFalse, + remotesPerSearchMax, + remotesPerSearchAvg, + skippedRemotes, + featureCounts, + clientCounts, + byRemoteCluster + ); + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java new file mode 100644 index 0000000000000..b2d75ac8f61f3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ShardOperationFailedException; +import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.query.SearchTimeoutException; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.NoSeedNodeLeftException; +import org.elasticsearch.transport.NoSuchRemoteClusterException; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.transport.RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; + +/** + * This is a container for telemetry data from an individual cross-cluster search for _search or _async_search (or + * other search endpoints that use the {@link TransportSearchAction} such as _msearch). + */ +public class CCSUsage { + private final long took; + private final Result status; + private final Set features; + private final int remotesCount; + + private final String client; + + private final Set skippedRemotes; + private final Map perClusterUsage; + + public static class Builder { + private long took; + private final Set features; + private Result status = Result.SUCCESS; + private int remotesCount; + private String client; + private final Set skippedRemotes; + private final Map perClusterUsage; + + public Builder() { + features = new HashSet<>(); + skippedRemotes = new HashSet<>(); + perClusterUsage = new HashMap<>(); + } + + public Builder took(long took) { + this.took = took; + return this; + } + + public Builder setFailure(Result failureType) { + this.status = failureType; + return this; + } + + public Builder setFailure(Exception e) { + return setFailure(getFailureType(e)); + } + + public Builder setFeature(String feature) { + this.features.add(feature); + return this; + } + + public Builder setClient(String client) { + this.client = client; + return this; + } + + public Builder skippedRemote(String remote) { + this.skippedRemotes.add(remote); + return this; + } + + public Builder perClusterUsage(String remote, TimeValue took) { + this.perClusterUsage.put(remote, new PerClusterUsage(took)); + return this; + } + + public CCSUsage build() { + return new CCSUsage(took, status, remotesCount, skippedRemotes, features, client, perClusterUsage); + } + + public Builder setRemotesCount(int remotesCount) { + this.remotesCount = remotesCount; + return this; + } + + public int getRemotesCount() { + return remotesCount; + } + + /** + * Get failure type as {@link Result} from the search failure exception. + */ + public static Result getFailureType(Exception e) { + var unwrapped = ExceptionsHelper.unwrapCause(e); + if (unwrapped instanceof Exception) { + e = (Exception) unwrapped; + } + if (isRemoteUnavailable(e)) { + return Result.REMOTES_UNAVAILABLE; + } + if (ExceptionsHelper.unwrap(e, ResourceNotFoundException.class) != null) { + return Result.NOT_FOUND; + } + if (e instanceof TaskCancelledException || (ExceptionsHelper.unwrap(e, TaskCancelledException.class) != null)) { + return Result.CANCELED; + } + if (ExceptionsHelper.unwrap(e, SearchTimeoutException.class) != null) { + return Result.TIMEOUT; + } + if (ExceptionsHelper.unwrap(e, ElasticsearchSecurityException.class) != null) { + return Result.SECURITY; + } + if (ExceptionsHelper.unwrapCorruption(e) != null) { + return Result.CORRUPTION; + } + // This is kind of last resort check - if we still don't know the reason but all shard failures are remote, + // we assume it's remote's fault somehow. + if (e instanceof SearchPhaseExecutionException spe) { + // If this is a failure that happened because of remote failures only + var groupedFails = ExceptionsHelper.groupBy(spe.shardFailures()); + if (Arrays.stream(groupedFails).allMatch(Builder::isRemoteFailure)) { + return Result.REMOTES_UNAVAILABLE; + } + } + // OK we don't know what happened + return Result.UNKNOWN; + } + + /** + * Is this failure exception because remote was unavailable? + * See also: TransportResolveClusterAction#notConnectedError + */ + static boolean isRemoteUnavailable(Exception e) { + if (ExceptionsHelper.unwrap( + e, + ConnectTransportException.class, + NoSuchRemoteClusterException.class, + NoSeedNodeLeftException.class + ) != null) { + return true; + } + Throwable ill = ExceptionsHelper.unwrap(e, IllegalStateException.class, IllegalArgumentException.class); + if (ill != null && (ill.getMessage().contains("Unable to open any connections") || ill.getMessage().contains("unknown host"))) { + return true; + } + // Ok doesn't look like any of the known remote exceptions + return false; + } + + /** + * Is this failure coming from a remote cluster? + */ + static boolean isRemoteFailure(ShardOperationFailedException failure) { + if (failure instanceof ShardSearchFailure shardFailure) { + SearchShardTarget shard = shardFailure.shard(); + return shard != null && shard.getClusterAlias() != null && LOCAL_CLUSTER_GROUP_KEY.equals(shard.getClusterAlias()) == false; + } + return false; + } + } + + private CCSUsage( + long took, + Result status, + int remotesCount, + Set skippedRemotes, + Set features, + String client, + Map perClusterUsage + ) { + this.status = status; + this.remotesCount = remotesCount; + this.features = features; + this.client = client; + this.took = took; + this.skippedRemotes = skippedRemotes; + this.perClusterUsage = perClusterUsage; + } + + public Map getPerClusterUsage() { + return perClusterUsage; + } + + public Result getStatus() { + return status; + } + + public Set getFeatures() { + return features; + } + + public long getRemotesCount() { + return remotesCount; + } + + public String getClient() { + return client; + } + + public long getTook() { + return took; + } + + public Set getSkippedRemotes() { + return skippedRemotes; + } + + public static class PerClusterUsage { + + // if MRT=true, the took time on the remote cluster (if MRT=true), otherwise the overall took time + private long took; + + public PerClusterUsage(TimeValue took) { + if (took != null) { + this.took = took.millis(); + } + } + + public long getTook() { + return took; + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java new file mode 100644 index 0000000000000..60766bd4068e3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.common.util.Maps; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.LongAdder; + +/** + * Service holding accumulated CCS search usage statistics. Individual cross-cluster searches will pass + * CCSUsage data here to have it collated and aggregated. Snapshots of the current CCS Telemetry Usage + * can be obtained by getting {@link CCSTelemetrySnapshot} objects. + *
+ * Theory of operation: + * Each search creates a {@link CCSUsage.Builder}, which can be updated during the progress of the search request, + * and then it instantiates a {@link CCSUsage} object when the request is finished. + * That object is passed to {@link #updateUsage(CCSUsage)} on the request processing end (whether successful or not). + * The {@link #updateUsage(CCSUsage)} method will then update the internal counters and metrics. + *
+ * When we need to return the current state of the telemetry, we can call {@link #getCCSTelemetrySnapshot()} which produces + * a snapshot of the current state of the telemetry as {@link CCSTelemetrySnapshot}. These snapshots are additive so + * when collecting the snapshots from multiple nodes, an empty snapshot is created and then all the node's snapshots are added + * to it to obtain the summary telemetry. + */ +public class CCSUsageTelemetry { + + /** + * Result of the request execution. + * Either "success" or a failure reason. + */ + public enum Result { + SUCCESS("success"), + REMOTES_UNAVAILABLE("remotes_unavailable"), + CANCELED("canceled"), + NOT_FOUND("not_found"), + TIMEOUT("timeout"), + CORRUPTION("corruption"), + SECURITY("security"), + // May be helpful if there's a lot of other reasons, and it may be hard to calculate the unknowns for some clients. + UNKNOWN("other"); + + private final String name; + + Result(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + // Not enum because we won't mind other places adding their own features + public static final String MRT_FEATURE = "mrt_on"; + public static final String ASYNC_FEATURE = "async"; + public static final String WILDCARD_FEATURE = "wildcards"; + + // The list of known Elastic clients. May be incomplete. + public static final Set KNOWN_CLIENTS = Set.of( + "kibana", + "cloud", + "logstash", + "beats", + "fleet", + "ml", + "security", + "observability", + "enterprise-search", + "elasticsearch", + "connectors", + "connectors-cli" + ); + + private final LongAdder totalCount; + private final LongAdder successCount; + private final Map failureReasons; + + /** + * Latency metrics overall + */ + private final LongMetric took; + /** + * Latency metrics with minimize_roundtrips=true + */ + private final LongMetric tookMrtTrue; + /** + * Latency metrics with minimize_roundtrips=false + */ + private final LongMetric tookMrtFalse; + private final LongMetric remotesPerSearch; + private final LongAdder skippedRemotes; + + private final Map featureCounts; + + private final Map clientCounts; + private final Map byRemoteCluster; + + public CCSUsageTelemetry() { + this.byRemoteCluster = new ConcurrentHashMap<>(); + totalCount = new LongAdder(); + successCount = new LongAdder(); + failureReasons = new ConcurrentHashMap<>(); + took = new LongMetric(); + tookMrtTrue = new LongMetric(); + tookMrtFalse = new LongMetric(); + remotesPerSearch = new LongMetric(); + skippedRemotes = new LongAdder(); + featureCounts = new ConcurrentHashMap<>(); + clientCounts = new ConcurrentHashMap<>(); + } + + public void updateUsage(CCSUsage ccsUsage) { + assert ccsUsage.getRemotesCount() > 0 : "Expected at least one remote cluster in CCSUsage"; + // TODO: fork this to a background thread? + doUpdate(ccsUsage); + } + + // This is not synchronized, instead we ensure that every metric in the class is thread-safe. + private void doUpdate(CCSUsage ccsUsage) { + totalCount.increment(); + long searchTook = ccsUsage.getTook(); + if (isSuccess(ccsUsage)) { + successCount.increment(); + took.record(searchTook); + if (isMRT(ccsUsage)) { + tookMrtTrue.record(searchTook); + } else { + tookMrtFalse.record(searchTook); + } + ccsUsage.getPerClusterUsage().forEach((r, u) -> byRemoteCluster.computeIfAbsent(r, PerClusterCCSTelemetry::new).update(u)); + } else { + failureReasons.computeIfAbsent(ccsUsage.getStatus(), k -> new LongAdder()).increment(); + } + + remotesPerSearch.record(ccsUsage.getRemotesCount()); + if (ccsUsage.getSkippedRemotes().isEmpty() == false) { + skippedRemotes.increment(); + ccsUsage.getSkippedRemotes().forEach(remote -> byRemoteCluster.computeIfAbsent(remote, PerClusterCCSTelemetry::new).skipped()); + } + ccsUsage.getFeatures().forEach(f -> featureCounts.computeIfAbsent(f, k -> new LongAdder()).increment()); + String client = ccsUsage.getClient(); + if (client != null && KNOWN_CLIENTS.contains(client)) { + // We count only known clients for now + clientCounts.computeIfAbsent(ccsUsage.getClient(), k -> new LongAdder()).increment(); + } + } + + private boolean isMRT(CCSUsage ccsUsage) { + return ccsUsage.getFeatures().contains(MRT_FEATURE); + } + + private boolean isSuccess(CCSUsage ccsUsage) { + return ccsUsage.getStatus() == Result.SUCCESS; + } + + public Map getTelemetryByCluster() { + return byRemoteCluster; + } + + /** + * Telemetry of each remote involved in cross cluster searches + */ + public static class PerClusterCCSTelemetry { + private final String clusterAlias; + // The number of successful (not skipped) requests to this cluster. + private final LongAdder count; + private final LongAdder skippedCount; + // This is only over the successful requetss, skipped ones do not count here. + private final LongMetric took; + + PerClusterCCSTelemetry(String clusterAlias) { + this.clusterAlias = clusterAlias; + this.count = new LongAdder(); + took = new LongMetric(); + this.skippedCount = new LongAdder(); + } + + void update(CCSUsage.PerClusterUsage remoteUsage) { + count.increment(); + took.record(remoteUsage.getTook()); + } + + void skipped() { + skippedCount.increment(); + } + + public long getCount() { + return count.longValue(); + } + + @Override + public String toString() { + return "PerClusterCCSTelemetry{" + + "clusterAlias='" + + clusterAlias + + '\'' + + ", count=" + + count + + ", latency=" + + took.toString() + + '}'; + } + + public long getSkippedCount() { + return skippedCount.longValue(); + } + + public CCSTelemetrySnapshot.PerClusterCCSTelemetry getSnapshot() { + return new CCSTelemetrySnapshot.PerClusterCCSTelemetry(count.longValue(), skippedCount.longValue(), took.getValue()); + } + + } + + public CCSTelemetrySnapshot getCCSTelemetrySnapshot() { + Map reasonsMap = Maps.newMapWithExpectedSize(failureReasons.size()); + failureReasons.forEach((k, v) -> reasonsMap.put(k.getName(), v.longValue())); + + LongMetric.LongMetricValue remotes = remotesPerSearch.getValue(); + + // Maps returned here are unmodifiable, but the empty ctor produces modifiable maps + return new CCSTelemetrySnapshot( + totalCount.longValue(), + successCount.longValue(), + Collections.unmodifiableMap(reasonsMap), + took.getValue(), + tookMrtTrue.getValue(), + tookMrtFalse.getValue(), + remotes.max(), + remotes.avg(), + skippedRemotes.longValue(), + Collections.unmodifiableMap(Maps.transformValues(featureCounts, LongAdder::longValue)), + Collections.unmodifiableMap(Maps.transformValues(clientCounts, LongAdder::longValue)), + Collections.unmodifiableMap(Maps.transformValues(byRemoteCluster, PerClusterCCSTelemetry::getSnapshot)) + ); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java new file mode 100644 index 0000000000000..f3bb936b108c0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.HdrHistogram.ConcurrentHistogram; +import org.HdrHistogram.Histogram; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.zip.DataFormatException; + +/** + * Metric class that accepts longs and provides count, average, max and percentiles. + * Abstracts out the details of how exactly the values are stored and calculated. + * {@link LongMetricValue} is a snapshot of the current state of the metric. + */ +public class LongMetric { + private final Histogram values; + private static final int SIGNIFICANT_DIGITS = 2; + + LongMetric() { + values = new ConcurrentHistogram(SIGNIFICANT_DIGITS); + } + + void record(long v) { + values.recordValue(v); + } + + LongMetricValue getValue() { + return new LongMetricValue(values); + } + + /** + * Snapshot of {@link LongMetric} value that provides the current state of the metric. + * Can be added with another {@link LongMetricValue} object. + */ + public static final class LongMetricValue implements Writeable { + // We have to carry the full histogram around since we might need to calculate aggregate percentiles + // after collecting individual stats from the nodes, and we can't do that without having the full histogram. + // This costs about 2K per metric, which was deemed acceptable. + private final Histogram values; + + public LongMetricValue(Histogram values) { + // Copy here since we don't want the snapshot value to change if somebody updates the original one + this.values = values.copy(); + } + + public LongMetricValue(LongMetricValue v) { + this.values = v.values.copy(); + } + + LongMetricValue() { + this.values = new Histogram(SIGNIFICANT_DIGITS); + } + + public void add(LongMetricValue v) { + this.values.add(v.values); + } + + public static LongMetricValue fromStream(StreamInput in) throws IOException { + byte[] b = in.readByteArray(); + ByteBuffer bb = ByteBuffer.wrap(b); + try { + // TODO: not sure what is the good value for minBarForHighestToLowestValueRatio here? + Histogram dh = Histogram.decodeFromCompressedByteBuffer(bb, 1); + return new LongMetricValue(dh); + } catch (DataFormatException e) { + throw new IOException(e); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ByteBuffer b = ByteBuffer.allocate(values.getNeededByteBufferCapacity()); + values.encodeIntoCompressedByteBuffer(b); + int size = b.position(); + out.writeVInt(size); + out.writeBytes(b.array(), 0, size); + } + + public long count() { + return values.getTotalCount(); + } + + public long max() { + return values.getMaxValue(); + } + + public long avg() { + return (long) Math.ceil(values.getMean()); + } + + public long p90() { + return values.getValueAtPercentile(90); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) return true; + if (obj == null || obj.getClass() != this.getClass()) return false; + var that = (LongMetricValue) obj; + return this.values.equals(that.values); + } + + @Override + public int hashCode() { + return Objects.hash(values); + } + + @Override + public String toString() { + return "LongMetricValue[count=" + count() + ", " + "max=" + max() + ", " + "avg=" + avg() + "]"; + } + + } +} diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java index 425461d1f4ba1..7c1304f92eefd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java @@ -101,11 +101,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(primaryResponse); } - public void writeThin(StreamOutput out) throws IOException { - out.writeVInt(id); - DocWriteRequest.writeDocumentRequestThin(out, request); - out.writeOptionalWriteable(primaryResponse == null ? null : primaryResponse::writeThin); - } + public static final Writer THIN_WRITER = (out, item) -> { + out.writeVInt(item.id); + DocWriteRequest.writeDocumentRequestThin(out, item.request); + out.writeOptional(BulkItemResponse.THIN_WRITER, item.primaryResponse); + }; @Override public long ramBytesUsed() { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java index 151e8795d0f82..d3e550eaf05b3 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java @@ -264,7 +264,7 @@ public String toString() { id = in.readVInt(); opType = OpType.fromId(in.readByte()); response = readResponse(shardId, in); - failure = in.readBoolean() ? new Failure(in) : null; + failure = in.readOptionalWriteable(Failure::new); assertConsistent(); } @@ -272,7 +272,7 @@ public String toString() { id = in.readVInt(); opType = OpType.fromId(in.readByte()); response = readResponse(in); - failure = in.readBoolean() ? new Failure(in) : null; + failure = in.readOptionalWriteable(Failure::new); assertConsistent(); } @@ -384,31 +384,21 @@ public void writeTo(StreamOutput out) throws IOException { writeResponseType(out); response.writeTo(out); } - if (failure == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - failure.writeTo(out); - } + out.writeOptionalWriteable(failure); } - public void writeThin(StreamOutput out) throws IOException { - out.writeVInt(id); - out.writeByte(opType.getId()); + public static final Writer THIN_WRITER = (out, item) -> { + out.writeVInt(item.id); + out.writeByte(item.opType.getId()); - if (response == null) { + if (item.response == null) { out.writeByte((byte) 2); } else { - writeResponseType(out); - response.writeThin(out); + item.writeResponseType(out); + item.response.writeThin(out); } - if (failure == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - failure.writeTo(out); - } - } + out.writeOptionalWriteable(item.failure); + }; private void writeResponseType(StreamOutput out) throws IOException { if (response instanceof SimulateIndexResponse) { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java index 0d2942e688382..f7860c47d8b73 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java @@ -130,14 +130,7 @@ public void writeTo(StreamOutput out) throws IOException { throw new IllegalStateException("Inference metadata should have been consumed before writing to the stream"); } super.writeTo(out); - out.writeArray((o, item) -> { - if (item != null) { - o.writeBoolean(true); - item.writeThin(o); - } else { - o.writeBoolean(false); - } - }, items); + out.writeArray((o, item) -> o.writeOptional(BulkItemRequest.THIN_WRITER, item), items); if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) { out.writeBoolean(isSimulated); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java index 3eeb96546c9b0..eb1bb0468c9bb 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java @@ -56,6 +56,6 @@ public void setForcedRefresh(boolean forcedRefresh) { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); shardId.writeTo(out); - out.writeArray((o, item) -> item.writeThin(o), responses); + out.writeArray(BulkItemResponse.THIN_WRITER, responses); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 45cb118691082..8d70e2dd6bb66 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -47,6 +47,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Predicate; import java.util.function.Supplier; @@ -701,6 +702,13 @@ public Cluster getCluster(String clusterAlias) { return clusterInfo.get(clusterAlias); } + /** + * @return collection of cluster aliases in the search response (including "(local)" if was searched). + */ + public Set getClusterAliases() { + return clusterInfo.keySet(); + } + /** * Utility to swap a Cluster object. Guidelines for the remapping function: *
    @@ -803,6 +811,7 @@ public boolean hasClusterObjects() { public boolean hasRemoteClusters() { return total > 1 || clusterInfo.keySet().stream().anyMatch(alias -> alias != RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); } + } /** diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 11e767df9c010..6e1645c1ed711 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -23,6 +23,8 @@ import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction; +import org.elasticsearch.action.admin.cluster.stats.CCSUsage; +import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.IndicesOptions; @@ -46,6 +48,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.ArrayUtils; @@ -84,6 +87,7 @@ import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.usage.UsageService; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentFactory; @@ -156,6 +160,7 @@ public class TransportSearchAction extends HandledTransportAction buildPerIndexOriginalIndices( @@ -305,43 +312,7 @@ public long buildTookInMillis() { @Override protected void doExecute(Task task, SearchRequest searchRequest, ActionListener listener) { - ActionListener loggingAndMetrics = new ActionListener<>() { - @Override - public void onResponse(SearchResponse searchResponse) { - try { - searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis()); - SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus = - SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS; - if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) { - // Deduplicate failures by exception message and index - ShardOperationFailedException[] groupedFailures = ExceptionsHelper.groupBy(searchResponse.getShardFailures()); - for (ShardOperationFailedException f : groupedFailures) { - boolean causeHas500Status = false; - if (f.getCause() != null) { - causeHas500Status = ExceptionsHelper.status(f.getCause()).getStatus() >= 500; - } - if ((f.status().getStatus() >= 500 || causeHas500Status) - && ExceptionsHelper.isNodeOrShardUnavailableTypeException(f.getCause()) == false) { - logger.warn("TransportSearchAction shard failure (partial results response)", f); - responseCountTotalStatus = SearchResponseMetrics.ResponseCountTotalStatus.PARTIAL_FAILURE; - } - } - } - listener.onResponse(searchResponse); - // increment after the delegated onResponse to ensure we don't - // record both a success and a failure if there is an exception - searchResponseMetrics.incrementResponseCount(responseCountTotalStatus); - } catch (Exception e) { - onFailure(e); - } - } - - @Override - public void onFailure(Exception e) { - searchResponseMetrics.incrementResponseCount(SearchResponseMetrics.ResponseCountTotalStatus.FAILURE); - listener.onFailure(e); - } - }; + ActionListener loggingAndMetrics = new SearchResponseActionListener((SearchTask) task, listener); executeRequest((SearchTask) task, searchRequest, loggingAndMetrics, AsyncSearchActionProvider::new); } @@ -396,8 +367,32 @@ void executeRequest( searchPhaseProvider.apply(delegate) ); } else { + if ((listener instanceof TelemetryListener tl) && CCS_TELEMETRY_FEATURE_FLAG.isEnabled()) { + tl.setRemotes(resolvedIndices.getRemoteClusterIndices().size()); + if (isAsyncSearchTask(task)) { + tl.setFeature(CCSUsageTelemetry.ASYNC_FEATURE); + } + String client = task.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER); + if (client != null) { + tl.setClient(client); + } + // Check if any of the index patterns are wildcard patterns + var localIndices = resolvedIndices.getLocalIndices(); + if (localIndices != null && Arrays.stream(localIndices.indices()).anyMatch(Regex::isSimpleMatchPattern)) { + tl.setFeature(CCSUsageTelemetry.WILDCARD_FEATURE); + } + if (resolvedIndices.getRemoteClusterIndices() + .values() + .stream() + .anyMatch(indices -> Arrays.stream(indices.indices()).anyMatch(Regex::isSimpleMatchPattern))) { + tl.setFeature(CCSUsageTelemetry.WILDCARD_FEATURE); + } + } final TaskId parentTaskId = task.taskInfo(clusterService.localNode().getId(), false).taskId(); if (shouldMinimizeRoundtrips(rewritten)) { + if ((listener instanceof TelemetryListener tl) && CCS_TELEMETRY_FEATURE_FLAG.isEnabled()) { + tl.setFeature(CCSUsageTelemetry.MRT_FEATURE); + } final AggregationReduceContext.Builder aggregationReduceContextBuilder = rewritten.source() != null && rewritten.source().aggregations() != null ? searchService.aggReduceContextBuilder(task::isCancelled, rewritten.source().aggregations()) @@ -805,27 +800,26 @@ static void collectSearchShards( for (Map.Entry entry : remoteIndicesByCluster.entrySet()) { final String clusterAlias = entry.getKey(); boolean skipUnavailable = remoteClusterService.isSkipUnavailable(clusterAlias); - TransportSearchAction.CCSActionListener> singleListener = - new TransportSearchAction.CCSActionListener<>( - clusterAlias, - skipUnavailable, - responsesCountDown, - exceptions, - clusters, - listener - ) { - @Override - void innerOnResponse(SearchShardsResponse searchShardsResponse) { - assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); - ccsClusterInfoUpdate(searchShardsResponse, clusters, clusterAlias, timeProvider); - searchShardsResponses.put(clusterAlias, searchShardsResponse); - } + CCSActionListener> singleListener = new CCSActionListener<>( + clusterAlias, + skipUnavailable, + responsesCountDown, + exceptions, + clusters, + listener + ) { + @Override + void innerOnResponse(SearchShardsResponse searchShardsResponse) { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); + ccsClusterInfoUpdate(searchShardsResponse, clusters, clusterAlias, timeProvider); + searchShardsResponses.put(clusterAlias, searchShardsResponse); + } - @Override - Map createFinalResponse() { - return searchShardsResponses; - } - }; + @Override + Map createFinalResponse() { + return searchShardsResponses; + } + }; remoteClusterService.maybeEnsureConnectedAndGetConnection( clusterAlias, skipUnavailable == false, @@ -1520,6 +1514,34 @@ public SearchPhase newSearchPhase( } } + /** + * TransportSearchAction cannot access async-search code, so can't check whether this the Task + * is an instance of AsyncSearchTask, so this roundabout method is used + * @param searchTask SearchTask to analyze + * @return true if this is an async search task; false if a synchronous search task + */ + private boolean isAsyncSearchTask(SearchTask searchTask) { + assert assertAsyncSearchTaskListener(searchTask) : "AsyncSearchTask SearchProgressListener is not one of the expected types"; + // AsyncSearchTask will not return SearchProgressListener.NOOP, since it uses its own progress listener + // which delegates to CCSSingleCoordinatorSearchProgressListener when minimizing roundtrips. + // Only synchronous SearchTask uses SearchProgressListener.NOOP or CCSSingleCoordinatorSearchProgressListener directly + return searchTask.getProgressListener() != SearchProgressListener.NOOP + && searchTask.getProgressListener() instanceof CCSSingleCoordinatorSearchProgressListener == false; + } + + /** + * @param searchTask SearchTask to analyze + * @return true if AsyncSearchTask still uses its own special listener, not one of the two that synchronous SearchTask uses + */ + private boolean assertAsyncSearchTaskListener(SearchTask searchTask) { + if (searchTask.getClass().getSimpleName().contains("AsyncSearchTask")) { + SearchProgressListener progressListener = searchTask.getProgressListener(); + return progressListener != SearchProgressListener.NOOP + && progressListener instanceof CCSSingleCoordinatorSearchProgressListener == false; + } + return true; + } + private static void validateAndResolveWaitForCheckpoint( ClusterState clusterState, IndexNameExpressionResolver resolver, @@ -1824,4 +1846,112 @@ List getLocalShardsIterator( // the returned list must support in-place sorting, so this is the most memory efficient we can do here return Arrays.asList(list); } + + private interface TelemetryListener { + void setRemotes(int count); + + void setFeature(String feature); + + void setClient(String client); + } + + private class SearchResponseActionListener implements ActionListener, TelemetryListener { + private final SearchTask task; + private final ActionListener listener; + private final CCSUsage.Builder usageBuilder; + + SearchResponseActionListener(SearchTask task, ActionListener listener) { + this.task = task; + this.listener = listener; + usageBuilder = new CCSUsage.Builder(); + } + + /** + * Should we collect telemetry for this search? + */ + private boolean collectTelemetry() { + return CCS_TELEMETRY_FEATURE_FLAG.isEnabled() && usageBuilder.getRemotesCount() > 0; + } + + public void setRemotes(int count) { + usageBuilder.setRemotesCount(count); + } + + @Override + public void setFeature(String feature) { + usageBuilder.setFeature(feature); + } + + @Override + public void setClient(String client) { + usageBuilder.setClient(client); + } + + @Override + public void onResponse(SearchResponse searchResponse) { + try { + searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis()); + SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus = + SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS; + if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) { + // Deduplicate failures by exception message and index + ShardOperationFailedException[] groupedFailures = ExceptionsHelper.groupBy(searchResponse.getShardFailures()); + for (ShardOperationFailedException f : groupedFailures) { + boolean causeHas500Status = false; + if (f.getCause() != null) { + causeHas500Status = ExceptionsHelper.status(f.getCause()).getStatus() >= 500; + } + if ((f.status().getStatus() >= 500 || causeHas500Status) + && ExceptionsHelper.isNodeOrShardUnavailableTypeException(f.getCause()) == false) { + logger.warn("TransportSearchAction shard failure (partial results response)", f); + responseCountTotalStatus = SearchResponseMetrics.ResponseCountTotalStatus.PARTIAL_FAILURE; + } + } + } + searchResponseMetrics.incrementResponseCount(responseCountTotalStatus); + + if (collectTelemetry()) { + extractCCSTelemetry(searchResponse); + recordTelemetry(); + } + } catch (Exception e) { + onFailure(e); + return; + } + // This is last because we want to collect telemetry before returning the response. + listener.onResponse(searchResponse); + } + + @Override + public void onFailure(Exception e) { + searchResponseMetrics.incrementResponseCount(SearchResponseMetrics.ResponseCountTotalStatus.FAILURE); + if (collectTelemetry()) { + usageBuilder.setFailure(e); + recordTelemetry(); + } + listener.onFailure(e); + } + + private void recordTelemetry() { + usageService.getCcsUsageHolder().updateUsage(usageBuilder.build()); + } + + /** + * Extract telemetry data from the search response. + * @param searchResponse The final response from the search. + */ + private void extractCCSTelemetry(SearchResponse searchResponse) { + usageBuilder.took(searchResponse.getTookInMillis()); + for (String clusterAlias : searchResponse.getClusters().getClusterAliases()) { + SearchResponse.Cluster cluster = searchResponse.getClusters().getCluster(clusterAlias); + if (cluster.getStatus() == SearchResponse.Cluster.Status.SKIPPED) { + usageBuilder.skippedRemote(clusterAlias); + } else { + usageBuilder.perClusterUsage(clusterAlias, cluster.getTook()); + } + } + + } + + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java index 8fb91d89417e0..b6c19f331c712 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java @@ -40,9 +40,11 @@ import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.health.Diagnosis; import org.elasticsearch.health.HealthIndicatorDetails; import org.elasticsearch.health.HealthIndicatorImpact; @@ -56,6 +58,7 @@ import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.snapshots.SnapshotShardSizeInfo; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -108,11 +111,29 @@ public class ShardsAvailabilityHealthIndicatorService implements HealthIndicator private static final String DATA_TIER_ALLOCATION_DECIDER_NAME = "data_tier"; + /** + * Changes the behavior of isNewlyCreatedAndInitializingReplica so that the + * shard_availability health indicator returns YELLOW if a primary + * is STARTED, but a replica is still INITIALIZING and the replica has been + * unassigned for less than the value of this setting. This function is + * only used in serverless, so this setting has no effect in stateless. + */ + public static final Setting REPLICA_UNASSIGNED_BUFFER_TIME = Setting.timeSetting( + "health.shards_availability.replica_unassigned_buffer_time", + TimeValue.timeValueSeconds(3), + TimeValue.timeValueSeconds(0), + TimeValue.timeValueSeconds(20), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private final ClusterService clusterService; private final AllocationService allocationService; private final SystemIndices systemIndices; + private volatile TimeValue replicaUnassignedBufferTime = TimeValue.timeValueSeconds(0); + public ShardsAvailabilityHealthIndicatorService( ClusterService clusterService, AllocationService allocationService, @@ -121,6 +142,11 @@ public ShardsAvailabilityHealthIndicatorService( this.clusterService = clusterService; this.allocationService = allocationService; this.systemIndices = systemIndices; + clusterService.getClusterSettings().addSettingsUpdateConsumer(REPLICA_UNASSIGNED_BUFFER_TIME, this::setReplicaUnassignedBufferTime); + } + + private void setReplicaUnassignedBufferTime(TimeValue replicaUnassignedBufferTime) { + this.replicaUnassignedBufferTime = replicaUnassignedBufferTime; } @Override @@ -144,7 +170,7 @@ public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResources var state = clusterService.state(); var shutdown = state.getMetadata().custom(NodesShutdownMetadata.TYPE, NodesShutdownMetadata.EMPTY); var status = createNewStatus(state.getMetadata()); - updateShardAllocationStatus(status, state, shutdown, verbose); + updateShardAllocationStatus(status, state, shutdown, verbose, replicaUnassignedBufferTime); return createIndicator( status.getStatus(), status.getSymptom(), @@ -158,14 +184,15 @@ static void updateShardAllocationStatus( ShardAllocationStatus status, ClusterState state, NodesShutdownMetadata shutdown, - boolean verbose + boolean verbose, + TimeValue replicaUnassignedBufferTime ) { for (IndexRoutingTable indexShardRouting : state.routingTable()) { for (int i = 0; i < indexShardRouting.size(); i++) { IndexShardRoutingTable shardRouting = indexShardRouting.shard(i); status.addPrimary(shardRouting.primaryShard(), state, shutdown, verbose); for (ShardRouting replicaShard : shardRouting.replicaShards()) { - status.addReplica(replicaShard, state, shutdown, verbose); + status.addReplica(replicaShard, state, shutdown, verbose, replicaUnassignedBufferTime); } } } @@ -438,11 +465,18 @@ public class ShardAllocationCounts { public SearchableSnapshotsState searchableSnapshotsState = new SearchableSnapshotsState(); final Map> diagnosisDefinitions = new HashMap<>(); - public void increment(ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose) { + public void increment( + ShardRouting routing, + ClusterState state, + NodesShutdownMetadata shutdowns, + boolean verbose, + TimeValue replicaUnassignedBufferTime + ) { boolean isNew = isUnassignedDueToNewInitialization(routing, state); boolean isRestarting = isUnassignedDueToTimelyRestart(routing, shutdowns); + long replicaUnassignedCutoffTime = Instant.now().toEpochMilli() - replicaUnassignedBufferTime.millis(); boolean allUnavailable = areAllShardsOfThisTypeUnavailable(routing, state) - && isNewlyCreatedAndInitializingReplica(routing, state) == false; + && isNewlyCreatedAndInitializingReplica(routing, state, replicaUnassignedCutoffTime) == false; if (allUnavailable) { indicesWithAllShardsUnavailable.add(routing.getIndexName()); } @@ -520,7 +554,7 @@ boolean areAllShardsOfThisTypeUnavailable(ShardRouting routing, ClusterState sta * (a newly created index having unassigned replicas for example), we don't want the cluster * to turn "unhealthy" for the tiny amount of time before the shards are allocated. */ - static boolean isNewlyCreatedAndInitializingReplica(ShardRouting routing, ClusterState state) { + static boolean isNewlyCreatedAndInitializingReplica(ShardRouting routing, ClusterState state, long replicaUnassignedCutoffTime) { if (routing.active()) { return false; } @@ -528,10 +562,15 @@ static boolean isNewlyCreatedAndInitializingReplica(ShardRouting routing, Cluste return false; } ShardRouting primary = state.routingTable().shardRoutingTable(routing.shardId()).primaryShard(); - if (primary.active()) { - return false; + if (primary.active() == false) { + return ClusterShardHealth.getInactivePrimaryHealth(primary) == ClusterHealthStatus.YELLOW; } - return ClusterShardHealth.getInactivePrimaryHealth(primary) == ClusterHealthStatus.YELLOW; + + Optional ui = Optional.ofNullable(routing.unassignedInfo()); + return ui.filter(info -> info.failedAllocations() == 0) + .filter(info -> info.lastAllocationStatus() != UnassignedInfo.AllocationStatus.DECIDERS_NO) + .filter(info -> info.unassignedTimeMillis() > replicaUnassignedCutoffTime) + .isPresent(); } private static boolean isUnassignedDueToTimelyRestart(ShardRouting routing, NodesShutdownMetadata shutdowns) { @@ -910,11 +949,17 @@ public ShardAllocationStatus(Metadata clusterMetadata) { } void addPrimary(ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose) { - primaries.increment(routing, state, shutdowns, verbose); + primaries.increment(routing, state, shutdowns, verbose, TimeValue.MINUS_ONE); } - void addReplica(ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose) { - replicas.increment(routing, state, shutdowns, verbose); + void addReplica( + ShardRouting routing, + ClusterState state, + NodesShutdownMetadata shutdowns, + boolean verbose, + TimeValue replicaUnassignedBufferTime + ) { + replicas.increment(routing, state, shutdowns, verbose, replicaUnassignedBufferTime); } void updateSearchableSnapshotsOfAvailableIndices() { diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index ec0edb2d07e5a..497028ef37c69 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -1095,8 +1095,23 @@ public T[] readOptionalArray(Writeable.Reader reader, IntFunction ar return readBoolean() ? readArray(reader, arraySupplier) : null; } + /** + * Reads a possibly-null value using the given {@link org.elasticsearch.common.io.stream.Writeable.Reader}. + * + * @see StreamOutput#writeOptionalWriteable + */ + // just an alias for readOptional() since we don't actually care whether T extends Writeable @Nullable public T readOptionalWriteable(Writeable.Reader reader) throws IOException { + return readOptional(reader); + } + + /** + * Reads a possibly-null value using the given {@link org.elasticsearch.common.io.stream.Writeable.Reader}. + * + * @see StreamOutput#writeOptional + */ + public T readOptional(Writeable.Reader reader) throws IOException { if (readBoolean()) { T t = reader.read(this); if (t == null) { diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index c65ae2e3463d4..5780885473b00 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -1015,6 +1015,12 @@ public void writeOptionalArray(@Nullable T[] array) throws writeOptionalArray(StreamOutput::writeWriteable, array); } + /** + * Writes a boolean value indicating whether the given object is {@code null}, followed by the object's serialization if it is not + * {@code null}. + * + * @see StreamInput#readOptionalWriteable + */ public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOException { if (writeable != null) { writeBoolean(true); @@ -1024,6 +1030,21 @@ public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOExcep } } + /** + * Writes a boolean value indicating whether the given object is {@code null}, followed by the object's serialization if it is not + * {@code null}. + * + * @see StreamInput#readOptional + */ + public void writeOptional(Writer writer, @Nullable T maybeItem) throws IOException { + if (maybeItem != null) { + writeBoolean(true); + writer.write(this, maybeItem); + } else { + writeBoolean(false); + } + } + /** * This method allow to use a method reference when writing collection elements such as * {@code out.writeMap(map, StreamOutput::writeString, StreamOutput::writeWriteable)} diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 8d9d8452b12bb..3c60d63f78991 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -55,6 +55,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.shards.ShardsAvailabilityHealthIndicatorService; import org.elasticsearch.cluster.service.ClusterApplierService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterService; @@ -598,6 +599,7 @@ public void apply(Settings value, Settings current, Settings previous) { MergePolicyConfig.DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT_SETTING, TransportService.ENABLE_STACK_OVERFLOW_AVOIDANCE, DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING, - DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING + DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING, + ShardsAvailabilityHealthIndicatorService.REPLICA_UNASSIGNED_BUFFER_TIME ); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index fa40c0316fdcc..608d88fdef664 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -221,7 +221,7 @@ static Collection getAllVersions() { return VERSION_IDS.values(); } - static final IntFunction VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(IndexVersions.class); + static final IntFunction VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(IndexVersions.class, LATEST_DEFINED.id()); // no instance private IndexVersions() {} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 7810fcdc64773..6dce9d6c7b86e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -32,7 +32,8 @@ public Set getFeatures() { IndexModeFieldMapper.QUERYING_INDEX_MODE, NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS, BooleanFieldMapper.BOOLEAN_DIMENSION, - ObjectMapper.SUBOBJECTS_AUTO + ObjectMapper.SUBOBJECTS_AUTO, + SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 908108bce31da..8d34d3188a388 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -38,6 +38,9 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final NodeFeature SYNTHETIC_SOURCE_FALLBACK = new NodeFeature("mapper.source.synthetic_source_fallback"); + public static final NodeFeature SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX = new NodeFeature( + "mapper.source.synthetic_source_stored_fields_advance_fix" + ); public static final String NAME = "_source"; public static final String RECOVERY_SOURCE_NAME = "_recovery_source"; diff --git a/server/src/main/java/org/elasticsearch/injection/Injector.java b/server/src/main/java/org/elasticsearch/injection/Injector.java new file mode 100644 index 0000000000000..03fcf18509fcc --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/Injector.java @@ -0,0 +1,314 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection; + +import org.elasticsearch.injection.api.Inject; +import org.elasticsearch.injection.spec.ExistingInstanceSpec; +import org.elasticsearch.injection.spec.InjectionSpec; +import org.elasticsearch.injection.spec.MethodHandleSpec; +import org.elasticsearch.injection.spec.ParameterSpec; +import org.elasticsearch.injection.step.InjectionStep; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Constructor; +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.stream.Stream; + +import static java.util.function.Predicate.not; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toCollection; +import static java.util.stream.Collectors.toMap; + +/** + * The main object for dependency injection. + *

    + * Allows the user to specify the requirements, then call {@link #inject} to create an object plus all its dependencies. + *

    + * Implementation note: this class itself contains logic for specifying the injection requirements; + * the actual injection operations are performed in other classes like {@link Planner} and {@link PlanInterpreter}, + */ +public final class Injector { + private static final Logger logger = LogManager.getLogger(Injector.class); + + /** + * The specifications supplied by the user, as opposed to those inferred by the injector. + */ + private final Map, InjectionSpec> seedSpecs; + + Injector(Map, InjectionSpec> seedSpecs) { + this.seedSpecs = seedSpecs; + } + + public static Injector create() { + return new Injector(new LinkedHashMap<>()); + } + + /** + * Instructs the injector to instantiate classToProcess + * in accordance with whatever annotations may be present on that class. + *

    + * There are only three ways the injector can find out that it must instantiate some class: + *

      + *
    1. + * This method + *
      2. + *
    2. + * The parameter passed to {@link #inject} + *
      3. + *
    3. + * A constructor parameter of some other class being instantiated, + * having exactly the right class (not a supertype) + *
      4. + *
    + * + * @return this + */ + public Injector addClass(Class classToProcess) { + MethodHandleSpec methodHandleSpec = methodHandleSpecFor(classToProcess); + var existing = seedSpecs.put(classToProcess, methodHandleSpec); + if (existing != null) { + throw new IllegalArgumentException("class " + classToProcess.getSimpleName() + " has already been added"); + } + return this; + } + + /** + * Equivalent to multiple chained calls to {@link #addClass}. + */ + public Injector addClasses(Collection> classesToProcess) { + classesToProcess.forEach(this::addClass); + return this; + } + + /** + * Equivalent to {@link #addInstance addInstance(object.getClass(), object)}. + */ + public Injector addInstance(Object object) { + @SuppressWarnings("unchecked") + Class actualClass = (Class) object.getClass(); // Whatever the runtime type is, it's represented by T + return addInstance(actualClass, actualClass.cast(object)); + } + + /** + * Equivalent to multiple calls to {@link #addInstance(Object)}. + */ + public Injector addInstances(Collection objects) { + for (var x : objects) { + addInstance(x); + } + return this; + } + + /** + * Indicates that object is to be injected for parameters of type type. + * The given object is treated as though it had been instantiated by the injector. + */ + public Injector addInstance(Class type, T object) { + assert type.isInstance(object); // No unchecked casting shenanigans allowed + var existing = seedSpecs.put(type, new ExistingInstanceSpec(type, object)); + if (existing != null) { + throw new IllegalStateException("There's already an object for " + type); + } + return this; + } + + /** + * Main entry point. Causes objects to be constructed. + * @return {@link Map} whose keys are all the requested resultTypes and whose values are all the instances of those types. + */ + public Map, Object> inject(Collection> resultTypes) { + resultTypes.forEach(this::ensureClassIsSpecified); + PlanInterpreter i = doInjection(); + return resultTypes.stream().collect(toMap(c -> c, i::theInstanceOf)); + } + + private void ensureClassIsSpecified(Class resultType) { + if (seedSpecs.containsKey(resultType) == false) { + addClass(resultType); + } + } + + private PlanInterpreter doInjection() { + logger.debug("Starting injection"); + Map, InjectionSpec> specMap = specClosure(seedSpecs); + Map, Object> existingInstances = new LinkedHashMap<>(); + specMap.values().forEach((spec) -> { + if (spec instanceof ExistingInstanceSpec e) { + existingInstances.put(e.requestedType(), e.instance()); + } + }); + PlanInterpreter interpreter = new PlanInterpreter(existingInstances); + interpreter.executePlan(injectionPlan(seedSpecs.keySet(), specMap)); + logger.debug("Done injection"); + return interpreter; + } + + /** + * Finds an {@link InjectionSpec} for every class the injector is capable of injecting. + *

    + * We do this once the injector is fully configured, with all calls to {@link #addClass} and {@link #addInstance} finished, + * so that we can easily build the complete picture of how injection should occur. + *

    + * This is not part of the planning process; it's just discovering all the things + * the injector needs to know about. This logic isn't concerned with ordering or dependency cycles. + * + * @param seedMap the injections the user explicitly asked for + * @return an {@link InjectionSpec} for every class the injector is capable of injecting. + */ + private static Map, InjectionSpec> specClosure(Map, InjectionSpec> seedMap) { + assert seedMapIsValid(seedMap); + + // For convenience, we pretend there's a gigantic method out there that takes + // all the seed types as parameters. + Queue workQueue = seedMap.values() + .stream() + .map(InjectionSpec::requestedType) + .map(Injector::syntheticParameterSpec) + .collect(toCollection(ArrayDeque::new)); + + // This map doubles as a checklist of classes we're already finished processing + Map, InjectionSpec> result = new LinkedHashMap<>(); + + ParameterSpec p; + while ((p = workQueue.poll()) != null) { + Class c = p.injectableType(); + InjectionSpec existingResult = result.get(c); + if (existingResult != null) { + logger.trace("Spec for {} already exists", c.getSimpleName()); + continue; + } + + InjectionSpec spec = seedMap.get(c); + if (spec instanceof ExistingInstanceSpec) { + // simple! + result.put(c, spec); + continue; + } + + // At this point, we know we'll need a MethodHandleSpec + MethodHandleSpec methodHandleSpec; + if (spec == null) { + // The user didn't specify this class; we must infer it now + spec = methodHandleSpec = methodHandleSpecFor(c); + } else if (spec instanceof MethodHandleSpec m) { + methodHandleSpec = m; + } else { + throw new AssertionError("Unexpected spec: " + spec); + } + + logger.trace("Inspecting parameters for constructor of {}", c); + for (var ps : methodHandleSpec.parameters()) { + logger.trace("Enqueue {}", ps); + workQueue.add(ps); + } + + registerSpec(spec, result); + } + + if (logger.isTraceEnabled()) { + logger.trace("Specs: {}", result.values().stream().map(Object::toString).collect(joining("\n\t", "\n\t", ""))); + } + return result; + } + + private static MethodHandleSpec methodHandleSpecFor(Class c) { + Constructor constructor = getSuitableConstructorIfAny(c); + if (constructor == null) { + throw new IllegalStateException("No suitable constructor for " + c); + } + + MethodHandle ctorHandle; + try { + ctorHandle = lookup().unreflectConstructor(constructor); + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } + + List parameters = Stream.of(constructor.getParameters()).map(ParameterSpec::from).toList(); + + return new MethodHandleSpec(c, ctorHandle, parameters); + } + + /** + * @return true (unless an assertion fails). Never returns false. + */ + private static boolean seedMapIsValid(Map, InjectionSpec> seed) { + seed.forEach( + (c, s) -> { assert s.requestedType().equals(c) : "Spec must be associated with its requestedType, not " + c + ": " + s; } + ); + return true; + } + + /** + * For the classes we've been explicitly asked to inject, + * pretend there's some massive method taking all of them as parameters + */ + private static ParameterSpec syntheticParameterSpec(Class c) { + return new ParameterSpec("synthetic_" + c.getSimpleName(), c, c); + } + + private static Constructor getSuitableConstructorIfAny(Class type) { + var constructors = Stream.of(type.getConstructors()).filter(not(Constructor::isSynthetic)).toList(); + if (constructors.size() == 1) { + return constructors.get(0); + } + var injectConstructors = constructors.stream().filter(c -> c.isAnnotationPresent(Inject.class)).toList(); + if (injectConstructors.size() == 1) { + return injectConstructors.get(0); + } + logger.trace("No suitable constructor for {}", type); + return null; + } + + private static void registerSpec(InjectionSpec spec, Map, InjectionSpec> specsByClass) { + Class requestedType = spec.requestedType(); + var existing = specsByClass.put(requestedType, spec); + if (existing == null || existing.equals(spec)) { + logger.trace("Register spec: {}", spec); + } else { + throw new IllegalStateException("Ambiguous specifications for " + requestedType + ": " + existing + " and " + spec); + } + } + + private List injectionPlan(Set> requiredClasses, Map, InjectionSpec> specsByClass) { + logger.trace("Constructing instantiation plan"); + Set> allParameterTypes = new HashSet<>(); + specsByClass.values().forEach(spec -> { + if (spec instanceof MethodHandleSpec m) { + m.parameters().stream().map(ParameterSpec::injectableType).forEachOrdered(allParameterTypes::add); + } + }); + + var plan = new Planner(specsByClass, requiredClasses, allParameterTypes).injectionPlan(); + if (logger.isDebugEnabled()) { + logger.debug("Injection plan: {}", plan.stream().map(Object::toString).collect(joining("\n\t", "\n\t", ""))); + } + return plan; + } + + /** + * Evolution note: there may be cases in the where we allow the user to + * supply a {@link java.lang.invoke.MethodHandles.Lookup} for convenience, + * so that they aren't required to make things public just to participate in injection. + */ + private static MethodHandles.Lookup lookup() { + return MethodHandles.publicLookup(); + } + +} diff --git a/server/src/main/java/org/elasticsearch/injection/PlanInterpreter.java b/server/src/main/java/org/elasticsearch/injection/PlanInterpreter.java new file mode 100644 index 0000000000000..cf38dbcb24b7d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/PlanInterpreter.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection; + +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.injection.spec.MethodHandleSpec; +import org.elasticsearch.injection.spec.ParameterSpec; +import org.elasticsearch.injection.step.InjectionStep; +import org.elasticsearch.injection.step.InstantiateStep; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Performs the actual injection operations by running the {@link InjectionStep}s. + *

    + * The intent is that this logic is as simple as possible so that we don't run complex injection + * logic alongside the user-supplied constructor logic. All the injector complexity is already + * supposed to have happened in the planning phase. In particular, no injection-related errors + * are supposed to be detected during execution; they should be detected during planning and validation. + * All exceptions thrown during execution are supposed to be caused by user-supplied code. + * + *

    + * Execution model: + * The state of the injector during injection comprises a map from classes to objects. + * Before any steps execute, the map is pre-populated by object instances added via + * {@link Injector#addInstance(Object)} Injector.addInstance}, + * and then the steps begin to execute, reading and writing from this map. + * Some steps create objects and add them to this map; others manipulate the map itself. + */ +final class PlanInterpreter { + private static final Logger logger = LogManager.getLogger(PlanInterpreter.class); + private final Map, Object> instances = new LinkedHashMap<>(); + + PlanInterpreter(Map, Object> existingInstances) { + existingInstances.forEach(this::addInstance); + } + + /** + * Main entry point. Contains the implementation logic for each {@link InjectionStep}. + */ + void executePlan(List plan) { + int numConstructorCalls = 0; + for (InjectionStep step : plan) { + if (step instanceof InstantiateStep i) { + MethodHandleSpec spec = i.spec(); + logger.trace("Instantiating {}", spec.requestedType().getSimpleName()); + addInstance(spec.requestedType(), instantiate(spec)); + ++numConstructorCalls; + } else { + // TODO: switch patterns would make this unnecessary + assert false : "Unexpected step type: " + step.getClass().getSimpleName(); + throw new IllegalStateException("Unexpected step type: " + step.getClass().getSimpleName()); + } + } + logger.debug("Instantiated {} objects", numConstructorCalls); + } + + /** + * @return the list element corresponding to instances.get(type).get(0), + * assuming that instances.get(type) has exactly one element. + * @throws IllegalStateException if instances.get(type) does not have exactly one element + */ + public T theInstanceOf(Class type) { + Object instance = instances.get(type); + if (instance == null) { + throw new IllegalStateException("No object of type " + type.getSimpleName()); + } + return type.cast(instance); + } + + private void addInstance(Class requestedType, Object instance) { + Object old = instances.put(requestedType, instance); + if (old != null) { + throw new IllegalStateException("Multiple objects for " + requestedType); + } + } + + /** + * @throws IllegalStateException if the MethodHandle throws. + */ + @SuppressForbidden( + reason = "Can't call invokeExact because we don't know the method argument types statically, " + + "since each constructor has a different signature" + ) + private Object instantiate(MethodHandleSpec spec) { + Object[] args = spec.parameters().stream().map(this::parameterValue).toArray(); + try { + return spec.methodHandle().invokeWithArguments(args); + } catch (Throwable e) { + throw new IllegalStateException("Unexpected exception while instantiating {}" + spec, e); + } + } + + private Object parameterValue(ParameterSpec parameterSpec) { + return theInstanceOf(parameterSpec.formalType()); + } + +} diff --git a/server/src/main/java/org/elasticsearch/injection/Planner.java b/server/src/main/java/org/elasticsearch/injection/Planner.java new file mode 100644 index 0000000000000..4b6af05d57c04 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/Planner.java @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection; + +import org.elasticsearch.injection.spec.ExistingInstanceSpec; +import org.elasticsearch.injection.spec.InjectionSpec; +import org.elasticsearch.injection.spec.MethodHandleSpec; +import org.elasticsearch.injection.step.InjectionStep; +import org.elasticsearch.injection.step.InstantiateStep; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import static java.util.Collections.unmodifiableMap; +import static java.util.Collections.unmodifiableSet; + +/** + * Evolution note: the intent is to plan one domain/subsystem at a time. + */ +final class Planner { + private static final Logger logger = LogManager.getLogger(Planner.class); + + final List plan; + final Map, InjectionSpec> specsByClass; + final Set> requiredTypes; // The injector's job is to ensure there is an instance of these; this is like the "root set" + final Set> allParameterTypes; // All the injectable types in all dependencies (recursively) of all required types + final Set startedPlanning; + final Set finishedPlanning; + final Set> alreadyProxied; + + /** + * @param specsByClass an {@link InjectionSpec} indicating how each class should be injected + * @param requiredTypes the classes of which we need instances + * @param allParameterTypes the classes that appear as the type of any parameter of any constructor we might call + */ + Planner(Map, InjectionSpec> specsByClass, Set> requiredTypes, Set> allParameterTypes) { + this.requiredTypes = requiredTypes; + this.plan = new ArrayList<>(); + this.specsByClass = unmodifiableMap(specsByClass); + this.allParameterTypes = unmodifiableSet(allParameterTypes); + this.startedPlanning = new HashSet<>(); + this.finishedPlanning = new HashSet<>(); + this.alreadyProxied = new HashSet<>(); + } + + /** + * Intended to be called once. + *

    + * Note that not all proxies are resolved once this plan has been executed. + *

    + * + * Evolution note: in a world with multiple domains/subsystems, + * it will become necessary to defer proxy resolution until after other plans + * have been executed, because they could create additional objects that ought + * to be included in the proxies created by this plan. + * + * @return the {@link InjectionStep} objects listed in execution order. + */ + List injectionPlan() { + for (Class c : requiredTypes) { + planForClass(c, 0); + } + return plan; + } + + /** + * Recursive procedure that determines what effect requestedClass + * should have on the plan under construction. + * + * @param depth is used just for indenting the logs + */ + private void planForClass(Class requestedClass, int depth) { + InjectionSpec spec = specsByClass.get(requestedClass); + if (spec == null) { + throw new IllegalStateException("Cannot instantiate " + requestedClass + ": no specification provided"); + } + planForSpec(spec, depth); + } + + private void planForSpec(InjectionSpec spec, int depth) { + if (finishedPlanning.contains(spec)) { + logger.trace("{}Already planned {}", indent(depth), spec); + return; + } + + logger.trace("{}Planning for {}", indent(depth), spec); + if (startedPlanning.add(spec) == false) { + // TODO: Better cycle detection and reporting. Use SCCs + throw new IllegalStateException("Cyclic dependency involving " + spec); + } + + if (spec instanceof MethodHandleSpec m) { + for (var p : m.parameters()) { + logger.trace("{}- Recursing into {} for actual parameter {}", indent(depth), p.injectableType(), p); + planForClass(p.injectableType(), depth + 1); + } + addStep(new InstantiateStep(m), depth); + } else if (spec instanceof ExistingInstanceSpec e) { + logger.trace("{}- Plan {}", indent(depth), e); + // Nothing to do. The injector will already have the required object. + } else { + throw new AssertionError("Unexpected injection spec: " + spec); + } + + finishedPlanning.add(spec); + } + + private void addStep(InjectionStep newStep, int depth) { + logger.trace("{}- Add step {}", indent(depth), newStep); + plan.add(newStep); + } + + private static Supplier indent(int depth) { + return () -> "\t".repeat(depth); + } +} diff --git a/server/src/main/java/org/elasticsearch/injection/api/Inject.java b/server/src/main/java/org/elasticsearch/injection/api/Inject.java new file mode 100644 index 0000000000000..d5c57d1e5e2e2 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/api/Inject.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection.api; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * Designates a constructor to be called by the injector. + */ +@Target(CONSTRUCTOR) +@Retention(RUNTIME) +public @interface Inject { +} diff --git a/server/src/main/java/org/elasticsearch/injection/package-info.java b/server/src/main/java/org/elasticsearch/injection/package-info.java new file mode 100644 index 0000000000000..01dd1e878651c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/package-info.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * Our dependency injection technologies: our bespoke injector, plus our legacy vendored version of Google Guice. + *

    Usage

    + * The new injector is {@link org.elasticsearch.injection.Injector}. + * You create an instance using {@link org.elasticsearch.injection.Injector#create()}, + * call various methods like {@link org.elasticsearch.injection.Injector#addClass} to configure it, + * then call {@link org.elasticsearch.injection.Injector#inject} to cause the constructors to be called. + * + *

    Operation

    + * Injection proceeds in three phases: + *
      + *
    1. + * Configuration: the {@link org.elasticsearch.injection.Injector} captures the user's + * intent in the form of {@link org.elasticsearch.injection.spec.InjectionSpec} objects, + * one for each class. + *
      2. + *
    2. + * Planning: the {@link org.elasticsearch.injection.Planner} analyzes the + * {@link org.elasticsearch.injection.spec.InjectionSpec} objects, validates them, + * and generates a plan in the form of a list of {@link org.elasticsearch.injection.step.InjectionStep} objects. + *
      3. + *
    3. + * Execution: the {@link org.elasticsearch.injection.PlanInterpreter} runs + * the steps in the plan, in sequence, to actually instantiate the objects and pass them + * to each others' constructors. + *
      4. + *
    + * + *

    Google Guice

    + * The older injector, based on Google Guice, is in the {@code guice} package. + * The new injector is unrelated to Guice, and is intended to replace Guice eventually. + */ +package org.elasticsearch.injection; diff --git a/server/src/main/java/org/elasticsearch/injection/spec/ExistingInstanceSpec.java b/server/src/main/java/org/elasticsearch/injection/spec/ExistingInstanceSpec.java new file mode 100644 index 0000000000000..f443e045442c9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/spec/ExistingInstanceSpec.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection.spec; + +public record ExistingInstanceSpec(Class requestedType, Object instance) implements InjectionSpec { + @Override + public String toString() { + // Don't call instance.toString; who knows what that will return + return "ExistingInstanceSpec[" + "requestedType=" + requestedType + ']'; + } +} diff --git a/server/src/main/java/org/elasticsearch/injection/spec/InjectionSpec.java b/server/src/main/java/org/elasticsearch/injection/spec/InjectionSpec.java new file mode 100644 index 0000000000000..552d2c2ba9ebb --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/spec/InjectionSpec.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection.spec; + +public sealed interface InjectionSpec permits MethodHandleSpec, ExistingInstanceSpec { + Class requestedType(); +} diff --git a/server/src/main/java/org/elasticsearch/injection/spec/MethodHandleSpec.java b/server/src/main/java/org/elasticsearch/injection/spec/MethodHandleSpec.java new file mode 100644 index 0000000000000..06c4cd0faac63 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/spec/MethodHandleSpec.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection.spec; + +import java.lang.invoke.MethodHandle; +import java.util.List; +import java.util.Objects; + +/** + * Indicates that a type should be instantiated by calling the given {@link java.lang.invoke.MethodHandle}. + *

    + * Design note: the intent is that the semantics are fully specified by this record, + * and no additional reflection logic is required to determine how the object should be injected. + * Roughly speaking: all the reflection should be finished, and the results should be stored in this object. + */ +public record MethodHandleSpec(Class requestedType, MethodHandle methodHandle, List parameters) implements InjectionSpec { + public MethodHandleSpec { + assert Objects.equals(methodHandle.type().parameterList(), parameters.stream().map(ParameterSpec::formalType).toList()) + : "MethodHandle parameter types must match the supplied parameter info; " + + methodHandle.type().parameterList() + + " vs " + + parameters; + } +} diff --git a/server/src/main/java/org/elasticsearch/injection/spec/ParameterSpec.java b/server/src/main/java/org/elasticsearch/injection/spec/ParameterSpec.java new file mode 100644 index 0000000000000..da15bd024fbf4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/spec/ParameterSpec.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection.spec; + +import java.lang.reflect.Parameter; + +/** + * Captures the pertinent info required to inject one of the arguments of a constructor. + * @param name is for troubleshooting; it's not strictly needed + * @param formalType is the declared class of the parameter + * @param injectableType is the target type of the injection dependency + */ +public record ParameterSpec(String name, Class formalType, Class injectableType) { + public static ParameterSpec from(Parameter parameter) { + // We currently have no cases where the formal and injectable types are different. + return new ParameterSpec(parameter.getName(), parameter.getType(), parameter.getType()); + } +} diff --git a/server/src/main/java/org/elasticsearch/injection/spec/package-info.java b/server/src/main/java/org/elasticsearch/injection/spec/package-info.java new file mode 100644 index 0000000000000..26cb1e8ff8543 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/spec/package-info.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * Objects that describe the means by which an object instance is created for (or associated with) some given type. + *

    + * The hierarchy is rooted at {@link org.elasticsearch.injection.spec.InjectionSpec}. + *

    + * Differs from {@link org.elasticsearch.injection.step.InjectionStep InjectionStep} in that: + * + *

      + *
    • + * this describes the requirements, while InjectionStep describes the solution + *
      • + *
    • + * this is declarative, while InjectionStep is imperative + *
      • + *
    + */ +package org.elasticsearch.injection.spec; diff --git a/server/src/main/java/org/elasticsearch/injection/step/InjectionStep.java b/server/src/main/java/org/elasticsearch/injection/step/InjectionStep.java new file mode 100644 index 0000000000000..6e27f45b4f4df --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/step/InjectionStep.java @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection.step; + +public sealed interface InjectionStep permits InstantiateStep {} diff --git a/server/src/main/java/org/elasticsearch/injection/step/InstantiateStep.java b/server/src/main/java/org/elasticsearch/injection/step/InstantiateStep.java new file mode 100644 index 0000000000000..2342978dcfdb0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/step/InstantiateStep.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection.step; + +import org.elasticsearch.injection.spec.MethodHandleSpec; + +/** + * Constructs a new object by invoking a {@link java.lang.invoke.MethodHandle} + * as specified by a given {@link MethodHandleSpec}. + */ +public record InstantiateStep(MethodHandleSpec spec) implements InjectionStep {} diff --git a/server/src/main/java/org/elasticsearch/injection/step/package-info.java b/server/src/main/java/org/elasticsearch/injection/step/package-info.java new file mode 100644 index 0000000000000..c0a3e05cb53f6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/injection/step/package-info.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * Objects that describe one operation to be performed by the PlanInterpreter. + * Injection is achieved by executing the steps in order. + *

    + * See PlanInterpreter for more details on the execution model. + */ +package org.elasticsearch.injection.step; diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index ec0d293dc0064..eb9ef08b329ab 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -80,6 +80,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.discovery.DiscoveryModule; @@ -216,6 +217,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.IdentityHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -228,6 +230,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.lang.invoke.MethodHandles.lookup; +import static java.util.Collections.newSetFromMap; +import static java.util.function.Predicate.not; import static org.elasticsearch.core.Types.forciblyCast; /** @@ -831,27 +836,6 @@ private void construct( metadataCreateIndexService ); - record PluginServiceInstances( - Client client, - ClusterService clusterService, - RerouteService rerouteService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - RepositoriesService repositoriesService, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService, - FeatureService featureService, - SystemIndices systemIndices, - DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings, - DocumentParsingProvider documentParsingProvider - ) implements Plugin.PluginServices {} PluginServiceInstances pluginServices = new PluginServiceInstances( client, clusterService, @@ -874,7 +858,30 @@ record PluginServiceInstances( documentParsingProvider ); - Collection pluginComponents = pluginsService.flatMap(p -> p.createComponents(pluginServices)).toList(); + Collection pluginComponents = pluginsService.flatMap(plugin -> { + Collection allItems = plugin.createComponents(pluginServices); + List componentObjects = allItems.stream().filter(not(x -> x instanceof Class)).toList(); + List> classes = allItems.stream().filter(x -> x instanceof Class).map(x -> (Class) x).toList(); + + // Then, injection + Collection componentsFromInjector; + if (classes.isEmpty()) { + componentsFromInjector = Set.of(); + } else { + logger.debug("Using injector to instantiate classes for {}: {}", plugin.getClass().getSimpleName(), classes); + var injector = org.elasticsearch.injection.Injector.create(); + injector.addInstances(componentObjects); + addRecordContents(injector, pluginServices); + var resultMap = injector.inject(classes); + // For now, assume we want all components added to the Guice injector + var distinctObjects = newSetFromMap(new IdentityHashMap<>()); + distinctObjects.addAll(resultMap.values()); + componentsFromInjector = distinctObjects; + } + + // Return both + return Stream.of(componentObjects, componentsFromInjector).flatMap(Collection::stream).toList(); + }).toList(); var terminationHandlers = pluginsService.loadServiceProviders(TerminationHandlerProvider.class) .stream() @@ -1175,6 +1182,24 @@ record PluginServiceInstances( postInjection(clusterModule, actionModule, clusterService, transportService, featureService); } + /** + * For each "component" (getter) c of a {@link Record}, + * calls {@link org.elasticsearch.injection.Injector#addInstance(Object) Injector.addInstance} + * to register the value with the component's declared type. + */ + @SuppressForbidden(reason = "Can't call invokeExact because we don't know the exact Record subtype statically") + private static void addRecordContents(org.elasticsearch.injection.Injector injector, Record r) { + for (var c : r.getClass().getRecordComponents()) { + try { + @SuppressWarnings("unchecked") + Class type = (Class) c.getType(); // T represents the declared type of the record component, whatever it is + injector.addInstance(type, type.cast(lookup().unreflect(c.getAccessor()).invoke(r))); + } catch (Throwable e) { + throw new IllegalStateException("Unable to read record component " + c, e); + } + } + } + private ClusterService createClusterService(SettingsModule settingsModule, ThreadPool threadPool, TaskManager taskManager) { ClusterService clusterService = new ClusterService( settingsModule.getSettings(), @@ -1595,4 +1620,5 @@ private Module loadPersistentTasksService( b.bind(PersistentTasksClusterService.class).toInstance(persistentTasksClusterService); }; } + } diff --git a/server/src/main/java/org/elasticsearch/node/PluginServiceInstances.java b/server/src/main/java/org/elasticsearch/node/PluginServiceInstances.java new file mode 100644 index 0000000000000..7c8775502fd64 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/node/PluginServiceInstances.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.node; + +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.routing.RerouteService; +import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.internal.DocumentParsingProvider; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.telemetry.TelemetryProvider; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xcontent.NamedXContentRegistry; + +public record PluginServiceInstances( + Client client, + ClusterService clusterService, + RerouteService rerouteService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + RepositoriesService repositoriesService, + TelemetryProvider telemetryProvider, + AllocationService allocationService, + IndicesService indicesService, + FeatureService featureService, + SystemIndices systemIndices, + DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings, + DocumentParsingProvider documentParsingProvider +) implements Plugin.PluginServices {} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java index 2b5bcd9931f6e..528c37de7a4a8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java @@ -72,8 +72,8 @@ private static Bucket createFromStream(StreamInput in, DocValueFormat format, bo String key = in.getTransportVersion().equals(TransportVersions.V_8_0_0) ? in.readString() : in.getTransportVersion().onOrAfter(TransportVersions.V_7_17_1) ? in.readOptionalString() : in.readString(); - BytesRef from = in.readBoolean() ? in.readBytesRef() : null; - BytesRef to = in.readBoolean() ? in.readBytesRef() : null; + BytesRef from = in.readOptional(StreamInput::readBytesRef); + BytesRef to = in.readOptional(StreamInput::readBytesRef); long docCount = in.readLong(); InternalAggregations aggregations = InternalAggregations.readFrom(in); @@ -89,14 +89,8 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeString(key == null ? generateKey(from, to, format) : key); } - out.writeBoolean(from != null); - if (from != null) { - out.writeBytesRef(from); - } - out.writeBoolean(to != null); - if (to != null) { - out.writeBytesRef(to); - } + out.writeOptional(StreamOutput::writeBytesRef, from); + out.writeOptional(StreamOutput::writeBytesRef, to); out.writeLong(docCount); aggregations.writeTo(out); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java index 7516ab93f75a5..6f38669edf716 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java @@ -8,12 +8,13 @@ package org.elasticsearch.search.lookup; -import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import java.io.IOException; +import java.util.Map; // NB This is written under the assumption that individual segments are accessed by a single // thread, even if separate segments may be searched concurrently. If we ever implement @@ -21,7 +22,7 @@ class StoredFieldSourceProvider implements SourceProvider { private final StoredFieldLoader storedFieldLoader; - private volatile LeafStoredFieldSourceProvider[] leaves; + private final Map leaves = ConcurrentCollections.newConcurrentMap(); StoredFieldSourceProvider(StoredFieldLoader storedFieldLoader) { this.storedFieldLoader = storedFieldLoader; @@ -29,32 +30,14 @@ class StoredFieldSourceProvider implements SourceProvider { @Override public Source getSource(LeafReaderContext ctx, int doc) throws IOException { - LeafStoredFieldSourceProvider[] leaves = getLeavesUnderLock(findParentContext(ctx)); - if (leaves[ctx.ord] == null) { - // individual segments are currently only accessed on one thread so there's no need - // for locking here. - leaves[ctx.ord] = new LeafStoredFieldSourceProvider(storedFieldLoader.getLoader(ctx, null)); + final Object id = ctx.id(); + var provider = leaves.get(id); + if (provider == null) { + provider = new LeafStoredFieldSourceProvider(storedFieldLoader.getLoader(ctx, null)); + var existing = leaves.put(id, provider); + assert existing == null : "unexpected source provider [" + existing + "]"; } - return leaves[ctx.ord].getSource(doc); - } - - private static IndexReaderContext findParentContext(LeafReaderContext ctx) { - if (ctx.parent != null) { - return ctx.parent; - } - assert ctx.isTopLevel; - return ctx; - } - - private LeafStoredFieldSourceProvider[] getLeavesUnderLock(IndexReaderContext parentCtx) { - if (leaves == null) { - synchronized (this) { - if (leaves == null) { - leaves = new LeafStoredFieldSourceProvider[parentCtx.leaves().size()]; - } - } - } - return leaves; + return provider.getSource(doc); } private static class LeafStoredFieldSourceProvider { diff --git a/server/src/main/java/org/elasticsearch/usage/UsageService.java b/server/src/main/java/org/elasticsearch/usage/UsageService.java index e11b343c7055a..573332060f55d 100644 --- a/server/src/main/java/org/elasticsearch/usage/UsageService.java +++ b/server/src/main/java/org/elasticsearch/usage/UsageService.java @@ -9,6 +9,7 @@ package org.elasticsearch.usage; import org.elasticsearch.action.admin.cluster.node.usage.NodeUsage; +import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry; import org.elasticsearch.rest.BaseRestHandler; import java.util.HashMap; @@ -23,10 +24,12 @@ public class UsageService { private final Map handlers; private final SearchUsageHolder searchUsageHolder; + private final CCSUsageTelemetry ccsUsageHolder; public UsageService() { this.handlers = new HashMap<>(); this.searchUsageHolder = new SearchUsageHolder(); + this.ccsUsageHolder = new CCSUsageTelemetry(); } /** @@ -81,4 +84,8 @@ public Map getRestUsageStats() { public SearchUsageHolder getSearchUsageHolder() { return searchUsageHolder; } + + public CCSUsageTelemetry getCcsUsageHolder() { + return ccsUsageHolder; + } } diff --git a/server/src/test/java/org/elasticsearch/ReleaseVersionsTests.java b/server/src/test/java/org/elasticsearch/ReleaseVersionsTests.java index b80e953bd8aea..3b5f5eea57f66 100644 --- a/server/src/test/java/org/elasticsearch/ReleaseVersionsTests.java +++ b/server/src/test/java/org/elasticsearch/ReleaseVersionsTests.java @@ -17,19 +17,20 @@ public class ReleaseVersionsTests extends ESTestCase { public void testReleaseVersions() { - IntFunction versions = ReleaseVersions.generateVersionsLookup(ReleaseVersionsTests.class); + IntFunction versions = ReleaseVersions.generateVersionsLookup(ReleaseVersionsTests.class, 23); assertThat(versions.apply(10), equalTo("8.0.0")); assertThat(versions.apply(14), equalTo("8.1.0-8.1.1")); assertThat(versions.apply(21), equalTo("8.2.0")); assertThat(versions.apply(22), equalTo("8.2.1")); + assertThat(versions.apply(23), equalTo(Version.CURRENT.toString())); } public void testReturnsRange() { - IntFunction versions = ReleaseVersions.generateVersionsLookup(ReleaseVersionsTests.class); + IntFunction versions = ReleaseVersions.generateVersionsLookup(ReleaseVersionsTests.class, 23); assertThat(versions.apply(17), equalTo("8.1.2-8.2.0")); assertThat(versions.apply(9), equalTo("0.0.0")); - assertThat(versions.apply(24), equalTo("8.2.2-snapshot[24]")); + assertThat(versions.apply(24), equalTo(new Version(Version.CURRENT.id + 100) + "-[24]")); } } diff --git a/server/src/test/java/org/elasticsearch/TransportVersionTests.java b/server/src/test/java/org/elasticsearch/TransportVersionTests.java index 2de973622248b..a3728f20a23d4 100644 --- a/server/src/test/java/org/elasticsearch/TransportVersionTests.java +++ b/server/src/test/java/org/elasticsearch/TransportVersionTests.java @@ -186,6 +186,10 @@ public void testCURRENTIsLatest() { assertThat(Collections.max(TransportVersions.getAllVersions()), is(TransportVersion.current())); } + public void testToReleaseVersion() { + assertThat(TransportVersion.current().toReleaseVersion(), equalTo(Version.CURRENT.toString())); + } + public void testToString() { assertEquals("5000099", TransportVersion.fromId(5_00_00_99).toString()); assertEquals("2030099", TransportVersion.fromId(2_03_00_99).toString()); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ApproximateMatcher.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ApproximateMatcher.java new file mode 100644 index 0000000000000..3ceda1c7f4651 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ApproximateMatcher.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; + +/** + * Matches a value that is within given range (currently 1%) of an expected value. + * + * We need this because histograms do not store exact values, but only value ranges. + * Since we have 2 significant digits, the value should be within 1% of the expected value. + */ +public class ApproximateMatcher extends TypeSafeMatcher { + public static double ACCURACY = 0.01; + private final long expectedValue; + + public ApproximateMatcher(long expectedValue) { + this.expectedValue = expectedValue; + } + + @Override + protected boolean matchesSafely(Long actualValue) { + double lowerBound = Math.floor(expectedValue * (1.00 - ACCURACY)); + double upperBound = Math.ceil(expectedValue * (1.00 + ACCURACY)); + return actualValue >= lowerBound && actualValue <= upperBound; + } + + @Override + public void describeTo(Description description) { + description.appendText("a long value within 1% of ").appendValue(expectedValue); + } + + /** + * Matches a value that is within given range (currently 1%) of an expected value. + */ + public static ApproximateMatcher closeTo(long expectedValue) { + return new ApproximateMatcher(expectedValue); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java new file mode 100644 index 0000000000000..9f08934503b69 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java @@ -0,0 +1,324 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot.PerClusterCCSTelemetry; +import org.elasticsearch.action.admin.cluster.stats.LongMetric.LongMetricValue; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + +public class CCSTelemetrySnapshotTests extends AbstractWireSerializingTestCase { + + private LongMetricValue randomLongMetricValue() { + LongMetric v = new LongMetric(); + for (int i = 0; i < randomIntBetween(1, 10); i++) { + v.record(randomIntBetween(0, 1_000_000)); + } + return v.getValue(); + } + + private PerClusterCCSTelemetry randomPerClusterCCSTelemetry() { + return new PerClusterCCSTelemetry(randomLongBetween(0, 1_000_000), randomLongBetween(0, 1_000_000), randomLongMetricValue()); + } + + @Override + protected CCSTelemetrySnapshot createTestInstance() { + if (randomBoolean()) { + return new CCSTelemetrySnapshot(); + } else { + return randomCCSTelemetrySnapshot(); + } + } + + private CCSTelemetrySnapshot randomCCSTelemetrySnapshot() { + return new CCSTelemetrySnapshot( + randomLongBetween(0, 1_000_000), + randomLongBetween(0, 1_000_000), + Map.of(), + randomLongMetricValue(), + randomLongMetricValue(), + randomLongMetricValue(), + randomLongBetween(0, 1_000_000), + randomDoubleBetween(0.0, 100.0, false), + randomLongBetween(0, 1_000_000), + Map.of(), + Map.of(), + randomMap(1, 10, () -> new Tuple<>(randomAlphaOfLengthBetween(5, 10), randomPerClusterCCSTelemetry())) + ); + } + + @Override + protected Writeable.Reader instanceReader() { + return CCSTelemetrySnapshot::new; + } + + @Override + protected CCSTelemetrySnapshot mutateInstance(CCSTelemetrySnapshot instance) throws IOException { + // create a copy of CCSTelemetrySnapshot by extracting each field and mutating it + long totalCount = instance.getTotalCount(); + long successCount = instance.getSuccessCount(); + var failureReasons = instance.getFailureReasons(); + LongMetricValue took = instance.getTook(); + LongMetricValue tookMrtTrue = instance.getTookMrtTrue(); + LongMetricValue tookMrtFalse = instance.getTookMrtFalse(); + long skippedRemotes = instance.getSearchCountWithSkippedRemotes(); + long remotesPerSearchMax = instance.getRemotesPerSearchMax(); + double remotesPerSearchAvg = instance.getRemotesPerSearchAvg(); + var featureCounts = instance.getFeatureCounts(); + var clientCounts = instance.getClientCounts(); + var perClusterCCSTelemetries = instance.getByRemoteCluster(); + + // Mutate values + int i = randomInt(11); + switch (i) { + case 0: + totalCount += randomNonNegativeLong(); + break; + case 1: + successCount += randomNonNegativeLong(); + break; + case 2: + failureReasons = new HashMap<>(failureReasons); + if (failureReasons.isEmpty() || randomBoolean()) { + failureReasons.put(randomAlphaOfLengthBetween(5, 10), randomNonNegativeLong()); + } else { + // modify random element of the map + String key = randomFrom(failureReasons.keySet()); + failureReasons.put(key, randomNonNegativeLong()); + } + break; + case 3: + took = randomLongMetricValue(); + break; + case 4: + tookMrtTrue = randomLongMetricValue(); + break; + case 5: + tookMrtFalse = randomLongMetricValue(); + break; + case 6: + skippedRemotes += randomNonNegativeLong(); + break; + case 7: + remotesPerSearchMax += randomNonNegativeLong(); + break; + case 8: + remotesPerSearchAvg = randomDoubleBetween(0.0, 100.0, false); + break; + case 9: + featureCounts = new HashMap<>(featureCounts); + if (featureCounts.isEmpty() || randomBoolean()) { + featureCounts.put(randomAlphaOfLengthBetween(5, 10), randomNonNegativeLong()); + } else { + // modify random element of the map + String key = randomFrom(featureCounts.keySet()); + featureCounts.put(key, randomNonNegativeLong()); + } + break; + case 10: + clientCounts = new HashMap<>(clientCounts); + if (clientCounts.isEmpty() || randomBoolean()) { + clientCounts.put(randomAlphaOfLengthBetween(5, 10), randomNonNegativeLong()); + } else { + // modify random element of the map + String key = randomFrom(clientCounts.keySet()); + clientCounts.put(key, randomNonNegativeLong()); + } + break; + case 11: + perClusterCCSTelemetries = new HashMap<>(perClusterCCSTelemetries); + if (perClusterCCSTelemetries.isEmpty() || randomBoolean()) { + perClusterCCSTelemetries.put(randomAlphaOfLengthBetween(5, 10), randomPerClusterCCSTelemetry()); + } else { + // modify random element of the map + String key = randomFrom(perClusterCCSTelemetries.keySet()); + perClusterCCSTelemetries.put(key, randomPerClusterCCSTelemetry()); + } + break; + } + // Return new instance + return new CCSTelemetrySnapshot( + totalCount, + successCount, + failureReasons, + took, + tookMrtTrue, + tookMrtFalse, + remotesPerSearchMax, + remotesPerSearchAvg, + skippedRemotes, + featureCounts, + clientCounts, + perClusterCCSTelemetries + ); + } + + public void testAdd() { + CCSTelemetrySnapshot empty = new CCSTelemetrySnapshot(); + CCSTelemetrySnapshot full = randomCCSTelemetrySnapshot(); + empty.add(full); + assertThat(empty, equalTo(full)); + // Add again + empty.add(full); + assertThat(empty.getTotalCount(), equalTo(full.getTotalCount() * 2)); + assertThat(empty.getSuccessCount(), equalTo(full.getSuccessCount() * 2)); + // check that each element of the map is doubled + empty.getFailureReasons().forEach((k, v) -> assertThat(v, equalTo(full.getFailureReasons().get(k) * 2))); + assertThat(empty.getTook().count(), equalTo(full.getTook().count() * 2)); + assertThat(empty.getTookMrtTrue().count(), equalTo(full.getTookMrtTrue().count() * 2)); + assertThat(empty.getTookMrtFalse().count(), equalTo(full.getTookMrtFalse().count() * 2)); + assertThat(empty.getSearchCountWithSkippedRemotes(), equalTo(full.getSearchCountWithSkippedRemotes() * 2)); + assertThat(empty.getRemotesPerSearchMax(), equalTo(full.getRemotesPerSearchMax())); + assertThat(empty.getRemotesPerSearchAvg(), closeTo(full.getRemotesPerSearchAvg(), 0.01)); + empty.getFeatureCounts().forEach((k, v) -> assertThat(v, equalTo(full.getFeatureCounts().get(k) * 2))); + empty.getClientCounts().forEach((k, v) -> assertThat(v, equalTo(full.getClientCounts().get(k) * 2))); + empty.getByRemoteCluster().forEach((k, v) -> { + assertThat(v.getCount(), equalTo(full.getByRemoteCluster().get(k).getCount() * 2)); + assertThat(v.getSkippedCount(), equalTo(full.getByRemoteCluster().get(k).getSkippedCount() * 2)); + assertThat(v.getTook().count(), equalTo(full.getByRemoteCluster().get(k).getTook().count() * 2)); + }); + } + + public void testAddTwo() { + CCSTelemetrySnapshot empty = new CCSTelemetrySnapshot(); + CCSTelemetrySnapshot full = randomCCSTelemetrySnapshot(); + CCSTelemetrySnapshot full2 = randomCCSTelemetrySnapshot(); + + empty.add(full); + empty.add(full2); + assertThat(empty.getTotalCount(), equalTo(full.getTotalCount() + full2.getTotalCount())); + assertThat(empty.getSuccessCount(), equalTo(full.getSuccessCount() + full2.getSuccessCount())); + empty.getFailureReasons() + .forEach( + (k, v) -> assertThat( + v, + equalTo(full.getFailureReasons().getOrDefault(k, 0L) + full2.getFailureReasons().getOrDefault(k, 0L)) + ) + ); + assertThat(empty.getTook().count(), equalTo(full.getTook().count() + full2.getTook().count())); + assertThat(empty.getTookMrtTrue().count(), equalTo(full.getTookMrtTrue().count() + full2.getTookMrtTrue().count())); + assertThat(empty.getTookMrtFalse().count(), equalTo(full.getTookMrtFalse().count() + full2.getTookMrtFalse().count())); + assertThat( + empty.getSearchCountWithSkippedRemotes(), + equalTo(full.getSearchCountWithSkippedRemotes() + full2.getSearchCountWithSkippedRemotes()) + ); + assertThat(empty.getRemotesPerSearchMax(), equalTo(Math.max(full.getRemotesPerSearchMax(), full2.getRemotesPerSearchMax()))); + double expectedAvg = (full.getRemotesPerSearchAvg() * full.getTotalCount() + full2.getRemotesPerSearchAvg() * full2.getTotalCount()) + / empty.getTotalCount(); + assertThat(empty.getRemotesPerSearchAvg(), closeTo(expectedAvg, 0.01)); + empty.getFeatureCounts() + .forEach( + (k, v) -> assertThat(v, equalTo(full.getFeatureCounts().getOrDefault(k, 0L) + full2.getFeatureCounts().getOrDefault(k, 0L))) + ); + empty.getClientCounts() + .forEach( + (k, v) -> assertThat(v, equalTo(full.getClientCounts().getOrDefault(k, 0L) + full2.getClientCounts().getOrDefault(k, 0L))) + ); + PerClusterCCSTelemetry zeroDummy = new PerClusterCCSTelemetry(); + empty.getByRemoteCluster().forEach((k, v) -> { + assertThat( + v.getCount(), + equalTo( + full.getByRemoteCluster().getOrDefault(k, zeroDummy).getCount() + full2.getByRemoteCluster() + .getOrDefault(k, zeroDummy) + .getCount() + ) + ); + assertThat( + v.getSkippedCount(), + equalTo( + full.getByRemoteCluster().getOrDefault(k, zeroDummy).getSkippedCount() + full2.getByRemoteCluster() + .getOrDefault(k, zeroDummy) + .getSkippedCount() + ) + ); + assertThat( + v.getTook().count(), + equalTo( + full.getByRemoteCluster().getOrDefault(k, zeroDummy).getTook().count() + full2.getByRemoteCluster() + .getOrDefault(k, zeroDummy) + .getTook() + .count() + ) + ); + }); + } + + private LongMetricValue manyValuesHistogram(long startingWith) { + LongMetric metric = new LongMetric(); + // Produce 100 values from startingWith to 2 * startingWith with equal intervals + // We need to space values relative to initial value, otherwise the histogram would put them all in one bucket + for (long i = startingWith; i < 2 * startingWith; i += startingWith / 100) { + metric.record(i); + } + return metric.getValue(); + } + + public void testToXContent() throws IOException { + long totalCount = 10; + long successCount = 20; + // Using TreeMap's here to ensure consistent ordering in the JSON output + var failureReasons = new TreeMap<>(Map.of("reason1", 1L, "reason2", 2L, "unknown", 3L)); + LongMetricValue took = manyValuesHistogram(1000); + LongMetricValue tookMrtTrue = manyValuesHistogram(5000); + LongMetricValue tookMrtFalse = manyValuesHistogram(10000); + long skippedRemotes = 5; + long remotesPerSearchMax = 6; + double remotesPerSearchAvg = 7.89; + var featureCounts = new TreeMap<>(Map.of("async", 10L, "mrt", 20L, "wildcard", 30L)); + var clientCounts = new TreeMap<>(Map.of("kibana", 40L, "other", 500L)); + var perClusterCCSTelemetries = new TreeMap<>( + Map.of( + "", + new PerClusterCCSTelemetry(12, 0, manyValuesHistogram(2000)), + "remote1", + new PerClusterCCSTelemetry(100, 22, manyValuesHistogram(2000)), + "remote2", + new PerClusterCCSTelemetry(300, 42, manyValuesHistogram(500000)) + ) + ); + + var snapshot = new CCSTelemetrySnapshot( + totalCount, + successCount, + failureReasons, + took, + tookMrtTrue, + tookMrtFalse, + remotesPerSearchMax, + remotesPerSearchAvg, + skippedRemotes, + featureCounts, + clientCounts, + perClusterCCSTelemetries + ); + String expected = readJSONFromResource("telemetry_test.json"); + assertEquals(expected, snapshot.toString()); + } + + private String readJSONFromResource(String fileName) throws IOException { + try (InputStream inputStream = getClass().getResourceAsStream("/org/elasticsearch/action/admin/cluster/stats/" + fileName)) { + if (inputStream == null) { + throw new IOException("Resource not found: " + fileName); + } + return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java new file mode 100644 index 0000000000000..bd36f89f38e4d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java @@ -0,0 +1,342 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.action.admin.cluster.stats.ApproximateMatcher.closeTo; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.KNOWN_CLIENTS; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.MRT_FEATURE; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result.CANCELED; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.WILDCARD_FEATURE; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class CCSUsageTelemetryTests extends ESTestCase { + + public void testSuccessfulSearchResults() { + CCSUsageTelemetry ccsUsageHolder = new CCSUsageTelemetry(); + + long expectedAsyncCount = 0L; + long expectedMinRTCount = 0L; + long expectedSearchesWithSkippedRemotes = 0L; + long took1 = 0L; + long took1Remote1 = 0L; + + // first search + { + boolean minimizeRoundTrips = randomBoolean(); + boolean async = randomBoolean(); + took1 = randomLongBetween(5, 10000); + boolean skippedRemote = randomBoolean(); + expectedSearchesWithSkippedRemotes = skippedRemote ? 1 : 0; + expectedAsyncCount = async ? 1 : 0; + expectedMinRTCount = minimizeRoundTrips ? 1 : 0; + + // per cluster telemetry + long tookLocal = randomLongBetween(2, 8000); + took1Remote1 = randomLongBetween(2, 8000); + + CCSUsage.Builder builder = new CCSUsage.Builder(); + builder.took(took1).setRemotesCount(1); + if (async) { + builder.setFeature(ASYNC_FEATURE); + } + if (minimizeRoundTrips) { + builder.setFeature(MRT_FEATURE); + } + if (skippedRemote) { + builder.skippedRemote("remote1"); + } + builder.perClusterUsage("(local)", new TimeValue(tookLocal)); + builder.perClusterUsage("remote1", new TimeValue(took1Remote1)); + + CCSUsage ccsUsage = builder.build(); + ccsUsageHolder.updateUsage(ccsUsage); + + CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot(); + + assertThat(snapshot.getTotalCount(), equalTo(1L)); + assertThat(snapshot.getSuccessCount(), equalTo(1L)); + assertThat(snapshot.getFeatureCounts().getOrDefault(ASYNC_FEATURE, 0L), equalTo(expectedAsyncCount)); + assertThat(snapshot.getFeatureCounts().getOrDefault(MRT_FEATURE, 0L), equalTo(expectedMinRTCount)); + assertThat(snapshot.getSearchCountWithSkippedRemotes(), equalTo(expectedSearchesWithSkippedRemotes)); + assertThat(snapshot.getTook().avg(), greaterThan(0L)); + // Expect it to be within 1% of the actual value + assertThat(snapshot.getTook().avg(), closeTo(took1)); + assertThat(snapshot.getTook().max(), closeTo(took1)); + if (minimizeRoundTrips) { + assertThat(snapshot.getTookMrtTrue().count(), equalTo(1L)); + assertThat(snapshot.getTookMrtTrue().avg(), greaterThan(0L)); + assertThat(snapshot.getTookMrtTrue().avg(), closeTo(took1)); + assertThat(snapshot.getTookMrtFalse().count(), equalTo(0L)); + assertThat(snapshot.getTookMrtFalse().max(), equalTo(0L)); + } else { + assertThat(snapshot.getTookMrtFalse().count(), equalTo(1L)); + assertThat(snapshot.getTookMrtFalse().avg(), greaterThan(0L)); + assertThat(snapshot.getTookMrtFalse().avg(), closeTo(took1)); + assertThat(snapshot.getTookMrtTrue().count(), equalTo(0L)); + assertThat(snapshot.getTookMrtTrue().max(), equalTo(0L)); + } + // We currently don't count unknown clients + assertThat(snapshot.getClientCounts().size(), equalTo(0)); + + // per cluster telemetry asserts + + var telemetryByCluster = snapshot.getByRemoteCluster(); + assertThat(telemetryByCluster.size(), equalTo(2)); + var localClusterTelemetry = telemetryByCluster.get("(local)"); + assertNotNull(localClusterTelemetry); + assertThat(localClusterTelemetry.getCount(), equalTo(1L)); + assertThat(localClusterTelemetry.getSkippedCount(), equalTo(0L)); + assertThat(localClusterTelemetry.getTook().count(), equalTo(1L)); + assertThat(localClusterTelemetry.getTook().avg(), greaterThan(0L)); + assertThat(localClusterTelemetry.getTook().avg(), closeTo(tookLocal)); + // assertThat(localClusterTelemetry.getTook().max(), greaterThanOrEqualTo(tookLocal)); + + var remote1ClusterTelemetry = telemetryByCluster.get("remote1"); + assertNotNull(remote1ClusterTelemetry); + assertThat(remote1ClusterTelemetry.getCount(), equalTo(1L)); + assertThat(remote1ClusterTelemetry.getSkippedCount(), equalTo(expectedSearchesWithSkippedRemotes)); + assertThat(remote1ClusterTelemetry.getTook().avg(), greaterThan(0L)); + assertThat(remote1ClusterTelemetry.getTook().count(), equalTo(1L)); + assertThat(remote1ClusterTelemetry.getTook().avg(), greaterThan(0L)); + assertThat(remote1ClusterTelemetry.getTook().avg(), closeTo(took1Remote1)); + // assertThat(remote1ClusterTelemetry.getTook().max(), greaterThanOrEqualTo(took1Remote1)); + } + + // second search + { + boolean minimizeRoundTrips = randomBoolean(); + boolean async = randomBoolean(); + expectedAsyncCount += async ? 1 : 0; + expectedMinRTCount += minimizeRoundTrips ? 1 : 0; + long took2 = randomLongBetween(5, 10000); + boolean skippedRemote = randomBoolean(); + expectedSearchesWithSkippedRemotes += skippedRemote ? 1 : 0; + long took2Remote1 = randomLongBetween(2, 8000); + + CCSUsage.Builder builder = new CCSUsage.Builder(); + builder.took(took2).setRemotesCount(1).setClient("kibana"); + if (async) { + builder.setFeature(ASYNC_FEATURE); + } + if (minimizeRoundTrips) { + builder.setFeature(MRT_FEATURE); + } + if (skippedRemote) { + builder.skippedRemote("remote1"); + } + builder.perClusterUsage("remote1", new TimeValue(took2Remote1)); + + CCSUsage ccsUsage = builder.build(); + ccsUsageHolder.updateUsage(ccsUsage); + + CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot(); + + assertThat(snapshot.getTotalCount(), equalTo(2L)); + assertThat(snapshot.getSuccessCount(), equalTo(2L)); + assertThat(snapshot.getFeatureCounts().getOrDefault(ASYNC_FEATURE, 0L), equalTo(expectedAsyncCount)); + assertThat(snapshot.getFeatureCounts().getOrDefault(MRT_FEATURE, 0L), equalTo(expectedMinRTCount)); + assertThat(snapshot.getSearchCountWithSkippedRemotes(), equalTo(expectedSearchesWithSkippedRemotes)); + assertThat(snapshot.getTook().avg(), greaterThan(0L)); + assertThat(snapshot.getTook().avg(), closeTo((took1 + took2) / 2)); + // assertThat(snapshot.getTook().max(), greaterThanOrEqualTo(Math.max(took1, took2))); + + // Counting only known clients + assertThat(snapshot.getClientCounts().get("kibana"), equalTo(1L)); + assertThat(snapshot.getClientCounts().size(), equalTo(1)); + + // per cluster telemetry asserts + + var telemetryByCluster = snapshot.getByRemoteCluster(); + assertThat(telemetryByCluster.size(), equalTo(2)); + var localClusterTelemetry = telemetryByCluster.get("(local)"); + assertNotNull(localClusterTelemetry); + assertThat(localClusterTelemetry.getCount(), equalTo(1L)); + assertThat(localClusterTelemetry.getSkippedCount(), equalTo(0L)); + assertThat(localClusterTelemetry.getTook().count(), equalTo(1L)); + + var remote1ClusterTelemetry = telemetryByCluster.get("remote1"); + assertNotNull(remote1ClusterTelemetry); + assertThat(remote1ClusterTelemetry.getCount(), equalTo(2L)); + assertThat(remote1ClusterTelemetry.getSkippedCount(), equalTo(expectedSearchesWithSkippedRemotes)); + assertThat(remote1ClusterTelemetry.getTook().avg(), greaterThan(0L)); + assertThat(remote1ClusterTelemetry.getTook().count(), equalTo(2L)); + assertThat(remote1ClusterTelemetry.getTook().avg(), greaterThan(0L)); + assertThat(remote1ClusterTelemetry.getTook().avg(), closeTo((took1Remote1 + took2Remote1) / 2)); + // assertThat(remote1ClusterTelemetry.getTook().max(), greaterThanOrEqualTo(Math.max(took1Remote1, took2Remote1))); + } + } + + public void testClientsLimit() { + CCSUsageTelemetry ccsUsageHolder = new CCSUsageTelemetry(); + // Add known clients + for (String knownClient : KNOWN_CLIENTS) { + CCSUsage.Builder builder = new CCSUsage.Builder(); + builder.took(randomLongBetween(5, 10000)).setRemotesCount(1).setClient(knownClient); + CCSUsage ccsUsage = builder.build(); + ccsUsageHolder.updateUsage(ccsUsage); + } + var counts = ccsUsageHolder.getCCSTelemetrySnapshot().getClientCounts(); + for (String knownClient : KNOWN_CLIENTS) { + assertThat(counts.get(knownClient), equalTo(1L)); + } + // Check that knowns are counted + for (String knownClient : KNOWN_CLIENTS) { + CCSUsage.Builder builder = new CCSUsage.Builder(); + builder.took(randomLongBetween(5, 10000)).setRemotesCount(1).setClient(knownClient); + CCSUsage ccsUsage = builder.build(); + ccsUsageHolder.updateUsage(ccsUsage); + } + counts = ccsUsageHolder.getCCSTelemetrySnapshot().getClientCounts(); + for (String knownClient : KNOWN_CLIENTS) { + assertThat(counts.get(knownClient), equalTo(2L)); + } + // Check that new clients are not counted + CCSUsage.Builder builder = new CCSUsage.Builder(); + String randomClient = randomAlphaOfLength(10); + builder.took(randomLongBetween(5, 10000)).setRemotesCount(1).setClient(randomClient); + CCSUsage ccsUsage = builder.build(); + ccsUsageHolder.updateUsage(ccsUsage); + counts = ccsUsageHolder.getCCSTelemetrySnapshot().getClientCounts(); + assertThat(counts.get(randomClient), equalTo(null)); + } + + public void testFailures() { + CCSUsageTelemetry ccsUsageHolder = new CCSUsageTelemetry(); + + // first search + { + boolean skippedRemote = randomBoolean(); + boolean minimizeRoundTrips = randomBoolean(); + boolean async = randomBoolean(); + + CCSUsage.Builder builder = new CCSUsage.Builder(); + builder.setRemotesCount(1).took(10L); + if (skippedRemote) { + builder.skippedRemote("remote1"); + } + builder.perClusterUsage("(local)", new TimeValue(1)); + builder.perClusterUsage("remote1", new TimeValue(2)); + builder.setFailure(CANCELED); + if (async) { + builder.setFeature(ASYNC_FEATURE); + } + if (minimizeRoundTrips) { + builder.setFeature(MRT_FEATURE); + } + + CCSUsage ccsUsage = builder.build(); + ccsUsageHolder.updateUsage(ccsUsage); + + CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot(); + + assertThat(snapshot.getTotalCount(), equalTo(1L)); + assertThat(snapshot.getSuccessCount(), equalTo(0L)); + assertThat(snapshot.getSearchCountWithSkippedRemotes(), equalTo(skippedRemote ? 1L : 0L)); + assertThat(snapshot.getTook().count(), equalTo(0L)); + assertThat(snapshot.getFailureReasons().size(), equalTo(1)); + assertThat(snapshot.getFailureReasons().get(CANCELED.getName()), equalTo(1L)); + // still counting features on failure + assertThat(snapshot.getFeatureCounts().getOrDefault(ASYNC_FEATURE, 0L), equalTo(async ? 1L : 0L)); + assertThat(snapshot.getFeatureCounts().getOrDefault(MRT_FEATURE, 0L), equalTo(minimizeRoundTrips ? 1L : 0L)); + } + + // second search + { + CCSUsage.Builder builder = new CCSUsage.Builder(); + boolean skippedRemote = randomBoolean(); + builder.setRemotesCount(1).took(10L).setClient("kibana"); + if (skippedRemote) { + builder.skippedRemote("remote1"); + } + builder.setFailure(CANCELED); + CCSUsage ccsUsage = builder.build(); + + ccsUsageHolder.updateUsage(ccsUsage); + + CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot(); + + assertThat(snapshot.getTotalCount(), equalTo(2L)); + assertThat(snapshot.getSuccessCount(), equalTo(0L)); + assertThat(snapshot.getTook().count(), equalTo(0L)); + assertThat(snapshot.getFailureReasons().size(), equalTo(1)); + assertThat(snapshot.getFailureReasons().get(CANCELED.getName()), equalTo(2L)); + assertThat(snapshot.getClientCounts().get("kibana"), equalTo(1L)); + } + } + + public void testConcurrentUpdates() throws InterruptedException { + CCSUsageTelemetry ccsUsageHolder = new CCSUsageTelemetry(); + CCSUsageTelemetry expectedHolder = new CCSUsageTelemetry(); + int numSearches = randomIntBetween(1000, 5000); + int numThreads = randomIntBetween(10, 20); + Thread[] threads = new Thread[numThreads]; + CCSUsage[] ccsUsages = new CCSUsage[numSearches]; + + // Make random usage objects + for (int i = 0; i < numSearches; i++) { + CCSUsage.Builder builder = new CCSUsage.Builder(); + builder.took(randomLongBetween(5, 10000)).setRemotesCount(randomIntBetween(1, 10)); + if (randomBoolean()) { + builder.setFeature(ASYNC_FEATURE); + } + if (randomBoolean()) { + builder.setFeature(WILDCARD_FEATURE); + } + if (randomBoolean()) { + builder.setFeature(MRT_FEATURE); + } + if (randomBoolean()) { + builder.setClient("kibana"); + } + if (randomInt(20) == 7) { + // 5% of requests will fail + builder.setFailure(randomFrom(CCSUsageTelemetry.Result.values())); + ccsUsages[i] = builder.build(); + continue; + } + builder.perClusterUsage("", new TimeValue(randomLongBetween(1, 10000))); + if (randomBoolean()) { + builder.skippedRemote("remote1"); + } else { + builder.perClusterUsage("remote1", new TimeValue(randomLongBetween(1, 10000))); + } + builder.perClusterUsage(randomFrom("remote2", "remote3", "remote4"), new TimeValue(randomLongBetween(1, 10000))); + ccsUsages[i] = builder.build(); + } + + // Add each of the search objects to the telemetry holder in a different thread + for (int i = 0; i < numThreads; i++) { + final int threadNo = i; + threads[i] = new Thread(() -> { + for (int j = threadNo; j < numSearches; j += numThreads) { + ccsUsageHolder.updateUsage(ccsUsages[j]); + } + }); + threads[i].start(); + } + + for (int i = 0; i < numThreads; i++) { + threads[i].join(); + } + + // Add the same search objects to the expected holder in a single thread + for (int i = 0; i < numSearches; i++) { + expectedHolder.updateUsage(ccsUsages[i]); + } + + CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot(); + CCSTelemetrySnapshot expectedSnapshot = ccsUsageHolder.getCCSTelemetrySnapshot(); + assertThat(snapshot, equalTo(expectedSnapshot)); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 487d8c6f3a7ee..f68e5f06bcf08 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -99,6 +99,7 @@ import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.usage.UsageService; import java.io.IOException; import java.util.ArrayList; @@ -1765,7 +1766,8 @@ protected void doWriteTo(StreamOutput out) throws IOException { null, new SearchTransportAPMMetrics(TelemetryProvider.NOOP.getMeterRegistry()), new SearchResponseMetrics(TelemetryProvider.NOOP.getMeterRegistry()), - client + client, + new UsageService() ); CountDownLatch latch = new CountDownLatch(1); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java index b731fd79c82fe..994e892e3ac3c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.cluster.routing.allocation.shards.ShardsAvailabilityHealthIndicatorService; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.test.ESTestCase; @@ -33,14 +34,17 @@ import static org.elasticsearch.cluster.routing.allocation.shards.ShardsAvailabilityHealthIndicatorService.TIER_CAPACITY_ACTION_GUIDE; import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ShardsAvailabilityActionGuideTests extends ESTestCase { - private final ShardsAvailabilityHealthIndicatorService service = new ShardsAvailabilityHealthIndicatorService( - mock(ClusterService.class), - mock(AllocationService.class), - mock(SystemIndices.class) - ); + private final ShardsAvailabilityHealthIndicatorService service; + + public ShardsAvailabilityActionGuideTests() { + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(ClusterSettings.createBuiltInClusterSettings()); + service = new ShardsAvailabilityHealthIndicatorService(clusterService, mock(AllocationService.class), mock(SystemIndices.class)); + } public void testRestoreFromSnapshotAction() { assertThat(ACTION_RESTORE_FROM_SNAPSHOT.helpURL(), is(RESTORE_FROM_SNAPSHOT_ACTION_GUIDE)); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java index 0e3041dda9853..ad30c79a01334 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java @@ -42,6 +42,8 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.health.Diagnosis; import org.elasticsearch.health.HealthIndicatorDetails; import org.elasticsearch.health.HealthIndicatorImpact; @@ -61,9 +63,9 @@ import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.mockito.Mockito; import org.mockito.stubbing.Answer; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -104,6 +106,7 @@ import static org.elasticsearch.cluster.routing.allocation.shards.ShardsAvailabilityHealthIndicatorServiceTests.ShardState.UNAVAILABLE; import static org.elasticsearch.common.util.CollectionUtils.concatLists; import static org.elasticsearch.core.TimeValue.timeValueSeconds; +import static org.elasticsearch.core.Tuple.tuple; import static org.elasticsearch.health.Diagnosis.Resource.Type.FEATURE_STATE; import static org.elasticsearch.health.Diagnosis.Resource.Type.INDEX; import static org.elasticsearch.health.HealthStatus.GREEN; @@ -337,7 +340,8 @@ public void testAllReplicasUnassigned() { status, clusterState, NodesShutdownMetadata.EMPTY, - randomBoolean() + randomBoolean(), + timeValueSeconds(0) ); assertFalse(status.replicas.doAnyIndicesHaveAllUnavailable()); } @@ -359,7 +363,8 @@ public void testAllReplicasUnassigned() { status, clusterState, NodesShutdownMetadata.EMPTY, - randomBoolean() + randomBoolean(), + timeValueSeconds(0) ); assertFalse(status.replicas.doAnyIndicesHaveAllUnavailable()); } @@ -381,7 +386,8 @@ public void testAllReplicasUnassigned() { status, clusterState, NodesShutdownMetadata.EMPTY, - randomBoolean() + randomBoolean(), + timeValueSeconds(0) ); assertTrue(status.replicas.doAnyIndicesHaveAllUnavailable()); } @@ -398,13 +404,15 @@ public void testAllReplicasUnassigned() { ), List.of() ); + var service = createShardsAvailabilityIndicatorService(clusterState); ShardAllocationStatus status = service.createNewStatus(clusterState.metadata()); ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus( status, clusterState, NodesShutdownMetadata.EMPTY, - randomBoolean() + randomBoolean(), + timeValueSeconds(0) ); assertTrue(status.replicas.doAnyIndicesHaveAllUnavailable()); } @@ -440,7 +448,8 @@ public void testAllReplicasUnassigned() { status, clusterState, NodesShutdownMetadata.EMPTY, - randomBoolean() + randomBoolean(), + timeValueSeconds(0) ); // Here because the replica is unassigned due to the primary being created, it's treated as though the replica can be ignored. assertFalse( @@ -469,7 +478,8 @@ public void testAllReplicasUnassigned() { status, clusterState, NodesShutdownMetadata.EMPTY, - randomBoolean() + randomBoolean(), + timeValueSeconds(0) ); var shardRouting = routingTable.shardsWithState(ShardRoutingState.UNASSIGNED).get(0); assertTrue(service.areAllShardsOfThisTypeUnavailable(shardRouting, clusterState)); @@ -492,7 +502,8 @@ public void testAllReplicasUnassigned() { status, clusterState, NodesShutdownMetadata.EMPTY, - randomBoolean() + randomBoolean(), + timeValueSeconds(0) ); var shardRouting = clusterState.routingTable().index("myindex").shardsWithState(ShardRoutingState.UNASSIGNED).get(0); assertFalse(service.areAllShardsOfThisTypeUnavailable(shardRouting, clusterState)); @@ -922,7 +933,7 @@ public void testRestoreFromSnapshotReportsFeatureStates() { ); HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO); - assertThat(result.status(), is(HealthStatus.RED)); + assertThat(result.status(), is(RED)); assertThat(result.diagnosisList().size(), is(1)); Diagnosis diagnosis = result.diagnosisList().get(0); List affectedResources = diagnosis.affectedResources(); @@ -1925,7 +1936,7 @@ private SystemIndices getSystemIndices( // We expose the indicator name and the diagnoses in the x-pack usage API. In order to index them properly in a telemetry index // they need to be declared in the health-api-indexer.edn in the telemetry repository. public void testMappedFieldsForTelemetry() { - assertThat(ShardsAvailabilityHealthIndicatorService.NAME, equalTo("shards_availability")); + assertThat(NAME, equalTo("shards_availability")); assertThat( ACTION_RESTORE_FROM_SNAPSHOT.getUniqueId(), equalTo("elasticsearch:health:shards_availability:diagnosis:restore_from_snapshot") @@ -1970,8 +1981,10 @@ public void testMappedFieldsForTelemetry() { DIAGNOSIS_WAIT_FOR_INITIALIZATION.getUniqueId(), equalTo("elasticsearch:health:shards_availability:diagnosis:initializing_shards") ); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(ClusterSettings.createBuiltInClusterSettings()); var service = new ShardsAvailabilityHealthIndicatorService( - mock(ClusterService.class), + clusterService, mock(AllocationService.class), mock(SystemIndices.class) ); @@ -2004,6 +2017,7 @@ public void testMappedFieldsForTelemetry() { } public void testIsNewlyCreatedAndInitializingReplica() { + ShardId id = new ShardId("index", "uuid", 0); IndexMetadata idxMeta = IndexMetadata.builder("index") .numberOfShards(1) @@ -2017,56 +2031,156 @@ public void testIsNewlyCreatedAndInitializingReplica() { .build() ) .build(); - ShardRouting primary = createShardRouting(id, true, new ShardAllocation("node", AVAILABLE)); - var state = createClusterStateWith(List.of(index("index", new ShardAllocation("node", AVAILABLE))), List.of()); - assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(primary, state)); - - ShardRouting replica = createShardRouting(id, false, new ShardAllocation("node", AVAILABLE)); - state = createClusterStateWith(List.of(index("index", new ShardAllocation("node", AVAILABLE))), List.of()); - assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(replica, state)); - - ShardRouting unassignedReplica = createShardRouting(id, false, new ShardAllocation("node", UNAVAILABLE)); - state = createClusterStateWith( - List.of(idxMeta), - List.of(index("index", "uuid", new ShardAllocation("node", UNAVAILABLE))), - List.of(), - List.of() - ); - assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(unassignedReplica, state)); - UnassignedInfo.Reason reason = randomFrom(UnassignedInfo.Reason.NODE_LEFT, UnassignedInfo.Reason.NODE_RESTARTING); - ShardAllocation allocation = new ShardAllocation( - "node", - UNAVAILABLE, - new UnassignedInfo( - reason, - "message", - null, - 0, - 0, - 0, - randomBoolean(), - randomFrom(UnassignedInfo.AllocationStatus.values()), - Set.of(), - reason == UnassignedInfo.Reason.NODE_LEFT ? null : randomAlphaOfLength(20) - ) - ); - ShardRouting unallocatedReplica = createShardRouting(id, false, allocation); - state = createClusterStateWith( - List.of(idxMeta), - List.of(index(idxMeta, new ShardAllocation("node", UNAVAILABLE), allocation)), - List.of(), - List.of() - ); - assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(unallocatedReplica, state)); + ClusterState state; - state = createClusterStateWith( - List.of(idxMeta), - List.of(index(idxMeta, new ShardAllocation("node", CREATING), allocation)), - List.of(), - List.of() - ); - assertTrue(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(unallocatedReplica, state)); + // --------- Test conditions that don't depend on threshold --------- + + TimeValue replicaUnassignedThreshold = randomFrom(timeValueSeconds(3), timeValueSeconds(0)); + { + // active, whether primary or replica + boolean primary = randomBoolean(); + ShardAllocation primaryAllocation = new ShardAllocation("node", AVAILABLE); + ShardRouting shard = createShardRouting(id, primary, primaryAllocation); + state = createClusterStateWith(List.of(index("index", primaryAllocation)), List.of()); + assertFalse( + ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + shard, + state, + Instant.now().toEpochMilli() - replicaUnassignedThreshold.millis() + ) + ); + } + + { // primary, but not active + var primaryAllocation = new ShardAllocation("node", INITIALIZING); + ShardRouting primary = createShardRouting(id, true, primaryAllocation); + state = createClusterStateWith(List.of(index("index", primaryAllocation)), List.of()); + assertFalse( + ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + primary, + state, + Instant.now().toEpochMilli() - replicaUnassignedThreshold.millis() + ) + ); + } + + // --------- Test conditions that depend on threshold, but with threshold of 0 --------- + replicaUnassignedThreshold = timeValueSeconds(0); + long now = Instant.now().toEpochMilli(); + TimeValue afterCutoffTime = TimeValue.timeValueMillis(now); + { + var unassignedInfo = randomFrom(decidersNo(afterCutoffTime), unassignedInfoNoFailures(afterCutoffTime)); + var replicaAllocation = new ShardAllocation("node", UNAVAILABLE, unassignedInfo); + var primaryAllocation = new ShardAllocation("node", randomFrom(INITIALIZING, UNAVAILABLE, AVAILABLE, RESTARTING)); + + ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation); + state = createClusterStateWith( + List.of(idxMeta), + List.of(index(idxMeta, primaryAllocation, replicaAllocation)), + List.of(), + List.of() + ); + assertFalse( + ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + unallocatedReplica, + state, + now - replicaUnassignedThreshold.millis() + ) + ); + } + + { + var unassignedInfo = randomFrom(decidersNo(afterCutoffTime), unassignedInfoNoFailures(afterCutoffTime)); + var replicaAllocation = new ShardAllocation("node", UNAVAILABLE, unassignedInfo); + var primaryAllocation = new ShardAllocation("node", CREATING); + + ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation); + state = createClusterStateWith( + List.of(idxMeta), + List.of(index(idxMeta, primaryAllocation, replicaAllocation)), + List.of(), + List.of() + ); + assertTrue( + ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + unallocatedReplica, + state, + now - replicaUnassignedThreshold.millis() + ) + ); + } + + // --------- Test conditions that do depend on threshold, but with non-zero threshold --------- + + replicaUnassignedThreshold = timeValueSeconds(3); + afterCutoffTime = TimeValue.timeValueMillis(now - 3000); + TimeValue beforeCutoffTime = TimeValue.timeValueMillis(now - 2999); + { + List> configs = new ArrayList<>(); + + // return false if primary is not creating and if unassigned info has failed allocations or is after cutoff + var uis = List.of(decidersNo(afterCutoffTime), decidersNo(beforeCutoffTime), unassignedInfoNoFailures(afterCutoffTime)); + var shardStates = List.of(UNAVAILABLE, INITIALIZING, RESTARTING, AVAILABLE); + for (var shardState : shardStates) { + for (var ui : uis) { + configs.add(tuple(shardState, ui)); + } + } + // return false if primary is not creating or available and unassigned time is before cutoff + for (var shardState : List.of(UNAVAILABLE, INITIALIZING, RESTARTING)) { + configs.add(tuple(shardState, unassignedInfoNoFailures(beforeCutoffTime))); + } + + for (var config : configs) { + var replicaAllocation = new ShardAllocation("node", UNAVAILABLE, config.v2()); + var primaryAllocation = new ShardAllocation("node", config.v1()); + ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation); + state = createClusterStateWith( + List.of(idxMeta), + List.of(index(idxMeta, primaryAllocation, replicaAllocation)), + List.of(), + List.of() + ); + assertFalse( + ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + unallocatedReplica, + state, + now - replicaUnassignedThreshold.millis() + ) + ); + } + } + + { + var configs = List.of( + // return true because primary is still creating + tuple(CREATING, decidersNo(afterCutoffTime)), + tuple(CREATING, decidersNo(beforeCutoffTime)), + tuple(CREATING, unassignedInfoNoFailures(afterCutoffTime)), + tuple(CREATING, unassignedInfoNoFailures(beforeCutoffTime)), + + // returns true because unassigned time is before cutoff, and no failedAllocations + tuple(AVAILABLE, unassignedInfoNoFailures(beforeCutoffTime)) + ); + + for (var config : configs) { + var replicaAllocation = new ShardAllocation("node", UNAVAILABLE, config.v2()); + var primaryAllocation = new ShardAllocation("node", config.v1()); + + ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation); + IndexRoutingTable index = index(idxMeta, primaryAllocation, replicaAllocation); + + state = createClusterStateWith(List.of(idxMeta), List.of(index), List.of(), List.of()); + assertTrue( + ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica( + unallocatedReplica, + state, + now - replicaUnassignedThreshold.millis() + ) + ); + } + } } private HealthIndicatorResult createExpectedResult( @@ -2373,14 +2487,34 @@ private static UnassignedInfo nodeLeft() { ); } + private static UnassignedInfo unassignedInfoNoFailures(TimeValue unassignedTime) { + UnassignedInfo.Reason reason = randomFrom(UnassignedInfo.Reason.NODE_LEFT, UnassignedInfo.Reason.NODE_RESTARTING); + return new UnassignedInfo( + reason, + "message", + null, + 0, + unassignedTime.nanos(), + unassignedTime.millis(), + randomBoolean(), + randomValueOtherThan(UnassignedInfo.AllocationStatus.DECIDERS_NO, () -> randomFrom(UnassignedInfo.AllocationStatus.values())), + Set.of(), + reason == UnassignedInfo.Reason.NODE_LEFT ? null : randomAlphaOfLength(20) + ); + } + private static UnassignedInfo decidersNo() { + return decidersNo(TimeValue.timeValueMillis(0)); + } + + private static UnassignedInfo decidersNo(TimeValue unassignedTime) { return new UnassignedInfo( UnassignedInfo.Reason.ALLOCATION_FAILED, null, null, 1, - 0, - 0, + unassignedTime.nanos(), + unassignedTime.millis(), false, UnassignedInfo.AllocationStatus.DECIDERS_NO, Collections.emptySet(), @@ -2423,7 +2557,7 @@ private static ShardsAvailabilityHealthIndicatorService createAllocationHealthIn when(clusterService.state()).thenReturn(clusterState); var clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - var allocationService = Mockito.mock(AllocationService.class); + var allocationService = mock(AllocationService.class); when(allocationService.explainShardAllocation(any(), any())).thenAnswer((Answer) invocation -> { ShardRouting shardRouting = invocation.getArgument(0); var key = new ShardRoutingKey(shardRouting.getIndexName(), shardRouting.getId(), shardRouting.primary()); diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java index b1104a72400ea..ae686afcbb296 100644 --- a/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java +++ b/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java @@ -761,6 +761,17 @@ public void checkZonedDateTimeSerialization(TransportVersion tv) throws IOExcept } } + public void testOptional() throws IOException { + try (var output = new BytesStreamOutput()) { + output.writeOptional(StreamOutput::writeString, "not-null"); + output.writeOptional(StreamOutput::writeString, null); + + final var input = getStreamInput(output.bytes()); + assertEquals("not-null", input.readOptional(StreamInput::readString)); + assertNull(input.readOptional(StreamInput::readString)); + } + } + private void assertSerialization( CheckedConsumer outputAssertions, CheckedConsumer inputAssertions, diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java index 88e3125655df0..568fa3e36c769 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java @@ -638,6 +638,7 @@ public void testResponseHeaders() { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/112256") public void testDropWarningsExceedingMaxSettings() { Settings settings = Settings.builder() .put(HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey(), 1) diff --git a/server/src/test/java/org/elasticsearch/injection/InjectorTests.java b/server/src/test/java/org/elasticsearch/injection/InjectorTests.java new file mode 100644 index 0000000000000..025596e640896 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/injection/InjectorTests.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.injection; + +import org.elasticsearch.test.ESTestCase; + +import java.lang.invoke.MethodHandles; +import java.util.List; +import java.util.Set; + +public class InjectorTests extends ESTestCase { + + public record First() {} + + public record Second(First first) {} + + public record Third(First first, Second second) {} + + public record ExistingInstances(First first, Second second) {} + + public void testMultipleResultsMap() { + Injector injector = Injector.create().addClasses(List.of(Service1.class, Component3.class)); + var resultMap = injector.inject(List.of(Service1.class, Component3.class)); + assertEquals(Set.of(Service1.class, Component3.class), resultMap.keySet()); + Service1 service1 = (Service1) resultMap.get(Service1.class); + Component3 component3 = (Component3) resultMap.get(Component3.class); + assertSame(service1, component3.service1()); + } + + /** + * In most cases, if there are two objects that are instances of a class, that's ambiguous. + * However, if a concrete (non-abstract) superclass is configured directly, that is not ambiguous: + * the instance of that superclass takes precedence over any instances of any subclasses. + */ + public void testConcreteSubclass() { + MethodHandles.lookup(); + assertEquals( + Superclass.class, + Injector.create() + .addClasses(List.of(Superclass.class, Subclass.class)) // Superclass first + .inject(List.of(Superclass.class)) + .get(Superclass.class) + .getClass() + ); + MethodHandles.lookup(); + assertEquals( + Superclass.class, + Injector.create() + .addClasses(List.of(Subclass.class, Superclass.class)) // Subclass first + .inject(List.of(Superclass.class)) + .get(Superclass.class) + .getClass() + ); + MethodHandles.lookup(); + assertEquals( + Superclass.class, + Injector.create() + .addClasses(List.of(Subclass.class)) + .inject(List.of(Superclass.class)) // Superclass is not mentioned until here + .get(Superclass.class) + .getClass() + ); + } + + // + // Sad paths + // + + public void testBadInterfaceClass() { + assertThrows(IllegalStateException.class, () -> { + MethodHandles.lookup(); + Injector.create().addClass(Listener.class).inject(List.of()); + }); + } + + public void testBadUnknownType() { + // Injector knows only about Component4, discovers Listener, but can't find any subtypes + MethodHandles.lookup(); + Injector injector = Injector.create().addClass(Component4.class); + + assertThrows(IllegalStateException.class, () -> injector.inject(List.of())); + } + + public void testBadCircularDependency() { + assertThrows(IllegalStateException.class, () -> { + MethodHandles.lookup(); + Injector injector = Injector.create(); + injector.addClasses(List.of(Circular1.class, Circular2.class)).inject(List.of()); + }); + } + + /** + * For this one, we don't explicitly tell the injector about the classes involved in the cycle; + * it finds them on its own. + */ + public void testBadCircularDependencyViaParameter() { + record UsesCircular1(Circular1 circular1) {} + assertThrows(IllegalStateException.class, () -> { + MethodHandles.lookup(); + Injector.create().addClass(UsesCircular1.class).inject(List.of()); + }); + } + + public void testBadCircularDependencyViaSupertype() { + interface Service1 {} + record Service2(Service1 service1) {} + record Service3(Service2 service2) implements Service1 {} + assertThrows(IllegalStateException.class, () -> { + MethodHandles.lookup(); + Injector injector = Injector.create(); + injector.addClasses(List.of(Service2.class, Service3.class)).inject(List.of()); + }); + } + + // Common injectable things + + public record Service1() {} + + public interface Listener {} + + public record Component1() implements Listener {} + + public record Component2(Component1 component1) {} + + public record Component3(Service1 service1) {} + + public record Component4(Listener listener) {} + + public record GoodService(List components) {} + + public record BadService(List components) { + public BadService { + // Shouldn't be using the component list here! + assert components.isEmpty() == false; + } + } + + public record MultiService(List component1s, List component2s) {} + + public record Circular1(Circular2 service2) {} + + public record Circular2(Circular1 service2) {} + + public static class Superclass {} + + public static class Subclass extends Superclass {} + +} diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index c6086a8259fbb..f5e69a65a6d06 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -196,6 +196,7 @@ import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.usage.UsageService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.junit.After; import org.junit.Before; @@ -2059,6 +2060,8 @@ private final class TestClusterNode { private final BigArrays bigArrays; + private final UsageService usageService; + private Coordinator coordinator; TestClusterNode(DiscoveryNode node, TransportInterceptorFactory transportInterceptorFactory) throws IOException { @@ -2069,6 +2072,7 @@ private final class TestClusterNode { masterService = new FakeThreadPoolMasterService(node.getName(), threadPool, deterministicTaskQueue::scheduleNow); final Settings settings = environment.settings(); client = new NodeClient(settings, threadPool); + this.usageService = new UsageService(); final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); clusterService = new ClusterService( settings, @@ -2486,7 +2490,8 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { EmptySystemIndices.INSTANCE.getExecutorSelector(), new SearchTransportAPMMetrics(TelemetryProvider.NOOP.getMeterRegistry()), new SearchResponseMetrics(TelemetryProvider.NOOP.getMeterRegistry()), - client + client, + usageService ) ); actions.put( diff --git a/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json b/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json new file mode 100644 index 0000000000000..fe9c77cb2a183 --- /dev/null +++ b/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json @@ -0,0 +1,67 @@ +{ + "_search" : { + "total" : 10, + "success" : 20, + "skipped" : 5, + "took" : { + "max" : 1991, + "avg" : 1496, + "p90" : 1895 + }, + "took_mrt_true" : { + "max" : 9983, + "avg" : 7476, + "p90" : 9471 + }, + "took_mrt_false" : { + "max" : 19967, + "avg" : 14952, + "p90" : 18943 + }, + "remotes_per_search_max" : 6, + "remotes_per_search_avg" : 7.89, + "failure_reasons" : { + "reason1" : 1, + "reason2" : 2, + "unknown" : 3 + }, + "features" : { + "async" : 10, + "mrt" : 20, + "wildcard" : 30 + }, + "clients" : { + "kibana" : 40, + "other" : 500 + }, + "clusters" : { + "(local)" : { + "total" : 12, + "skipped" : 0, + "took" : { + "max" : 3983, + "avg" : 2992, + "p90" : 3791 + } + }, + "remote1" : { + "total" : 100, + "skipped" : 22, + "took" : { + "max" : 3983, + "avg" : 2992, + "p90" : 3791 + } + }, + "remote2" : { + "total" : 300, + "skipped" : 42, + "took" : { + "max" : 995327, + "avg" : 747531, + "p90" : 946175 + } + } + } + } +} \ No newline at end of file diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionUtils.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionUtils.java new file mode 100644 index 0000000000000..3670013f571e0 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionUtils.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.repositories.blobstore; + +import org.apache.lucene.tests.mockfile.ExtrasFS; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Base64; + +import static org.elasticsearch.test.ESTestCase.between; +import static org.elasticsearch.test.ESTestCase.randomBoolean; +import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.elasticsearch.test.ESTestCase.randomValueOtherThan; +import static org.elasticsearch.test.ESTestCase.randomValueOtherThanMany; + +public class BlobStoreCorruptionUtils { + private static final Logger logger = LogManager.getLogger(BlobStoreCorruptionUtils.class); + + public static Path corruptRandomFile(Path repositoryRootPath) throws IOException { + final var corruptedFileType = getRandomCorruptibleFileType(); + final var corruptedFile = getRandomFileToCorrupt(repositoryRootPath, corruptedFileType); + if (randomBoolean()) { + logger.info("--> deleting [{}]", corruptedFile); + Files.delete(corruptedFile); + } else { + corruptFileContents(corruptedFile); + } + return corruptedFile; + } + + public static void corruptFileContents(Path fileToCorrupt) throws IOException { + final var oldFileContents = Files.readAllBytes(fileToCorrupt); + logger.info("--> contents of [{}] before corruption: [{}]", fileToCorrupt, Base64.getEncoder().encodeToString(oldFileContents)); + final byte[] newFileContents = new byte[randomBoolean() ? oldFileContents.length : between(0, oldFileContents.length)]; + System.arraycopy(oldFileContents, 0, newFileContents, 0, newFileContents.length); + if (newFileContents.length == oldFileContents.length) { + final var corruptionPosition = between(0, newFileContents.length - 1); + newFileContents[corruptionPosition] = randomValueOtherThan(oldFileContents[corruptionPosition], ESTestCase::randomByte); + logger.info( + "--> updating byte at position [{}] from [{}] to [{}]", + corruptionPosition, + oldFileContents[corruptionPosition], + newFileContents[corruptionPosition] + ); + } else { + logger.info("--> truncating file from length [{}] to length [{}]", oldFileContents.length, newFileContents.length); + } + Files.write(fileToCorrupt, newFileContents); + logger.info("--> contents of [{}] after corruption: [{}]", fileToCorrupt, Base64.getEncoder().encodeToString(newFileContents)); + } + + public static RepositoryFileType getRandomCorruptibleFileType() { + return randomValueOtherThanMany( + // these blob types do not have reliable corruption detection, so we must skip them + t -> t == RepositoryFileType.ROOT_INDEX_N || t == RepositoryFileType.ROOT_INDEX_LATEST, + () -> randomFrom(RepositoryFileType.values()) + ); + } + + public static Path getRandomFileToCorrupt(Path repositoryRootPath, RepositoryFileType corruptedFileType) throws IOException { + final var corruptibleFiles = new ArrayList(); + Files.walkFileTree(repositoryRootPath, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) throws IOException { + if (ExtrasFS.isExtra(filePath.getFileName().toString()) == false + && RepositoryFileType.getRepositoryFileType(repositoryRootPath, filePath) == corruptedFileType) { + corruptibleFiles.add(filePath); + } + return super.visitFile(filePath, attrs); + } + }); + return randomFrom(corruptibleFiles); + } +} diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java index db3b81891e87e..4a4ffca0f37aa 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java @@ -113,13 +113,13 @@ protected Collection getPlugins() { } public void testDataGeneratorStressTest() throws IOException { - // Let's generate 1000000 fields to test an extreme case (2 levels of objects + 1 leaf level with 100 fields per object). + // Let's generate 125000 fields to test an extreme case (2 levels of objects + 1 leaf level with 50 fields per object). var testChildFieldGenerator = new DataSourceResponse.ChildFieldGenerator() { private int generatedFields = 0; @Override public int generateChildFieldCount() { - return 100; + return 50; } @Override diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java new file mode 100644 index 0000000000000..ac0b26cb4f4cd --- /dev/null +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java @@ -0,0 +1,370 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.search; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot; +import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.TaskInfo; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.usage.UsageService; +import org.elasticsearch.xpack.async.AsyncResultsIndexPlugin; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; +import org.hamcrest.Matchers; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.MRT_FEATURE; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result.CANCELED; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class CCSUsageTelemetryAsyncSearchIT extends AbstractMultiClustersTestCase { + private static final String REMOTE1 = "cluster-a"; + private static final String REMOTE2 = "cluster-b"; + + @Override + protected boolean reuseClusters() { + return false; + } + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE1, REMOTE2); + } + + @Override + protected Map skipUnavailableForRemoteClusters() { + return Map.of(REMOTE1, true, REMOTE2, true); + } + + @Override + protected Collection> nodePlugins(String clusterAlias) { + List> plugs = Arrays.asList( + CrossClusterAsyncSearchIT.SearchListenerPlugin.class, + AsyncSearch.class, + AsyncResultsIndexPlugin.class, + LocalStateCompositeXPackPlugin.class, + CrossClusterAsyncSearchIT.TestQueryBuilderPlugin.class + ); + return Stream.concat(super.nodePlugins(clusterAlias).stream(), plugs.stream()).collect(Collectors.toList()); + } + + @Before + public void resetSearchListenerPlugin() { + CrossClusterAsyncSearchIT.SearchListenerPlugin.reset(); + } + + private SubmitAsyncSearchRequest makeSearchRequest(String... indices) { + CrossClusterAsyncSearchIT.SearchListenerPlugin.blockQueryPhase(); + + SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(indices); + request.setCcsMinimizeRoundtrips(randomBoolean()); + request.setWaitForCompletionTimeout(TimeValue.timeValueMillis(1)); + request.setKeepOnCompletion(true); + request.getSearchRequest().allowPartialSearchResults(false); + request.getSearchRequest().source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(10)); + if (randomBoolean()) { + request.setBatchedReduceSize(randomIntBetween(2, 256)); + } + + return request; + } + + /** + * Run async search request and get telemetry from it + */ + private CCSTelemetrySnapshot getTelemetryFromSearch(SubmitAsyncSearchRequest searchRequest) throws Exception { + // We want to send search to a specific node (we don't care which one) so that we could + // collect the CCS telemetry from it later + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + final AsyncSearchResponse response = cluster(LOCAL_CLUSTER).client(nodeName) + .execute(SubmitAsyncSearchAction.INSTANCE, searchRequest) + .get(); + // We don't care here too much about the response, we just want to trigger the telemetry collection. + // So we check it's not null and leave the rest to other tests. + final String responseId; + try { + assertNotNull(response.getSearchResponse()); + responseId = response.getId(); + } finally { + response.decRef(); + } + waitForSearchTasksToFinish(); + final AsyncSearchResponse finishedResponse = cluster(LOCAL_CLUSTER).client(nodeName) + .execute(GetAsyncSearchAction.INSTANCE, new GetAsyncResultRequest(responseId)) + .get(); + try { + assertNotNull(finishedResponse.getSearchResponse()); + } finally { + finishedResponse.decRef(); + } + return getTelemetrySnapshot(nodeName); + + } + + private void waitForSearchTasksToFinish() throws Exception { + assertBusy(() -> { + ListTasksResponse listTasksResponse = client(LOCAL_CLUSTER).admin() + .cluster() + .prepareListTasks() + .setActions(TransportSearchAction.TYPE.name()) + .get(); + List tasks = listTasksResponse.getTasks(); + assertThat(tasks.size(), equalTo(0)); + + for (String clusterAlias : remoteClusterAlias()) { + ListTasksResponse remoteTasksResponse = client(clusterAlias).admin() + .cluster() + .prepareListTasks() + .setActions(TransportSearchAction.TYPE.name()) + .get(); + List remoteTasks = remoteTasksResponse.getTasks(); + assertThat(remoteTasks.size(), equalTo(0)); + } + }); + + assertBusy(() -> { + for (String clusterAlias : remoteClusterAlias()) { + final Iterable transportServices = cluster(clusterAlias).getInstances(TransportService.class); + for (TransportService transportService : transportServices) { + assertThat(transportService.getTaskManager().getBannedTaskIds(), Matchers.empty()); + } + } + }); + } + + /** + * Create search request for indices and get telemetry from it + */ + private CCSTelemetrySnapshot getTelemetryFromSearch(String... indices) throws Exception { + return getTelemetryFromSearch(makeSearchRequest(indices)); + } + + /** + * Async search on all remotes + */ + public void testAllRemotesSearch() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SubmitAsyncSearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + boolean minimizeRoundtrips = TransportSearchAction.shouldMinimizeRoundtrips(searchRequest.getSearchRequest()); + CrossClusterAsyncSearchIT.SearchListenerPlugin.negate(); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(searchRequest); + + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + assertThat(telemetry.getTookMrtTrue().count(), equalTo(minimizeRoundtrips ? 1L : 0L)); + assertThat(telemetry.getTookMrtFalse().count(), equalTo(minimizeRoundtrips ? 0L : 1L)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(1L)); + if (minimizeRoundtrips) { + assertThat(telemetry.getFeatureCounts().get(MRT_FEATURE), equalTo(1L)); + } else { + assertThat(telemetry.getFeatureCounts().get(MRT_FEATURE), equalTo(null)); + } + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String clusterAlias : remoteClusterAlias()) { + var clusterTelemetry = perCluster.get(clusterAlias); + assertThat(clusterTelemetry.getCount(), equalTo(1L)); + assertThat(clusterTelemetry.getSkippedCount(), equalTo(0L)); + assertThat(clusterTelemetry.getTook().count(), equalTo(1L)); + } + } + + /** + * Search that is cancelled + */ + public void testCancelledSearch() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SubmitAsyncSearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex); + CrossClusterAsyncSearchIT.SearchListenerPlugin.blockQueryPhase(); + + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + final AsyncSearchResponse response = cluster(LOCAL_CLUSTER).client(nodeName) + .execute(SubmitAsyncSearchAction.INSTANCE, searchRequest) + .get(); + try { + assertNotNull(response.getSearchResponse()); + } finally { + response.decRef(); + assertTrue(response.isRunning()); + } + CrossClusterAsyncSearchIT.SearchListenerPlugin.waitSearchStarted(); + + ActionFuture cancelFuture; + try { + ListTasksResponse listTasksResponse = client(LOCAL_CLUSTER).admin() + .cluster() + .prepareListTasks() + .setActions(TransportSearchAction.TYPE.name()) + .get(); + List tasks = listTasksResponse.getTasks(); + assertThat(tasks.size(), equalTo(1)); + final TaskInfo rootTask = tasks.get(0); + + AtomicReference> remoteClusterSearchTasks = new AtomicReference<>(); + assertBusy(() -> { + List remoteSearchTasks = client(REMOTE1).admin() + .cluster() + .prepareListTasks() + .get() + .getTasks() + .stream() + .filter(t -> t.action().contains(TransportSearchAction.TYPE.name())) + .collect(Collectors.toList()); + assertThat(remoteSearchTasks.size(), greaterThan(0)); + remoteClusterSearchTasks.set(remoteSearchTasks); + }); + + for (TaskInfo taskInfo : remoteClusterSearchTasks.get()) { + assertFalse("taskInfo on remote cluster should not be cancelled yet: " + taskInfo, taskInfo.cancelled()); + } + + final CancelTasksRequest cancelRequest = new CancelTasksRequest().setTargetTaskId(rootTask.taskId()); + cancelRequest.setWaitForCompletion(randomBoolean()); + cancelFuture = client().admin().cluster().cancelTasks(cancelRequest); + assertBusy(() -> { + final Iterable transportServices = cluster(REMOTE1).getInstances(TransportService.class); + for (TransportService transportService : transportServices) { + Collection cancellableTasks = transportService.getTaskManager().getCancellableTasks().values(); + for (CancellableTask cancellableTask : cancellableTasks) { + if (cancellableTask.getAction().contains(TransportSearchAction.TYPE.name())) { + assertTrue(cancellableTask.getDescription(), cancellableTask.isCancelled()); + } + } + } + }); + + List remoteSearchTasksAfterCancellation = client(REMOTE1).admin() + .cluster() + .prepareListTasks() + .get() + .getTasks() + .stream() + .filter(t -> t.action().contains(TransportSearchAction.TYPE.name())) + .toList(); + for (TaskInfo taskInfo : remoteSearchTasksAfterCancellation) { + assertTrue(taskInfo.description(), taskInfo.cancelled()); + } + } finally { + CrossClusterAsyncSearchIT.SearchListenerPlugin.allowQueryPhase(); + } + + assertBusy(() -> assertTrue(cancelFuture.isDone())); + waitForSearchTasksToFinish(); + + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(1)); + assertThat(telemetry.getFailureReasons().get(CANCELED.getName()), equalTo(1L)); + assertThat(telemetry.getTook().count(), equalTo(0L)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(1.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L)); + // Still counts as async search + assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(1L)); + } + + private CCSTelemetrySnapshot getTelemetrySnapshot(String nodeName) { + var usage = cluster(LOCAL_CLUSTER).getInstance(UsageService.class, nodeName); + return usage.getCcsUsageHolder().getCCSTelemetrySnapshot(); + } + + private Map setupClusters() { + String localIndex = "demo"; + int numShardsLocal = randomIntBetween(2, 10); + Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build(); + assertAcked( + client(LOCAL_CLUSTER).admin() + .indices() + .prepareCreate(localIndex) + .setSettings(localSettings) + .setMapping("@timestamp", "type=date", "f", "type=text") + ); + indexDocs(client(LOCAL_CLUSTER), localIndex); + + String remoteIndex = "prod"; + int numShardsRemote = randomIntBetween(2, 10); + for (String clusterAlias : remoteClusterAlias()) { + final InternalTestCluster remoteCluster = cluster(clusterAlias); + remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(1, 3)); + assertAcked( + client(clusterAlias).admin() + .indices() + .prepareCreate(remoteIndex) + .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1))) + .setMapping("@timestamp", "type=date", "f", "type=text") + ); + assertFalse( + client(clusterAlias).admin() + .cluster() + .prepareHealth(remoteIndex) + .setWaitForYellowStatus() + .setTimeout(TimeValue.timeValueSeconds(10)) + .get() + .isTimedOut() + ); + indexDocs(client(clusterAlias), remoteIndex); + } + + Map clusterInfo = new HashMap<>(); + clusterInfo.put("local.num_shards", numShardsLocal); + clusterInfo.put("local.index", localIndex); + clusterInfo.put("remote.num_shards", numShardsRemote); + clusterInfo.put("remote.index", remoteIndex); + clusterInfo.put("remote.skip_unavailable", true); + return clusterInfo; + } + + private int indexDocs(Client client, String index) { + int numDocs = between(5, 20); + for (int i = 0; i < numDocs; i++) { + client.prepareIndex(index).setSource("f", "v", "@timestamp", randomNonNegativeLong()).get(); + } + client.admin().indices().prepareRefresh(index).get(); + return numDocs; + } +} diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java index 6a55738b864d1..3dfece0a9b20e 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java @@ -311,9 +311,9 @@ private CacheEntry(T chunk) { private final int numRegions; private final ConcurrentLinkedQueue freeRegions = new ConcurrentLinkedQueue<>(); - private final Cache cache; + private final Cache> cache; - private final ConcurrentHashMap regionOwners; // to assert exclusive access of regions + private final ConcurrentHashMap> regionOwners; // to assert exclusive access of regions private final LongAdder writeCount = new LongAdder(); private final LongAdder writeBytes = new LongAdder(); @@ -471,7 +471,7 @@ public int getRegionSize() { return regionSize; } - CacheFileRegion get(KeyType cacheKey, long fileLength, int region) { + CacheFileRegion get(KeyType cacheKey, long fileLength, int region) { return cache.get(cacheKey, fileLength, region).chunk; } @@ -516,7 +516,7 @@ public boolean maybeFetchFullEntry( return true; } final ActionListener regionListener = refCountingListener.acquire(ignored -> {}); - final CacheFileRegion entry; + final CacheFileRegion entry; try { entry = get(cacheKey, length, region); } catch (AlreadyClosedException e) { @@ -583,7 +583,7 @@ public void maybeFetchRegion( listener.onResponse(false); return; } - final CacheFileRegion entry = get(cacheKey, blobLength, region); + final CacheFileRegion entry = get(cacheKey, blobLength, region); entry.populate(regionRange, writer, fetchExecutor, listener); } catch (Exception e) { listener.onFailure(e); @@ -631,7 +631,7 @@ public void maybeFetchRange( listener.onResponse(false); return; } - final CacheFileRegion entry = get(cacheKey, blobLength, region); + final CacheFileRegion entry = get(cacheKey, blobLength, region); entry.populate( regionRange, writerWithOffset(writer, Math.toIntExact(range.start() - getRegionStart(region))), @@ -705,7 +705,7 @@ public int forceEvict(Predicate cacheKeyPredicate) { } // used by tests - int getFreq(CacheFileRegion cacheFileRegion) { + int getFreq(CacheFileRegion cacheFileRegion) { if (cache instanceof LFUCache lfuCache) { return lfuCache.getFreq(cacheFileRegion); } @@ -787,25 +787,45 @@ protected boolean assertOffsetsWithinFileLength(long offset, long length, long f /** * While this class has incRef and tryIncRef methods, incRefEnsureOpen and tryIncrefEnsureOpen should * always be used, ensuring the right ordering between incRef/tryIncRef and ensureOpen - * (see {@link LFUCache#maybeEvictAndTakeForFrequency(Runnable, int)}) + * (see {@link SharedBlobCacheService.LFUCache#maybeEvictAndTakeForFrequency(Runnable, int)}) */ - class CacheFileRegion extends EvictableRefCounted { + static class CacheFileRegion extends EvictableRefCounted { + + private static final VarHandle VH_IO = findIOVarHandle(); + + private static VarHandle findIOVarHandle() { + try { + return MethodHandles.lookup().in(CacheFileRegion.class).findVarHandle(CacheFileRegion.class, "io", SharedBytes.IO.class); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + final SharedBlobCacheService blobCacheService; final RegionKey regionKey; final SparseFileTracker tracker; // io can be null when not init'ed or after evict/take - volatile SharedBytes.IO io = null; - - CacheFileRegion(RegionKey regionKey, int regionSize) { + // io does not need volatile access on the read path, since it goes from null to a single value (and then possbily back to null). + // "cache.get" never returns a `CacheFileRegion` without checking the value is non-null (with a volatile read, ensuring the value is + // visible in that thread). + // We assume any IndexInput passing among threads is done with proper happens-before semantics (otherwise they'd themselves break). + // In general, assertions should use `nonVolatileIO` (when they can) to access this over `volatileIO` to avoid memory visibility + // side effects + private SharedBytes.IO io = null; + + CacheFileRegion(SharedBlobCacheService blobCacheService, RegionKey regionKey, int regionSize) { + this.blobCacheService = blobCacheService; this.regionKey = regionKey; assert regionSize > 0; // NOTE we use a constant string for description to avoid consume extra heap space tracker = new SparseFileTracker("file", regionSize); } - public long physicalStartOffset() { - var ioRef = io; - return ioRef == null ? -1L : (long) regionKey.region * regionSize; + // only used for logging + private long physicalStartOffset() { + var ioRef = nonVolatileIO(); + return ioRef == null ? -1L : (long) regionKey.region * blobCacheService.regionSize; } public boolean tryIncRefEnsureOpen() { @@ -832,10 +852,10 @@ private void ensureOpenOrDecRef() { // tries to evict this chunk if noone is holding onto its resources anymore // visible for tests. boolean tryEvict() { - assert Thread.holdsLock(SharedBlobCacheService.this) : "must hold lock when evicting"; + assert Thread.holdsLock(blobCacheService) : "must hold lock when evicting"; if (refCount() <= 1 && evict()) { logger.trace("evicted {} with channel offset {}", regionKey, physicalStartOffset()); - evictCount.increment(); + blobCacheService.evictCount.increment(); decRef(); return true; } @@ -843,10 +863,10 @@ boolean tryEvict() { } boolean tryEvictNoDecRef() { - assert Thread.holdsLock(SharedBlobCacheService.this) : "must hold lock when evicting"; + assert Thread.holdsLock(blobCacheService) : "must hold lock when evicting"; if (refCount() <= 1 && evict()) { logger.trace("evicted and take {} with channel offset {}", regionKey, physicalStartOffset()); - evictCount.increment(); + blobCacheService.evictCount.increment(); return true; } @@ -854,10 +874,10 @@ boolean tryEvictNoDecRef() { } public boolean forceEvict() { - assert Thread.holdsLock(SharedBlobCacheService.this) : "must hold lock when evicting"; + assert Thread.holdsLock(blobCacheService) : "must hold lock when evicting"; if (evict()) { logger.trace("force evicted {} with channel offset {}", regionKey, physicalStartOffset()); - evictCount.increment(); + blobCacheService.evictCount.increment(); decRef(); return true; } @@ -868,9 +888,10 @@ public boolean forceEvict() { protected void closeInternal() { // now actually free the region associated with this chunk // we held the "this" lock when this was evicted, hence if io is not filled in, chunk will never be registered. + SharedBytes.IO io = volatileIO(); if (io != null) { - assert regionOwners.remove(io) == this; - freeRegions.add(io); + assert blobCacheService.regionOwners.remove(io) == this; + blobCacheService.freeRegions.add(io); } logger.trace("closed {} with channel offset {}", regionKey, physicalStartOffset()); } @@ -879,14 +900,31 @@ private static void throwAlreadyEvicted() { throwAlreadyClosed("File chunk is evicted"); } + private SharedBytes.IO volatileIO() { + return (SharedBytes.IO) VH_IO.getVolatile(this); + } + + private void volatileIO(SharedBytes.IO io) { + VH_IO.setVolatile(this, io); + } + + private SharedBytes.IO nonVolatileIO() { + return io; + } + + // for use in tests *only* + SharedBytes.IO testOnlyNonVolatileIO() { + return io; + } + /** * Optimistically try to read from the region * @return true if successful, i.e., not evicted and data available, false if evicted */ boolean tryRead(ByteBuffer buf, long offset) throws IOException { - SharedBytes.IO ioRef = this.io; + SharedBytes.IO ioRef = nonVolatileIO(); if (ioRef != null) { - int readBytes = ioRef.read(buf, getRegionRelativePosition(offset)); + int readBytes = ioRef.read(buf, blobCacheService.getRegionRelativePosition(offset)); if (isEvicted()) { buf.position(buf.position() - readBytes); return false; @@ -922,7 +960,7 @@ void populate( rangeToWrite, rangeToWrite, Assertions.ENABLED ? ActionListener.releaseAfter(ActionListener.running(() -> { - assert regionOwners.get(io) == this; + assert blobCacheService.regionOwners.get(nonVolatileIO()) == this; }), refs.acquire()) : refs.acquireListener() ); if (gaps.isEmpty()) { @@ -958,8 +996,8 @@ void populateAndRead( rangeToWrite, rangeToRead, ActionListener.releaseAfter(listener, refs.acquire()).delegateFailureAndWrap((l, success) -> { - var ioRef = io; - assert regionOwners.get(ioRef) == this; + var ioRef = nonVolatileIO(); + assert blobCacheService.regionOwners.get(ioRef) == this; final int start = Math.toIntExact(rangeToRead.start()); final int read = reader.onRangeAvailable(ioRef, start, start, Math.toIntExact(rangeToRead.length())); assert read == rangeToRead.length() @@ -970,7 +1008,7 @@ void populateAndRead( + '-' + rangeToRead.start() + ']'; - readCount.increment(); + blobCacheService.readCount.increment(); l.onResponse(read); }) ); @@ -1016,8 +1054,8 @@ private Runnable fillGapRunnable( ActionListener listener ) { return () -> ActionListener.run(listener, l -> { - var ioRef = io; - assert regionOwners.get(ioRef) == CacheFileRegion.this; + var ioRef = nonVolatileIO(); + assert blobCacheService.regionOwners.get(ioRef) == CacheFileRegion.this; assert CacheFileRegion.this.hasReferences() : CacheFileRegion.this; int start = Math.toIntExact(gap.start()); writer.fillCacheRange( @@ -1028,9 +1066,9 @@ private Runnable fillGapRunnable( Math.toIntExact(gap.end() - start), progress -> gap.onProgress(start + progress), l.map(unused -> { - assert regionOwners.get(ioRef) == CacheFileRegion.this; + assert blobCacheService.regionOwners.get(ioRef) == CacheFileRegion.this; assert CacheFileRegion.this.hasReferences() : CacheFileRegion.this; - writeCount.increment(); + blobCacheService.writeCount.increment(); gap.onCompletion(); return null; }).delegateResponse((delegate, e) -> failGapAndListener(gap, delegate, e)) @@ -1058,7 +1096,7 @@ public class CacheFile { private final KeyType cacheKey; private final long length; - private CacheEntry lastAccessedRegion; + private CacheEntry> lastAccessedRegion; private CacheFile(KeyType cacheKey, long length) { this.cacheKey = cacheKey; @@ -1161,7 +1199,7 @@ private int readSingleRegion( int region ) throws InterruptedException, ExecutionException { final PlainActionFuture readFuture = new PlainActionFuture<>(); - final CacheFileRegion fileRegion = get(cacheKey, length, region); + final CacheFileRegion fileRegion = get(cacheKey, length, region); final long regionStart = getRegionStart(region); fileRegion.populateAndRead( mapSubRangeToRegion(rangeToWrite, region), @@ -1193,7 +1231,7 @@ private int readMultiRegions( } ActionListener listener = listeners.acquire(i -> bytesRead.updateAndGet(j -> Math.addExact(i, j))); try { - final CacheFileRegion fileRegion = get(cacheKey, length, region); + final CacheFileRegion fileRegion = get(cacheKey, length, region); final long regionStart = getRegionStart(region); fileRegion.populateAndRead( mapSubRangeToRegion(rangeToWrite, region), @@ -1213,7 +1251,7 @@ private int readMultiRegions( return bytesRead.get(); } - private RangeMissingHandler writerWithOffset(RangeMissingHandler writer, CacheFileRegion fileRegion, int writeOffset) { + private RangeMissingHandler writerWithOffset(RangeMissingHandler writer, CacheFileRegion fileRegion, int writeOffset) { final RangeMissingHandler adjustedWriter; if (writeOffset == 0) { // no need to allocate a new capturing lambda if the offset isn't adjusted @@ -1263,8 +1301,8 @@ public void fillCacheRange( len, progressUpdater, Assertions.ENABLED ? ActionListener.runBefore(completionListener, () -> { - assert regionOwners.get(fileRegion.io) == fileRegion - : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.io + "]"; + assert regionOwners.get(fileRegion.nonVolatileIO()) == fileRegion + : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.nonVolatileIO() + "]"; }) : completionListener ); } @@ -1274,7 +1312,7 @@ public void fillCacheRange( return adjustedWriter; } - private RangeAvailableHandler readerWithOffset(RangeAvailableHandler reader, CacheFileRegion fileRegion, int readOffset) { + private RangeAvailableHandler readerWithOffset(RangeAvailableHandler reader, CacheFileRegion fileRegion, int readOffset) { final RangeAvailableHandler adjustedReader = (channel, channelPos, relativePos, len) -> reader.onRangeAvailable( channel, channelPos, @@ -1285,18 +1323,18 @@ private RangeAvailableHandler readerWithOffset(RangeAvailableHandler reader, Cac return (channel, channelPos, relativePos, len) -> { assert assertValidRegionAndLength(fileRegion, channelPos, len); final int bytesRead = adjustedReader.onRangeAvailable(channel, channelPos, relativePos, len); - assert regionOwners.get(fileRegion.io) == fileRegion - : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.io + "]"; + assert regionOwners.get(fileRegion.nonVolatileIO()) == fileRegion + : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.nonVolatileIO() + "]"; return bytesRead; }; } return adjustedReader; } - private boolean assertValidRegionAndLength(CacheFileRegion fileRegion, int channelPos, int len) { - assert fileRegion.io != null; + private boolean assertValidRegionAndLength(CacheFileRegion fileRegion, int channelPos, int len) { + assert fileRegion.nonVolatileIO() != null; assert fileRegion.hasReferences(); - assert regionOwners.get(fileRegion.io) == fileRegion; + assert regionOwners.get(fileRegion.nonVolatileIO()) == fileRegion; assert channelPos >= 0 && channelPos + len <= regionSize; return true; } @@ -1421,15 +1459,15 @@ public record Stats( public static final Stats EMPTY = new Stats(0, 0L, 0L, 0L, 0L, 0L, 0L, 0L); } - private class LFUCache implements Cache { + private class LFUCache implements Cache> { - class LFUCacheEntry extends CacheEntry { + class LFUCacheEntry extends CacheEntry> { LFUCacheEntry prev; LFUCacheEntry next; int freq; volatile long lastAccessedEpoch; - LFUCacheEntry(CacheFileRegion chunk, long lastAccessed) { + LFUCacheEntry(CacheFileRegion chunk, long lastAccessed) { super(chunk); this.lastAccessedEpoch = lastAccessed; // todo: consider whether freq=1 is still right for new entries. @@ -1467,7 +1505,7 @@ public void close() { decayAndNewEpochTask.close(); } - int getFreq(CacheFileRegion cacheFileRegion) { + int getFreq(CacheFileRegion cacheFileRegion) { return keyMapping.get(cacheFileRegion.regionKey).freq; } @@ -1480,12 +1518,15 @@ public LFUCacheEntry get(KeyType cacheKey, long fileLength, int region) { var entry = keyMapping.get(regionKey); if (entry == null) { final int effectiveRegionSize = computeCacheFileRegionSize(fileLength, region); - entry = keyMapping.computeIfAbsent(regionKey, key -> new LFUCacheEntry(new CacheFileRegion(key, effectiveRegionSize), now)); + entry = keyMapping.computeIfAbsent( + regionKey, + key -> new LFUCacheEntry(new CacheFileRegion(SharedBlobCacheService.this, key, effectiveRegionSize), now) + ); } - // io is volatile, double locking is fine, as long as we assign it last. - if (entry.chunk.io == null) { + // checks using volatile, double locking is fine, as long as we assign io last. + if (entry.chunk.volatileIO() == null) { synchronized (entry.chunk) { - if (entry.chunk.io == null && entry.chunk.isEvicted() == false) { + if (entry.chunk.volatileIO() == null && entry.chunk.isEvicted() == false) { return initChunk(entry); } } @@ -1515,7 +1556,7 @@ public int forceEvict(Predicate cacheKeyPredicate) { for (LFUCacheEntry entry : matchingEntries) { int frequency = entry.freq; boolean evicted = entry.chunk.forceEvict(); - if (evicted && entry.chunk.io != null) { + if (evicted && entry.chunk.volatileIO() != null) { unlink(entry); keyMapping.remove(entry.chunk.regionKey, entry); evictedCount++; @@ -1576,7 +1617,7 @@ private void assignToSlot(LFUCacheEntry entry, SharedBytes.IO freeSlot) { } pushEntryToBack(entry); // assign io only when chunk is ready for use. Under lock to avoid concurrent tryEvict. - entry.chunk.io = freeSlot; + entry.chunk.volatileIO(freeSlot); } } @@ -1641,7 +1682,7 @@ private boolean assertChunkActiveOrEvicted(LFUCacheEntry entry) { assert entry.prev != null || entry.chunk.isEvicted(); } - SharedBytes.IO io = entry.chunk.io; + SharedBytes.IO io = entry.chunk.nonVolatileIO(); assert io != null || entry.chunk.isEvicted(); assert io == null || regionOwners.get(io) == entry.chunk || entry.chunk.isEvicted(); return true; @@ -1764,13 +1805,13 @@ private SharedBytes.IO maybeEvictAndTakeForFrequency(Runnable evictedNotificatio boolean evicted = entry.chunk.tryEvictNoDecRef(); if (evicted) { try { - SharedBytes.IO ioRef = entry.chunk.io; + SharedBytes.IO ioRef = entry.chunk.volatileIO(); if (ioRef != null) { try { if (entry.chunk.refCount() == 1) { // we own that one refcount (since we CAS'ed evicted to 1) // grab io, rely on incref'ers also checking evicted field. - entry.chunk.io = null; + entry.chunk.volatileIO(null); assert regionOwners.remove(ioRef) == entry.chunk; return ioRef; } @@ -1809,7 +1850,7 @@ public boolean maybeEvictLeastUsed() { synchronized (SharedBlobCacheService.this) { for (LFUCacheEntry entry = freqs[0]; entry != null; entry = entry.next) { boolean evicted = entry.chunk.tryEvict(); - if (evicted && entry.chunk.io != null) { + if (evicted && entry.chunk.volatileIO() != null) { unlink(entry); keyMapping.remove(entry.chunk.regionKey, entry); return true; diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java index 597180a1d1c31..0f3804baef42b 100644 --- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java +++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java @@ -149,7 +149,7 @@ public void testBasicEviction() throws IOException { } } - private static boolean tryEvict(SharedBlobCacheService.CacheFileRegion region1) { + private static boolean tryEvict(SharedBlobCacheService.CacheFileRegion region1) { if (randomBoolean()) { return region1.tryEvict(); } else { @@ -444,6 +444,7 @@ public void testMassiveDecay() throws IOException { * Exercise SharedBlobCacheService#get in multiple threads to trigger any assertion errors. * @throws IOException */ + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/112305") public void testGetMultiThreaded() throws IOException { final int threads = between(2, 10); final int regionCount = between(1, 20); @@ -486,7 +487,7 @@ public void testGetMultiThreaded() throws IOException { ready.await(); for (int i = 0; i < iterations; ++i) { try { - SharedBlobCacheService.CacheFileRegion cacheFileRegion; + SharedBlobCacheService.CacheFileRegion cacheFileRegion; try { cacheFileRegion = cacheService.get(cacheKeys[i], fileLength, regions[i]); } catch (AlreadyClosedException e) { @@ -497,6 +498,7 @@ public void testGetMultiThreaded() throws IOException { if (yield[i] == 0) { Thread.yield(); } + assertNotNull(cacheFileRegion.testOnlyNonVolatileIO()); cacheFileRegion.decRef(); } if (evict[i] == 0) { @@ -865,7 +867,7 @@ public void testMaybeEvictLeastUsed() throws Exception { final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(); try ( NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings)); - var cacheService = new SharedBlobCacheService<>( + var cacheService = new SharedBlobCacheService( environment, settings, taskQueue.getThreadPool(), @@ -873,7 +875,7 @@ public void testMaybeEvictLeastUsed() throws Exception { BlobCacheMetrics.NOOP ) ) { - final Map.CacheFileRegion> cacheEntries = new HashMap<>(); + final Map> cacheEntries = new HashMap<>(); assertThat("All regions are free", cacheService.freeRegionCount(), equalTo(numRegions)); assertThat("Cache has no entries", cacheService.maybeEvictLeastUsed(), is(false)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java index 1ba625a507a46..f7ad1f65628b2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java @@ -74,7 +74,7 @@ public RollupJobStatus(IndexerState state, @Nullable Map positio public RollupJobStatus(StreamInput in) throws IOException { state = IndexerState.fromStream(in); - currentPosition = in.readBoolean() ? new TreeMap<>(in.readGenericMap()) : null; + currentPosition = in.readOptional(CURRENT_POSITION_READER); if (in.getTransportVersion().before(TransportVersions.V_8_0_0)) { // 7.x nodes serialize `upgradedDocumentID` flag. We don't need it anymore, but // we need to pull it off the stream @@ -83,6 +83,8 @@ public RollupJobStatus(StreamInput in) throws IOException { } } + private static final Reader> CURRENT_POSITION_READER = in -> new TreeMap<>(in.readGenericMap()); + public IndexerState getIndexerState() { return state; } @@ -118,10 +120,7 @@ public String getWriteableName() { @Override public void writeTo(StreamOutput out) throws IOException { state.writeTo(out); - out.writeBoolean(currentPosition != null); - if (currentPosition != null) { - out.writeGenericMap(currentPosition); - } + out.writeOptional(StreamOutput::writeGenericMap, currentPosition); if (out.getTransportVersion().before(TransportVersions.V_8_0_0)) { // 7.x nodes expect a boolean `upgradedDocumentID` flag. We don't have it anymore, // but we need to tell them we are upgraded in case there is a mixed cluster diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java index 4f3d7a245fc8f..74434adf61fbb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @@ -868,6 +868,11 @@ private static RoleDescriptor buildViewerRoleDescriptor() { .indices("/~(([.]|ilm-history-).*)/") .privileges("read", "view_index_metadata") .build(), + // Observability + RoleDescriptor.IndicesPrivileges.builder() + .indices(".slo-observability.*") + .privileges("read", "view_index_metadata") + .build(), // Security RoleDescriptor.IndicesPrivileges.builder() .indices(ReservedRolesStore.ALERTS_LEGACY_INDEX, ReservedRolesStore.LISTS_INDEX, ReservedRolesStore.LISTS_ITEMS_INDEX) @@ -915,6 +920,10 @@ private static RoleDescriptor buildEditorRoleDescriptor() { .indices("observability-annotations") .privileges("read", "view_index_metadata", "write") .build(), + RoleDescriptor.IndicesPrivileges.builder() + .indices(".slo-observability.*") + .privileges("read", "view_index_metadata", "write", "manage") + .build(), // Security RoleDescriptor.IndicesPrivileges.builder() .indices(ReservedRolesStore.ALERTS_LEGACY_INDEX, ReservedRolesStore.LISTS_INDEX, ReservedRolesStore.LISTS_ITEMS_INDEX) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java index 681b004dd1d28..2f2617f956ed9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java @@ -59,12 +59,8 @@ public ExecuteWatchRequest(StreamInput in) throws IOException { id = in.readOptionalString(); ignoreCondition = in.readBoolean(); recordExecution = in.readBoolean(); - if (in.readBoolean()) { - alternativeInput = in.readGenericMap(); - } - if (in.readBoolean()) { - triggerData = in.readGenericMap(); - } + alternativeInput = in.readOptional(StreamInput::readGenericMap); + triggerData = in.readOptional(StreamInput::readGenericMap); long actionModesCount = in.readLong(); actionModes = new HashMap<>(); for (int i = 0; i < actionModesCount; i++) { @@ -83,14 +79,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeBoolean(ignoreCondition); out.writeBoolean(recordExecution); - out.writeBoolean(alternativeInput != null); - if (alternativeInput != null) { - out.writeGenericMap(alternativeInput); - } - out.writeBoolean(triggerData != null); - if (triggerData != null) { - out.writeGenericMap(triggerData); - } + out.writeOptional(StreamOutput::writeGenericMap, alternativeInput); + out.writeOptional(StreamOutput::writeGenericMap, triggerData); out.writeLong(actionModes.size()); for (Map.Entry entry : actionModes.entrySet()) { out.writeString(entry.getKey()); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index f0676f35ae316..0cdf7de63ca99 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; +import org.elasticsearch.action.admin.indices.mapping.put.TransportAutoPutMappingAction; import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction; import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction; @@ -3662,6 +3663,9 @@ public void testPredefinedViewerRole() { assertOnlyReadAllowed(role, ".profiling-" + randomIntBetween(0, 5)); assertOnlyReadAllowed(role, randomAlphaOfLength(5)); + assertOnlyReadAllowed(role, ".slo-observability." + randomIntBetween(0, 5)); + assertViewIndexMetadata(role, ".slo-observability." + randomIntBetween(0, 5)); + assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES); assertNoAccessAllowed(role, "." + randomAlphaOfLengthBetween(6, 10)); assertNoAccessAllowed(role, "ilm-history-" + randomIntBetween(0, 5)); @@ -3740,6 +3744,9 @@ public void testPredefinedEditorRole() { assertReadWriteDocsAndMaintenanceButNotDeleteIndexAllowed(role, ".preview.alerts-" + randomIntBetween(0, 5)); assertReadWriteDocsAndMaintenanceButNotDeleteIndexAllowed(role, ".internal.preview.alerts-" + randomIntBetween(0, 5)); + assertViewIndexMetadata(role, ".slo-observability." + randomIntBetween(0, 5)); + assertReadWriteAndManage(role, ".slo-observability." + randomIntBetween(0, 5)); + assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES); assertNoAccessAllowed(role, "." + randomAlphaOfLengthBetween(6, 10)); assertNoAccessAllowed(role, "ilm-history-" + randomIntBetween(0, 5)); @@ -3865,6 +3872,41 @@ private void assertReadWriteDocsButNotDeleteIndexAllowed(Role role, String index role.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(mockIndexAbstraction(index)), is(false) ); + + assertThat(role.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(mockIndexAbstraction(index)), is(true)); + assertThat(role.indices().allowedIndicesMatcher(TransportGetAction.TYPE.name()).test(mockIndexAbstraction(index)), is(true)); + assertThat(role.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(mockIndexAbstraction(index)), is(true)); + assertThat(role.indices().allowedIndicesMatcher(TransportUpdateAction.NAME).test(mockIndexAbstraction(index)), is(true)); + assertThat(role.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(mockIndexAbstraction(index)), is(true)); + assertThat(role.indices().allowedIndicesMatcher(TransportBulkAction.NAME).test(mockIndexAbstraction(index)), is(true)); + } + + private void assertReadWriteAndManage(Role role, String index) { + assertThat( + role.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(mockIndexAbstraction(index)), + is(true) + ); + assertThat( + role.indices().allowedIndicesMatcher(TransportFieldCapabilitiesAction.NAME + "*").test(mockIndexAbstraction(index)), + is(true) + ); + assertThat( + role.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(mockIndexAbstraction(index)), + is(true) + ); + assertThat( + role.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(mockIndexAbstraction(index)), + is(true) + ); + assertThat(role.indices().allowedIndicesMatcher(GetRollupIndexCapsAction.NAME + "*").test(mockIndexAbstraction(index)), is(true)); + assertThat(role.indices().allowedIndicesMatcher("indices:admin/*").test(mockIndexAbstraction(index)), is(true)); + assertThat(role.indices().allowedIndicesMatcher("indices:monitor/*").test(mockIndexAbstraction(index)), is(true)); + assertThat( + role.indices().allowedIndicesMatcher(TransportAutoPutMappingAction.TYPE.name()).test(mockIndexAbstraction(index)), + is(true) + ); + assertThat(role.indices().allowedIndicesMatcher(AutoCreateAction.NAME).test(mockIndexAbstraction(index)), is(true)); + assertThat(role.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(mockIndexAbstraction(index)), is(true)); assertThat(role.indices().allowedIndicesMatcher(TransportGetAction.TYPE.name()).test(mockIndexAbstraction(index)), is(true)); assertThat(role.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(mockIndexAbstraction(index)), is(true)); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java index a6ba4346b1a25..7dcda9c2b0032 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java @@ -137,6 +137,6 @@ public List getNamedWriteables() { @Override public Collection createComponents(PluginServices services) { - return List.of(new DownsampleMetrics(services.telemetryProvider().getMeterRegistry())); + return List.of(DownsampleMetrics.class); } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java index c950658b411ed..b5ac4b0ae37a3 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.downsample; import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.telemetry.metric.MeterRegistry; import java.io.IOException; @@ -36,8 +37,8 @@ public class DownsampleMetrics extends AbstractLifecycleComponent { private final MeterRegistry meterRegistry; - public DownsampleMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; + public DownsampleMetrics(TelemetryProvider telemetryProvider) { + this.meterRegistry = telemetryProvider.getMeterRegistry(); } @Override diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichReindexAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichReindexAction.java index 0eeb85f4574f7..cc42199ab1019 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichReindexAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichReindexAction.java @@ -61,7 +61,8 @@ public TransportEnrichReindexAction( autoCreateIndex, client, transportService, - new ReindexSslConfig(settings, environment, watcherService) + new ReindexSslConfig(settings, environment, watcherService), + null ); this.bulkClient = new OriginSettingClient(client, ENRICH_ORIGIN); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java index 92213eca7b477..2c36b42dee277 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java @@ -146,7 +146,7 @@ private void checkFailure() { Exception e = failureCollector.getFailure(); if (e != null) { discardPages(); - throw ExceptionsHelper.convertToElastic(e); + throw ExceptionsHelper.convertToRuntime(e); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java index 406dc4494208c..e3fc0e26e34e0 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java @@ -54,7 +54,7 @@ private class ExchangeSourceImpl implements ExchangeSource { private void checkFailure() { Exception e = failure.getFailure(); if (e != null) { - throw ExceptionsHelper.convertToElastic(e); + throw ExceptionsHelper.convertToRuntime(e); } } diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java index 0806e41186395..f2633dfffb0fe 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java @@ -67,7 +67,7 @@ public void testUnauthorizedIndices() throws IOException { var getResponse = runAsyncGet("user1", id); // sanity assertOK(getResponse); ResponseException error; - error = expectThrows(ResponseException.class, () -> runAsyncGet("user2", id)); + error = expectThrows(ResponseException.class, () -> runAsyncGet("user2", id, true)); // resource not found exception if the authenticated user is not the creator of the original task assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(404)); @@ -85,7 +85,7 @@ public void testUnauthorizedIndices() throws IOException { var getResponse = runAsyncGet("user2", id); // sanity assertOK(getResponse); ResponseException error; - error = expectThrows(ResponseException.class, () -> runAsyncGet("user1", id)); + error = expectThrows(ResponseException.class, () -> runAsyncGet("user1", id, true)); assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(404)); error = expectThrows(ResponseException.class, () -> runAsyncDelete("user1", id)); @@ -117,6 +117,10 @@ private Response runAsync(String user, String command) throws IOException { } private Response runAsyncGet(String user, String id) throws IOException { + return runAsyncGet(user, id, false); + } + + private Response runAsyncGet(String user, String id, boolean isAsyncIdNotFound_Expected) throws IOException { int tries = 0; while (tries < 10) { // Sometimes we get 404s fetching the task status. @@ -129,22 +133,32 @@ private Response runAsyncGet(String user, String id) throws IOException { logResponse(response); return response; } catch (ResponseException e) { - if (e.getResponse().getStatusLine().getStatusCode() == 404 - && EntityUtils.toString(e.getResponse().getEntity()).contains("no such index [.async-search]")) { - /* - * Work around https://github.com/elastic/elasticsearch/issues/110304 - the .async-search - * index may not exist when we try the fetch, but it should exist on next attempt. - */ + var statusCode = e.getResponse().getStatusLine().getStatusCode(); + var message = EntityUtils.toString(e.getResponse().getEntity()); + + if (statusCode == 404 && message.contains("no such index [.async-search]")) { + // Work around https://github.com/elastic/elasticsearch/issues/110304 - the .async-search + // index may not exist when we try the fetch, but it should exist on next attempt. logger.warn("async-search index does not exist", e); try { Thread.sleep(1000); } catch (InterruptedException ex) { throw new RuntimeException(ex); } + } else if (statusCode == 404 && false == isAsyncIdNotFound_Expected && message.contains("resource_not_found_exception")) { + // Work around for https://github.com/elastic/elasticsearch/issues/112110 + // The async id is not indexed quickly enough in .async-search index for us to retrieve it. + logger.warn("async id not found", e); + try { + Thread.sleep(500); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } } else { throw e; } tries++; + logger.warn("retry [" + tries + "] for GET /_query/async/" + id); } } throw new IllegalStateException("couldn't find task status"); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java index e7bb054221c89..dab99a0f719dd 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.action; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; @@ -16,6 +17,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.operator.DriverProfile; import org.elasticsearch.compute.operator.DriverStatus; @@ -30,6 +32,9 @@ import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.protocol.xpack.XPackInfoResponse; import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.XPackSettings; @@ -43,6 +48,7 @@ import org.elasticsearch.xpack.enrich.EnrichPlugin; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.junit.After; @@ -82,6 +88,7 @@ protected Collection> nodePlugins() { plugins.add(IngestCommonPlugin.class); plugins.add(ReindexPlugin.class); plugins.add(InternalTransportSettingPlugin.class); + plugins.add(MockTransportService.TestPlugin.class); return plugins; } @@ -420,6 +427,24 @@ public void testManyDocuments() { } } + public void testRejection() { + for (var ts : internalCluster().getInstances(TransportService.class)) { + ((MockTransportService) ts).addRequestHandlingBehavior(EnrichLookupService.LOOKUP_ACTION_NAME, (h, r, channel, t) -> { + EsRejectedExecutionException ex = new EsRejectedExecutionException("test", false); + channel.sendResponse(new RemoteTransportException("test", ex)); + }); + } + try { + String query = "FROM listen* | " + enrichSongCommand(); + Exception error = expectThrows(Exception.class, () -> run(query).close()); + assertThat(ExceptionsHelper.status(error), equalTo(RestStatus.TOO_MANY_REQUESTS)); + } finally { + for (var ts : internalCluster().getInstances(TransportService.class)) { + ((MockTransportService) ts).clearAllRules(); + } + } + } + public static class LocalStateEnrich extends LocalStateCompositeXPackPlugin { public LocalStateEnrich(final Settings settings, final Path configPath) throws Exception { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java index fb598cb855013..1ce92ded8acc6 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java @@ -8,14 +8,24 @@ package org.elasticsearch.xpack.esql.action; import org.apache.lucene.tests.util.LuceneTestCase; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.compute.operator.exchange.ExchangeService; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.MockSearchService; import org.elasticsearch.search.SearchService; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.RemoteTransportException; +import org.elasticsearch.transport.TransportChannel; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.hamcrest.Matchers; import org.junit.Before; @@ -27,6 +37,10 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; /** * Make sures that we can run many concurrent requests with large number of shards with any data_partitioning. @@ -38,6 +52,7 @@ public class ManyShardsIT extends AbstractEsqlIntegTestCase { protected Collection> getMockPlugins() { var plugins = new ArrayList<>(super.getMockPlugins()); plugins.add(MockSearchService.TestPlugin.class); + plugins.add(MockTransportService.TestPlugin.class); return plugins; } @@ -97,6 +112,51 @@ public void testConcurrentQueries() throws Exception { } } + public void testRejection() throws Exception { + String[] nodes = internalCluster().getNodeNames(); + for (String node : nodes) { + MockTransportService ts = (MockTransportService) internalCluster().getInstance(TransportService.class, node); + ts.addRequestHandlingBehavior(ExchangeService.EXCHANGE_ACTION_NAME, (handler, request, channel, task) -> { + handler.messageReceived(request, new TransportChannel() { + @Override + public String getProfileName() { + return channel.getProfileName(); + } + + @Override + public void sendResponse(TransportResponse response) { + channel.sendResponse(new RemoteTransportException("simulated", new EsRejectedExecutionException("test queue"))); + } + + @Override + public void sendResponse(Exception exception) { + channel.sendResponse(exception); + } + }, task); + }); + } + try { + AtomicReference failure = new AtomicReference<>(); + EsqlQueryRequest request = new EsqlQueryRequest(); + request.query("from test-* | stats count(user) by tags"); + request.acceptedPragmaRisks(true); + request.pragmas(randomPragmas()); + CountDownLatch queryLatch = new CountDownLatch(1); + client().execute(EsqlQueryAction.INSTANCE, request, ActionListener.runAfter(ActionListener.wrap(r -> { + r.close(); + throw new AssertionError("expected failure"); + }, failure::set), queryLatch::countDown)); + assertTrue(queryLatch.await(10, TimeUnit.SECONDS)); + assertThat(failure.get(), instanceOf(EsRejectedExecutionException.class)); + assertThat(ExceptionsHelper.status(failure.get()), equalTo(RestStatus.TOO_MANY_REQUESTS)); + assertThat(failure.get().getMessage(), equalTo("test queue")); + } finally { + for (String node : nodes) { + ((MockTransportService) internalCluster().getInstance(TransportService.class, node)).clearAllRules(); + } + } + } + static class SearchContextCounter { private final int maxAllowed; private final AtomicInteger current = new AtomicInteger(); diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index 897bfa5e1ce15..6570a25469971 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -29,7 +29,7 @@ options { * * Since the tokens/modes are in development, simply define them under the * "// in development section" and follow the section comments in that section. - * That is use the DEV_ prefix and use the {isDevVersion()}? conditional. + * That is use the DEV_ prefix and use the {this.isDevVersion()}? conditional. * They are defined at the end of the file, to minimize the impact on the existing * token types. * @@ -80,15 +80,15 @@ WHERE : 'where' -> pushMode(EXPRESSION_MODE); // Before adding a new in-development command, to sandbox the behavior when running in production environments // // For example: to add myCommand use the following declaration: -// DEV_MYCOMMAND : {isDevVersion()}? 'mycommand' -> ... +// DEV_MYCOMMAND : {this.isDevVersion()}? 'mycommand' -> ... // // Once the command has been stabilized, remove the DEV_ prefix and the {}? conditional and move the command to the // main section while preserving alphabetical order: // MYCOMMAND : 'mycommand' -> ... -DEV_INLINESTATS : {isDevVersion()}? 'inlinestats' -> pushMode(EXPRESSION_MODE); -DEV_LOOKUP : {isDevVersion()}? 'lookup' -> pushMode(LOOKUP_MODE); -DEV_MATCH : {isDevVersion()}? 'match' -> pushMode(EXPRESSION_MODE); -DEV_METRICS : {isDevVersion()}? 'metrics' -> pushMode(METRICS_MODE); +DEV_INLINESTATS : {this.isDevVersion()}? 'inlinestats' -> pushMode(EXPRESSION_MODE); +DEV_LOOKUP : {this.isDevVersion()}? 'lookup' -> pushMode(LOOKUP_MODE); +DEV_MATCH : {this.isDevVersion()}? 'match' -> pushMode(EXPRESSION_MODE); +DEV_METRICS : {this.isDevVersion()}? 'metrics' -> pushMode(METRICS_MODE); // // Catch-all for unrecognized commands - don't define any beyond this line @@ -211,7 +211,7 @@ SLASH : '/'; PERCENT : '%'; // move it in the main section if the feature gets promoted -DEV_MATCH_OP : {isDevVersion()}? DEV_MATCH -> type(DEV_MATCH); +DEV_MATCH_OP : {this.isDevVersion()}? DEV_MATCH -> type(DEV_MATCH); NAMED_OR_POSITIONAL_PARAM : PARAM (LETTER | UNDERSCORE) UNQUOTED_ID_BODY* diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index ce748b3af03d1..a3ef2471d4e56 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -36,7 +36,7 @@ sourceCommand | rowCommand | showCommand // in development - | {isDevVersion()}? metricsCommand + | {this.isDevVersion()}? metricsCommand ; processingCommand @@ -53,9 +53,9 @@ processingCommand | enrichCommand | mvExpandCommand // in development - | {isDevVersion()}? inlinestatsCommand - | {isDevVersion()}? lookupCommand - | {isDevVersion()}? matchCommand + | {this.isDevVersion()}? inlinestatsCommand + | {this.isDevVersion()}? lookupCommand + | {this.isDevVersion()}? matchCommand ; whereCommand @@ -70,7 +70,7 @@ booleanExpression | left=booleanExpression operator=OR right=booleanExpression #logicalBinary | valueExpression (NOT)? IN LP valueExpression (COMMA valueExpression)* RP #logicalIn | valueExpression IS NOT? NULL #isNull - | {isDevVersion()}? matchBooleanExpression #matchExpression + | {this.isDevVersion()}? matchBooleanExpression #matchExpression ; regexBooleanExpression diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index 77d982453203c..af82ceb4bf809 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -12,16 +12,13 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.iterable.Iterables; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Grok; @@ -56,8 +53,6 @@ import java.util.Set; import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.Entry.of; -import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; -import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; /** * A utility class that consists solely of static methods that describe how to serialize and @@ -93,9 +88,9 @@ public static List namedTypeEntries() { // Physical Plan Nodes of(PhysicalPlan.class, AggregateExec.ENTRY), of(PhysicalPlan.class, DissectExec.ENTRY), - of(PhysicalPlan.class, EsQueryExec.class, PlanNamedTypes::writeEsQueryExec, PlanNamedTypes::readEsQueryExec), + of(PhysicalPlan.class, EsQueryExec.ENTRY), of(PhysicalPlan.class, EsSourceExec.ENTRY), - of(PhysicalPlan.class, EvalExec.class, PlanNamedTypes::writeEvalExec, PlanNamedTypes::readEvalExec), + of(PhysicalPlan.class, EvalExec.ENTRY), of(PhysicalPlan.class, EnrichExec.class, PlanNamedTypes::writeEnrichExec, PlanNamedTypes::readEnrichExec), of(PhysicalPlan.class, ExchangeExec.class, PlanNamedTypes::writeExchangeExec, PlanNamedTypes::readExchangeExec), of(PhysicalPlan.class, ExchangeSinkExec.class, PlanNamedTypes::writeExchangeSinkExec, PlanNamedTypes::readExchangeSinkExec), @@ -123,57 +118,6 @@ public static List namedTypeEntries() { } // -- physical plan nodes - static EsQueryExec readEsQueryExec(PlanStreamInput in) throws IOException { - return new EsQueryExec( - Source.readFrom(in), - new EsIndex(in), - readIndexMode(in), - in.readNamedWriteableCollectionAsList(Attribute.class), - in.readOptionalNamedWriteable(QueryBuilder.class), - in.readOptionalNamed(Expression.class), - in.readOptionalCollectionAsList(readerFromPlanReader(PlanNamedTypes::readFieldSort)), - in.readOptionalVInt() - ); - } - - static void writeEsQueryExec(PlanStreamOutput out, EsQueryExec esQueryExec) throws IOException { - assert esQueryExec.children().size() == 0; - Source.EMPTY.writeTo(out); - esQueryExec.index().writeTo(out); - writeIndexMode(out, esQueryExec.indexMode()); - out.writeNamedWriteableCollection(esQueryExec.output()); - out.writeOptionalNamedWriteable(esQueryExec.query()); - out.writeOptionalNamedWriteable(esQueryExec.limit()); - out.writeOptionalCollection(esQueryExec.sorts(), writerFromPlanWriter(PlanNamedTypes::writeFieldSort)); - out.writeOptionalInt(esQueryExec.estimatedRowSize()); - } - - public static IndexMode readIndexMode(StreamInput in) throws IOException { - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_TO_SOURCE)) { - return IndexMode.fromString(in.readString()); - } else { - return IndexMode.STANDARD; - } - } - - public static void writeIndexMode(StreamOutput out, IndexMode indexMode) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_TO_SOURCE)) { - out.writeString(indexMode.getName()); - } else if (indexMode != IndexMode.STANDARD) { - throw new IllegalStateException("not ready to support index mode [" + indexMode + "]"); - } - } - - static EvalExec readEvalExec(PlanStreamInput in) throws IOException { - return new EvalExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readCollectionAsList(Alias::new)); - } - - static void writeEvalExec(PlanStreamOutput out, EvalExec evalExec) throws IOException { - Source.EMPTY.writeTo(out); - out.writePhysicalPlanNode(evalExec.child()); - out.writeCollection(evalExec.fields()); - } - static EnrichExec readEnrichExec(PlanStreamInput in) throws IOException { final Source source = Source.readFrom(in); final PhysicalPlan child = in.readPhysicalPlanNode(); @@ -426,20 +370,4 @@ static void writeTopNExec(PlanStreamOutput out, TopNExec topNExec) throws IOExce out.writeNamedWriteable(topNExec.limit()); out.writeOptionalVInt(topNExec.estimatedRowSize()); } - - // -- ancillary supporting classes of plan nodes, etc - - static EsQueryExec.FieldSort readFieldSort(PlanStreamInput in) throws IOException { - return new EsQueryExec.FieldSort( - FieldAttribute.readFrom(in), - in.readEnum(Order.OrderDirection.class), - in.readEnum(Order.NullsPosition.class) - ); - } - - static void writeFieldSort(PlanStreamOutput out, EsQueryExec.FieldSort fieldSort) throws IOException { - fieldSort.field().writeTo(out); - out.writeEnum(fieldSort.direction()); - out.writeEnum(fieldSort.nulls()); - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index 5fc5ab20810a6..a746a0d49004f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -238,35 +238,35 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { private boolean DEV_INLINESTATS_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 0: - return isDevVersion(); + return this.isDevVersion(); } return true; } private boolean DEV_LOOKUP_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 1: - return isDevVersion(); + return this.isDevVersion(); } return true; } private boolean DEV_MATCH_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 2: - return isDevVersion(); + return this.isDevVersion(); } return true; } private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 3: - return isDevVersion(); + return this.isDevVersion(); } return true; } private boolean DEV_MATCH_OP_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 4: - return isDevVersion(); + return this.isDevVersion(); } return true; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index 359abbc701dd3..fb63e31a37c90 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -447,7 +447,7 @@ public final SourceCommandContext sourceCommand() throws RecognitionException { enterOuterAlt(_localctx, 6); { setState(141); - if (!(isDevVersion())) throw new FailedPredicateException(this, "isDevVersion()"); + if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); setState(142); metricsCommand(); } @@ -627,7 +627,7 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce enterOuterAlt(_localctx, 13); { setState(157); - if (!(isDevVersion())) throw new FailedPredicateException(this, "isDevVersion()"); + if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); setState(158); inlinestatsCommand(); } @@ -636,7 +636,7 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce enterOuterAlt(_localctx, 14); { setState(159); - if (!(isDevVersion())) throw new FailedPredicateException(this, "isDevVersion()"); + if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); setState(160); lookupCommand(); } @@ -645,7 +645,7 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce enterOuterAlt(_localctx, 15); { setState(161); - if (!(isDevVersion())) throw new FailedPredicateException(this, "isDevVersion()"); + if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); setState(162); matchCommand(); } @@ -1018,7 +1018,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; setState(196); - if (!(isDevVersion())) throw new FailedPredicateException(this, "isDevVersion()"); + if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); setState(197); matchBooleanExpression(); } @@ -5339,25 +5339,25 @@ private boolean query_sempred(QueryContext _localctx, int predIndex) { private boolean sourceCommand_sempred(SourceCommandContext _localctx, int predIndex) { switch (predIndex) { case 1: - return isDevVersion(); + return this.isDevVersion(); } return true; } private boolean processingCommand_sempred(ProcessingCommandContext _localctx, int predIndex) { switch (predIndex) { case 2: - return isDevVersion(); + return this.isDevVersion(); case 3: - return isDevVersion(); + return this.isDevVersion(); case 4: - return isDevVersion(); + return this.isDevVersion(); } return true; } private boolean booleanExpression_sempred(BooleanExpressionContext _localctx, int predIndex) { switch (predIndex) { case 5: - return isDevVersion(); + return this.isDevVersion(); case 6: return precpred(_ctx, 5); case 7: diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java index 56c253f166762..b080c425d2312 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.index.EsIndex; -import org.elasticsearch.xpack.esql.io.stream.PlanNamedTypes; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; @@ -67,7 +66,7 @@ private static EsRelation readFrom(StreamInput in) throws IOException { in.readOptionalString(); in.readOptionalString(); } - IndexMode indexMode = PlanNamedTypes.readIndexMode(in); + IndexMode indexMode = readIndexMode(in); boolean frozen = in.readBoolean(); return new EsRelation(source, esIndex, attributes, indexMode, frozen); } @@ -83,7 +82,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(null); out.writeOptionalString(null); } - PlanNamedTypes.writeIndexMode(out, indexMode()); + writeIndexMode(out, indexMode()); out.writeBoolean(frozen()); } @@ -174,4 +173,20 @@ public boolean equals(Object obj) { public String nodeString() { return nodeName() + "[" + index + "]" + NodeUtils.limitedToString(attrs); } + + public static IndexMode readIndexMode(StreamInput in) throws IOException { + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_TO_SOURCE)) { + return IndexMode.fromString(in.readString()); + } else { + return IndexMode.STANDARD; + } + } + + public static void writeIndexMode(StreamOutput out, IndexMode indexMode) throws IOException { + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_TO_SOURCE)) { + out.writeString(indexMode.getName()); + } else if (indexMode != IndexMode.STANDARD) { + throw new IllegalStateException("not ready to support index mode [" + indexMode + "]"); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java index 5901d42abbc82..21aa2cb7d1860 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java @@ -8,6 +8,10 @@ package org.elasticsearch.xpack.esql.plan.physical; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; @@ -22,12 +26,21 @@ import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Objects; public class EsQueryExec extends LeafExec implements EstimatesRowSize { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + PhysicalPlan.class, + "EsQueryExec", + EsQueryExec::new + ); + public static final EsField DOC_ID_FIELD = new EsField("_doc", DataType.DOC_DATA_TYPE, Map.of(), false); private final EsIndex index; @@ -43,7 +56,7 @@ public class EsQueryExec extends LeafExec implements EstimatesRowSize { */ private final Integer estimatedRowSize; - public record FieldSort(FieldAttribute field, Order.OrderDirection direction, Order.NullsPosition nulls) { + public record FieldSort(FieldAttribute field, Order.OrderDirection direction, Order.NullsPosition nulls) implements Writeable { public FieldSortBuilder fieldSortBuilder() { FieldSortBuilder builder = new FieldSortBuilder(field.name()); builder.order(Direction.from(direction).asOrder()); @@ -51,6 +64,21 @@ public FieldSortBuilder fieldSortBuilder() { builder.unmappedType(field.dataType().esType()); return builder; } + + private static FieldSort readFrom(StreamInput in) throws IOException { + return new EsQueryExec.FieldSort( + FieldAttribute.readFrom(in), + in.readEnum(Order.OrderDirection.class), + in.readEnum(Order.NullsPosition.class) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + field().writeTo(out); + out.writeEnum(direction()); + out.writeEnum(nulls()); + } } public EsQueryExec(Source source, EsIndex index, IndexMode indexMode, List attributes, QueryBuilder query) { @@ -77,6 +105,36 @@ public EsQueryExec( this.estimatedRowSize = estimatedRowSize; } + private EsQueryExec(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + new EsIndex(in), + EsRelation.readIndexMode(in), + in.readNamedWriteableCollectionAsList(Attribute.class), + in.readOptionalNamedWriteable(QueryBuilder.class), + in.readOptionalNamedWriteable(Expression.class), + in.readOptionalCollectionAsList(FieldSort::readFrom), + in.readOptionalVInt() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + index().writeTo(out); + EsRelation.writeIndexMode(out, indexMode()); + out.writeNamedWriteableCollection(output()); + out.writeOptionalNamedWriteable(query()); + out.writeOptionalNamedWriteable(limit()); + out.writeOptionalCollection(sorts()); + out.writeOptionalVInt(estimatedRowSize()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + public static boolean isSourceAttribute(Attribute attr) { return DOC_ID_FIELD.getName().equals(attr.name()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java index 275f1182ff97c..cd167b4683493 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java @@ -17,7 +17,6 @@ import org.elasticsearch.xpack.esql.core.tree.NodeUtils; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.index.EsIndex; -import org.elasticsearch.xpack.esql.io.stream.PlanNamedTypes; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; @@ -55,7 +54,7 @@ private EsSourceExec(StreamInput in) throws IOException { new EsIndex(in), in.readNamedWriteableCollectionAsList(Attribute.class), in.readOptionalNamedWriteable(QueryBuilder.class), - PlanNamedTypes.readIndexMode(in) + EsRelation.readIndexMode(in) ); } @@ -65,7 +64,7 @@ public void writeTo(StreamOutput out) throws IOException { index().writeTo(out); out.writeNamedWriteableCollection(output()); out.writeOptionalNamedWriteable(query()); - PlanNamedTypes.writeIndexMode(out, indexMode()); + EsRelation.writeIndexMode(out, indexMode()); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EvalExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EvalExec.java index 3876891b27752..97b81914f8889 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EvalExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EvalExec.java @@ -7,17 +7,29 @@ package org.elasticsearch.xpack.esql.plan.physical; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import java.io.IOException; import java.util.List; import java.util.Objects; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; public class EvalExec extends UnaryExec implements EstimatesRowSize { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + PhysicalPlan.class, + "EvalExec", + EvalExec::new + ); + private final List fields; public EvalExec(Source source, PhysicalPlan child, List fields) { @@ -25,6 +37,22 @@ public EvalExec(Source source, PhysicalPlan child, List fields) { this.fields = fields; } + private EvalExec(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), ((PlanStreamInput) in).readPhysicalPlanNode(), in.readCollectionAsList(Alias::new)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + ((PlanStreamOutput) out).writePhysicalPlanNode(child()); + out.writeCollection(fields()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + public List fields() { return fields; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/PhysicalPlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/PhysicalPlan.java index 42a97802038a2..60e44a5140dfa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/PhysicalPlan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/PhysicalPlan.java @@ -23,7 +23,7 @@ */ public abstract class PhysicalPlan extends QueryPlan { public static List getNamedWriteables() { - return List.of(AggregateExec.ENTRY, DissectExec.ENTRY, EsSourceExec.ENTRY); + return List.of(AggregateExec.ENTRY, DissectExec.ENTRY, EsQueryExec.ENTRY, EsSourceExec.ENTRY, EvalExec.ENTRY); } public PhysicalPlan(Source source, List children) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java index 56ab1bd41693e..a3d1e70e558d6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java @@ -20,24 +20,9 @@ import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.Nullability; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.ArithmeticOperation; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; -import org.elasticsearch.xpack.esql.core.type.KeywordEsField; -import org.elasticsearch.xpack.esql.expression.Order; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.DissectExec; import org.elasticsearch.xpack.esql.plan.physical.EnrichExec; @@ -137,15 +122,6 @@ public void testWrappedStreamSimple() throws IOException { assertThat(in.readVInt(), equalTo(11_345)); } - public void testFieldSortSimple() throws IOException { - var orig = new EsQueryExec.FieldSort(field("val", DataType.LONG), Order.OrderDirection.ASC, Order.NullsPosition.FIRST); - BytesStreamOutput bso = new BytesStreamOutput(); - PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null); - PlanNamedTypes.writeFieldSort(out, orig); - var deser = PlanNamedTypes.readFieldSort(planStreamInput(bso)); - EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); - } - static FieldAttribute randomFieldAttributeOrNull() { return randomBoolean() ? randomFieldAttribute() : null; } @@ -163,46 +139,6 @@ static FieldAttribute randomFieldAttribute() { ); } - static KeywordEsField randomKeywordEsField() { - return new KeywordEsField( - randomAlphaOfLength(randomIntBetween(1, 25)), // name - randomProperties(), - randomBoolean(), // hasDocValues - randomIntBetween(1, 12), // precision - randomBoolean(), // normalized - randomBoolean() // alias - ); - } - - static EsqlBinaryComparison randomBinaryComparison() { - int v = randomIntBetween(0, 5); - var left = field(randomName(), randomDataType()); - var right = field(randomName(), randomDataType()); - return switch (v) { - case 0 -> new Equals(Source.EMPTY, left, right); - case 1 -> new NotEquals(Source.EMPTY, left, right); - case 2 -> new GreaterThan(Source.EMPTY, left, right); - case 3 -> new GreaterThanOrEqual(Source.EMPTY, left, right); - case 4 -> new LessThan(Source.EMPTY, left, right); - case 5 -> new LessThanOrEqual(Source.EMPTY, left, right); - default -> throw new AssertionError(v); - }; - } - - static ArithmeticOperation randomArithmeticOperation() { - int v = randomIntBetween(0, 4); - var left = field(randomName(), randomDataType()); - var right = field(randomName(), randomDataType()); - return switch (v) { - case 0 -> new Add(Source.EMPTY, left, right); - case 1 -> new Sub(Source.EMPTY, left, right); - case 2 -> new Mul(Source.EMPTY, left, right); - case 3 -> new Div(Source.EMPTY, left, right); - case 4 -> new Mod(Source.EMPTY, left, right); - default -> throw new AssertionError(v); - }; - } - static NameId nameIdOrNull() { return randomBoolean() ? new NameId() : null; } @@ -231,10 +167,6 @@ static EsField randomEsField(int depth) { ); } - static Map randomProperties() { - return randomProperties(0); - } - static Map randomProperties(int depth) { if (depth > 2) { return Map.of(); // prevent infinite recursion (between EsField and properties) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/AbstractPhysicalPlanSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/AbstractPhysicalPlanSerializationTests.java index 7a0d125ad85ba..2a05c472328e5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/AbstractPhysicalPlanSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/AbstractPhysicalPlanSerializationTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.esql.plan.AbstractNodeSerializationTests; import java.util.ArrayList; @@ -47,7 +48,8 @@ protected final NamedWriteableRegistry getNamedWriteableRegistry() { entries.addAll(Attribute.getNamedWriteables()); entries.addAll(Block.getNamedWriteables()); entries.addAll(NamedExpression.getNamedWriteables()); - entries.addAll(new SearchModule(Settings.EMPTY, List.of()).getNamedWriteables()); + entries.addAll(new SearchModule(Settings.EMPTY, List.of()).getNamedWriteables()); // Query builders + entries.add(Add.ENTRY); // Used by the eval tests return new NamedWriteableRegistry(entries); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExecSerializationTests.java new file mode 100644 index 0000000000000..6bb5111b154e6 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExecSerializationTests.java @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plan.physical; + +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.Order; +import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests; +import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.EsIndexSerializationTests; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.plan.logical.AbstractLogicalPlanSerializationTests.randomFieldAttributes; + +public class EsQueryExecSerializationTests extends AbstractPhysicalPlanSerializationTests { + public static EsQueryExec randomEsQueryExec() { + Source source = randomSource(); + EsIndex index = EsIndexSerializationTests.randomEsIndex(); + IndexMode indexMode = randomFrom(IndexMode.values()); + List attrs = randomFieldAttributes(1, 10, false); + QueryBuilder query = randomQuery(); + Expression limit = new Literal(randomSource(), between(0, Integer.MAX_VALUE), DataType.INTEGER); + List sorts = randomFieldSorts(); + Integer estimatedRowSize = randomEstimatedRowSize(); + return new EsQueryExec(source, index, indexMode, attrs, query, limit, sorts, estimatedRowSize); + } + + public static QueryBuilder randomQuery() { + return randomBoolean() ? new MatchAllQueryBuilder() : new TermQueryBuilder(randomAlphaOfLength(4), randomAlphaOfLength(4)); + } + + public static List randomFieldSorts() { + return randomList(0, 4, EsQueryExecSerializationTests::randomFieldSort); + } + + public static EsQueryExec.FieldSort randomFieldSort() { + FieldAttribute field = FieldAttributeTests.createFieldAttribute(0, false); + Order.OrderDirection direction = randomFrom(Order.OrderDirection.values()); + Order.NullsPosition nulls = randomFrom(Order.NullsPosition.values()); + return new EsQueryExec.FieldSort(field, direction, nulls); + } + + @Override + protected EsQueryExec createTestInstance() { + return randomEsQueryExec(); + } + + @Override + protected EsQueryExec mutateInstance(EsQueryExec instance) throws IOException { + EsIndex index = instance.index(); + IndexMode indexMode = instance.indexMode(); + List attrs = instance.attrs(); + QueryBuilder query = instance.query(); + Expression limit = instance.limit(); + List sorts = instance.sorts(); + Integer estimatedRowSize = instance.estimatedRowSize(); + switch (between(0, 6)) { + case 0 -> index = randomValueOtherThan(index, EsIndexSerializationTests::randomEsIndex); + case 1 -> indexMode = randomValueOtherThan(indexMode, () -> randomFrom(IndexMode.values())); + case 2 -> attrs = randomValueOtherThan(attrs, () -> randomFieldAttributes(1, 10, false)); + case 3 -> query = randomValueOtherThan(query, EsQueryExecSerializationTests::randomQuery); + case 4 -> limit = randomValueOtherThan( + limit, + () -> new Literal(randomSource(), between(0, Integer.MAX_VALUE), DataType.INTEGER) + ); + case 5 -> sorts = randomValueOtherThan(sorts, EsQueryExecSerializationTests::randomFieldSorts); + case 6 -> estimatedRowSize = randomValueOtherThan( + estimatedRowSize, + AbstractPhysicalPlanSerializationTests::randomEstimatedRowSize + ); + } + return new EsQueryExec(instance.source(), index, indexMode, attrs, query, limit, sorts, estimatedRowSize); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EvalExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EvalExecSerializationTests.java new file mode 100644 index 0000000000000..45baf4822b1d2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EvalExecSerializationTests.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plan.physical; + +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; + +import java.io.IOException; +import java.util.List; + +public class EvalExecSerializationTests extends AbstractPhysicalPlanSerializationTests { + public static EvalExec randomEvalExec(int depth) { + Source source = randomSource(); + PhysicalPlan child = randomChild(depth); + List fields = randomFields(); + return new EvalExec(source, child, fields); + } + + public static List randomFields() { + return randomList(1, 10, EvalExecSerializationTests::randomField); + } + + public static Alias randomField() { + Expression child = new Add( + randomSource(), + FieldAttributeTests.createFieldAttribute(0, true), + FieldAttributeTests.createFieldAttribute(0, true) + ); + return new Alias(randomSource(), randomAlphaOfLength(5), child); + } + + @Override + protected EvalExec createTestInstance() { + return randomEvalExec(0); + } + + @Override + protected EvalExec mutateInstance(EvalExec instance) throws IOException { + PhysicalPlan child = instance.child(); + List fields = instance.fields(); + if (randomBoolean()) { + child = randomValueOtherThan(child, () -> randomChild(0)); + } else { + fields = randomValueOtherThan(fields, EvalExecSerializationTests::randomFields); + } + return new EvalExec(instance.source(), child, fields); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java index 73897fc38633a..fb74631970813 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java @@ -13,13 +13,16 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSourceField; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.core.XPackSettings; +import org.junit.BeforeClass; import java.util.ArrayList; import java.util.Collections; import java.util.List; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @@ -29,9 +32,12 @@ public class DocumentLevelSecurityRandomTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; - // can't add a second test method, because each test run creates a new instance of this class and that will will result - // in a new random value: - private final int numberOfRoles = scaledRandomIntBetween(3, 99); + private static volatile int numberOfRoles; + + @BeforeClass + public static void setupRoleCount() throws Exception { + numberOfRoles = scaledRandomIntBetween(3, 99); + } @Override protected String configUsers() { @@ -119,4 +125,38 @@ public void testDuelWithAliasFilters() throws Exception { } } + public void testWithRuntimeFields() throws Exception { + assertAcked( + indicesAdmin().prepareCreate("test") + .setMapping( + XContentFactory.jsonBuilder() + .startObject() + .startObject("runtime") + .startObject("field1") + .field("type", "keyword") + .endObject() + .endObject() + .startObject("properties") + .startObject("field2") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + ) + ); + List requests = new ArrayList<>(47); + for (int i = 1; i <= 42; i++) { + requests.add(prepareIndex("test").setSource("field1", "value1", "field2", "foo" + i)); + } + for (int i = 42; i <= 57; i++) { + requests.add(prepareIndex("test").setSource("field1", "value2", "field2", "foo" + i)); + } + indexRandom(true, requests); + assertHitCount( + client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD))) + .prepareSearch("test"), + 42L + ); + } + }