Merge branch 'main' into security-tokens-no-index-refresh

build-tools-internal/version.properties

@@ -2,7 +2,7 @@ elasticsearch     = 8.10.0
 lucene            = 9.7.0
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 20.0.1+9@b4887098932d415489976708ad6d1a4b
+bundled_jdk = 20.0.2+9@6e380f22cbe7469fa75fb448bd903d8e
 
 # optional dependencies
 spatial4j         = 0.7

docs/changelog/97587.yaml

@@ -0,0 +1,5 @@
+pr: 97587
+summary: Fix `sub_searches` serialization bug
+area: Ranking
+type: bug
+issues: []

docs/changelog/97726.yaml

@@ -0,0 +1,5 @@
+pr: 97726
+summary: Wire `QueryPhaseCollectorManager` into the query phase
+area: Search
+type: enhancement
+issues: []

docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc

@@ -77,6 +77,8 @@ https://static.googleusercontent.com/media/research.google.com/fr//pubs/archive/
 algorithm, which counts based on the hashes of the values with some interesting
 properties:
 
+// tag::explanation[]
+
  * configurable precision, which decides on how to trade memory for accuracy,
  * excellent accuracy on low-cardinality sets,
  * fixed memory usage: no matter if there are tens or billions of unique values,
@@ -157,9 +159,11 @@ accuracy. Also note that even with a threshold as low as 100, the error
 remains very low (1-6% as seen in the above graph) even when counting millions of items.
 
 The HyperLogLog++ algorithm depends on the leading zeros of hashed
-values, the exact distributions of hashes in a dataset can affect the 
+values, the exact distributions of hashes in a dataset can affect the
 accuracy of the cardinality.
 
+// end::explanation[]
+
 ==== Pre-computed hashes
 
 On string fields that have a high cardinality, it might be faster to store the
@@ -249,7 +253,7 @@ There are different mechanisms by which cardinality aggregations can be executed
 
 Additionally, there are two "heuristic based" modes.  These modes will cause
 Elasticsearch to use some data about the state of the index to choose an
-appropriate execution method.  The two heuristics are: 
+appropriate execution method.  The two heuristics are:
  - `save_time_heuristic` - this is the default in Elasticsearch 8.4 and later.
  - `save_memory_heuristic` - this was the default in Elasticsearch 8.3 and
    earlier

docs/reference/aggregations/metrics/percentile-aggregation.asciidoc

@@ -234,6 +234,7 @@ GET latency/_search
 [[search-aggregations-metrics-percentile-aggregation-approximation]]
 ==== Percentiles are (usually) approximate
 
+// tag::approximate[]
 There are many different algorithms to calculate percentiles. The naive
 implementation simply stores all the values in a sorted array. To find the 50th
 percentile, you simply find the value that is at `my_array[count(my_array) * 0.5]`.
@@ -268,6 +269,8 @@ for large number of values is that the law of large numbers makes the distributi
 values more and more uniform and the t-digest tree can do a better job at summarizing
 it. It would not be the case on more skewed distributions.
 
+// end::approximate[]
+
 [WARNING]
 ====
 Percentile aggregations are also

docs/reference/setup/install/check-running.asciidoc

@@ -40,10 +40,8 @@ GET /
     "build_date" : "2016-03-30T09:51:41.449Z",
     "build_snapshot" : false,
     "lucene_version" : "{lucene_version}",
-    "index_version" : "67890",
     "minimum_wire_compatibility_version" : "1.2.3",
-    "minimum_index_compatibility_version" : "1.2.3",
-    "transport_version" : "12345"
+    "minimum_index_compatibility_version" : "1.2.3"
   },
   "tagline" : "You Know, for Search"
 }
@@ -56,6 +54,4 @@ GET /
 // TESTRESPONSE[s/"build_snapshot" : false,/"build_snapshot" : $body.version.build_snapshot,/]
 // TESTRESPONSE[s/"minimum_wire_compatibility_version" : "1.2.3"/"minimum_wire_compatibility_version" : $body.version.minimum_wire_compatibility_version/]
 // TESTRESPONSE[s/"minimum_index_compatibility_version" : "1.2.3"/"minimum_index_compatibility_version" : $body.version.minimum_index_compatibility_version/]
-// TESTRESPONSE[s/"transport_version" : "12345"/"transport_version" : $body.version.transport_version/]
-// TESTRESPONSE[s/"index_version" : "67890"/"index_version" : $body.version.index_version/]
 // So much s/// but at least we test that the layout is close to matching....

gradle/verification-metadata.xml

@@ -1559,25 +1559,25 @@
             <sha256 value="0a4aa0f2606bd6292a7ccd7d67a0db914bf5874dfb8a6184df3e6d63cdc93702" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="openjdk_20" name="linux" version="20.0.1">
-         <artifact name="linux-20.0.1-aarch64.tar.gz">
-            <sha256 value="f1935fb2046bf5cfb609522f243909ee8e47e6be3db925df40c46f0465b30434" origin="Generated by Gradle"/>
+      <component group="openjdk_20" name="linux" version="20.0.2">
+         <artifact name="linux-20.0.2-aarch64.tar.gz">
+            <sha256 value="3238c93267c663dbca00f5d5b0e3fbba40e1eea2b4281612f40542d208b6dd9a" origin="Generated by Gradle"/>
          </artifact>
-         <artifact name="linux-20.0.1-x64.tar.gz">
-            <sha256 value="4248a3af4602dbe2aefdb7010bc9086bf34a4155888e837649c90ff6d8e8cef9" origin="Generated by Gradle"/>
+         <artifact name="linux-20.0.2-x64.tar.gz">
+            <sha256 value="beaf61959c2953310595e1162b0c626aef33d58628771033ff2936609661956c" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="openjdk_20" name="macos" version="20.0.1">
-         <artifact name="macos-20.0.1-aarch64.tar.gz">
-            <sha256 value="78ae5bb4c96632df8d3f776919c95653d1afd3e715981c4d33be5b3c81d05420" origin="Generated by Gradle"/>
+      <component group="openjdk_20" name="macos" version="20.0.2">
+         <artifact name="macos-20.0.2-aarch64.tar.gz">
+            <sha256 value="2e6522bb574f76cd3f81156acd59115a014bf452bbe4107f0d31ff9b41b3da57" origin="Generated by Gradle"/>
          </artifact>
-         <artifact name="macos-20.0.1-x64.tar.gz">
-            <sha256 value="215a181fda2ac9f33d8262476eba6c9beb0ae20d2b592e03411fe71a7d89bb24" origin="Generated by Gradle"/>
+         <artifact name="macos-20.0.2-x64.tar.gz">
+            <sha256 value="c65ba92b73d8076e2a10029a0674d40ce45c3e0183a8063dd51281e92c9f43fc" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="openjdk_20" name="windows" version="20.0.1">
-         <artifact name="windows-20.0.1-x64.zip">
-            <sha256 value="31ca4a8cbdea1da7fb441194e756dd1adbedfc05bd1135a1ecc46b4288ea8942" origin="Generated by Gradle"/>
+      <component group="openjdk_20" name="windows" version="20.0.2">
+         <artifact name="windows-20.0.2-x64.zip">
+            <sha256 value="7e5870fd2e19b87cbd1981c4ff7203897384c2eb104977f40ce4951b40ab433e" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.abego.treelayout" name="org.abego.treelayout.core" version="1.0.3">

modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainResponse.java

@@ -29,8 +29,7 @@ public class MainResponse extends ActionResponse implements ToXContentObject {
 
     private String nodeName;
     private Version version;
-    private IndexVersion indexVersion;
-    private TransportVersion transportVersion;
+    private String luceneVersion;
     private ClusterName clusterName;
     private String clusterUuid;
     private Build build;
@@ -41,26 +40,35 @@ public class MainResponse extends ActionResponse implements ToXContentObject {
         super(in);
         nodeName = in.readString();
         version = Version.readVersion(in);
-        indexVersion = in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_031) ? IndexVersion.readVersion(in) : null;
-        transportVersion = in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_019) ? TransportVersion.readVersion(in) : null;
+
+        // Index version and transport version were briefly included in the main response, but
+        // removed before the 8.9.0 release. Reading code remains here (throwing away the values)
+        // for those versions until the new format has propagated through serverless. Additionally,
+        // the lucene version was previously read by inferring from either Version or IndexVersion.
+        // Now the lucene version is read explicitly.
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_037)) {
+            luceneVersion = in.readString();
+        } else {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_031)) {
+                luceneVersion = IndexVersion.readVersion(in).luceneVersion().toString();
+            } else if (version.before(Version.V_8_10_0)) {
+                luceneVersion = IndexVersion.fromId(version.id).luceneVersion().toString();
+            } else {
+                luceneVersion = "unknown";
+            }
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_500_019)) {
+                TransportVersion.readVersion(in);
+            }
+        }
         clusterName = new ClusterName(in);
         clusterUuid = in.readString();
         build = Build.readBuild(in);
     }
 
-    public MainResponse(
-        String nodeName,
-        Version version,
-        IndexVersion indexVersion,
-        TransportVersion transportVersion,
-        ClusterName clusterName,
-        String clusterUuid,
-        Build build
-    ) {
+    public MainResponse(String nodeName, Version version, String luceneVersion, ClusterName clusterName, String clusterUuid, Build build) {
         this.nodeName = nodeName;
         this.version = version;
-        this.indexVersion = indexVersion;
-        this.transportVersion = transportVersion;
+        this.luceneVersion = luceneVersion;
         this.clusterName = clusterName;
         this.clusterUuid = clusterUuid;
         this.build = build;
@@ -74,12 +82,8 @@ public Version getVersion() {
         return version;
     }
 
-    public IndexVersion getIndexVersion() {
-        return indexVersion;
-    }
-
-    public TransportVersion getTransportVersion() {
-        return transportVersion;
+    public String getLuceneVersion() {
+        return luceneVersion;
     }
 
     public ClusterName getClusterName() {
@@ -98,27 +102,27 @@ public Build getBuild() {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(nodeName);
         Version.writeVersion(version, out);
-        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_031)) {
-            IndexVersion.writeVersion(indexVersion, out);
-        }
-        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_019)) {
-            TransportVersion.writeVersion(transportVersion, out);
+
+        // Index version and transport version were briefly included in the main response, but
+        // removed before the 8.9.0 release. Writing code remains here (writing the latest versions)
+        // for those versions until the new format has propagated through serverless. Additionally,
+        // the lucene version was previously inferred from either Version or IndexVersion.
+        // Now the lucene version is written explicitly.
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_037)) {
+            out.writeString(luceneVersion);
+        } else {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_031)) {
+                IndexVersion.writeVersion(IndexVersion.current(), out);
+            }
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_500_019)) {
+                TransportVersion.writeVersion(TransportVersion.current(), out);
+            }
         }
         clusterName.writeTo(out);
         out.writeString(clusterUuid);
         Build.writeBuild(build, out);
     }
 
-    private String getLuceneVersion() {
-        if (indexVersion != null) {
-            return indexVersion.luceneVersion().toString();
-        } else if (version.before(Version.V_8_10_0)) {
-            return IndexVersion.fromId(version.id).luceneVersion().toString();
-        } else {
-            return "unknown";
-        }
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -132,11 +136,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             .field("build_hash", build.hash())
             .field("build_date", build.date())
             .field("build_snapshot", build.isSnapshot())
-            .field("lucene_version", getLuceneVersion())
-            .field("index_version", indexVersion != null ? indexVersion.toString() : "unknown")
+            .field("lucene_version", luceneVersion)
             .field("minimum_wire_compatibility_version", version.minimumCompatibilityVersion().toString())
             .field("minimum_index_compatibility_version", version.minimumIndexCompatibilityVersion().toString())
-            .field("transport_version", transportVersion != null ? transportVersion.toString() : "unknown")
             .endObject();
         builder.field("tagline", "You Know, for Search");
         builder.endObject();
@@ -170,16 +172,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             response.version = Version.fromString(
                 ((String) value.get("number")).replace("-SNAPSHOT", "").replaceFirst("-(alpha\\d+|beta\\d+|rc\\d+)", "")
             );
-
-            String indexVersion = (String) value.get("index_version");
-            response.indexVersion = indexVersion != null && indexVersion.equals("unknown") == false
-                ? IndexVersion.fromId(Integer.parseInt(indexVersion))
-                : null;
-
-            String transportVersion = (String) value.get("transport_version");
-            response.transportVersion = transportVersion != null && transportVersion.equals("unknown") == false
-                ? TransportVersion.fromString(transportVersion)
-                : null;
+            response.luceneVersion = ((String) value.get("lucene_version"));
         }, (parser, context) -> parser.map(), new ParseField("version"));
     }
 
@@ -198,16 +191,15 @@ public boolean equals(Object o) {
         MainResponse other = (MainResponse) o;
         return Objects.equals(nodeName, other.nodeName)
             && Objects.equals(version, other.version)
-            && Objects.equals(indexVersion, other.indexVersion)
-            && Objects.equals(transportVersion, other.transportVersion)
+            && Objects.equals(luceneVersion, other.luceneVersion)
             && Objects.equals(clusterUuid, other.clusterUuid)
             && Objects.equals(build, other.build)
             && Objects.equals(clusterName, other.clusterName);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(nodeName, version, indexVersion, transportVersion, clusterUuid, build, clusterName);
+        return Objects.hash(nodeName, version, luceneVersion, clusterUuid, build, clusterName);
     }
 
     @Override
@@ -218,10 +210,8 @@ public String toString() {
             + '\''
             + ", version="
             + version
-            + ", indexVersion="
-            + indexVersion
-            + ", transportVersion="
-            + transportVersion
+            + ", luceneVersion="
+            + luceneVersion
             + ", clusterName="
             + clusterName
             + ", clusterUuid='"

modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java

@@ -9,7 +9,6 @@
 package org.elasticsearch.rest.root;
 
 import org.elasticsearch.Build;
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
@@ -47,8 +46,7 @@ protected void doExecute(Task task, MainRequest request, ActionListener<MainResp
             new MainResponse(
                 nodeName,
                 Version.CURRENT,
-                IndexVersion.current(),
-                TransportVersion.current(),
+                IndexVersion.current().luceneVersion().toString(),
                 clusterState.getClusterName(),
                 clusterState.metadata().clusterUUID(),
                 Build.current()