Merge branch 'main' into query-roles

docs/changelog/109470.yaml

@@ -0,0 +1,5 @@
+pr: 109470
+summary: Enabling profiling for `RankBuilders` and adding tests for RRF
+area: Ranking
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java

@@ -223,6 +223,8 @@ public static void performOnPrimary(
 
             final long startBulkTime = System.nanoTime();
 
+            private final ActionListener<Void> onMappingUpdateDone = ActionListener.wrap(v -> executor.execute(this), this::onRejection);
+
             @Override
             protected void doRun() throws Exception {
                 while (context.hasMoreOperationsToExecute()) {
@@ -232,8 +234,7 @@ protected void doRun() throws Exception {
                         nowInMillisSupplier,
                         mappingUpdater,
                         waitForMappingUpdate,
-
-                        ActionListener.wrap(v -> executor.execute(this), this::onRejection),
+                        onMappingUpdateDone,
                         documentParsingProvider
                     ) == false) {
                         // We are waiting for a mapping update on another thread, that will invoke this action again once its done

server/src/main/java/org/elasticsearch/action/search/SearchRequest.java

@@ -434,9 +434,6 @@ public ActionRequestValidationException validate() {
                 if (source.pointInTimeBuilder() != null) {
                     validationException = addValidationError("[rank] cannot be used with [point in time]", validationException);
                 }
-                if (source.profile()) {
-                    validationException = addValidationError("[rank] requires [profile] is [false]", validationException);
-                }
             }
             if (source.rescores() != null) {
                 for (@SuppressWarnings("rawtypes")

server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java

@@ -517,18 +517,6 @@ public void testValidate() throws IOException {
             assertEquals(1, validationErrors.validationErrors().size());
             assertEquals("[rank] cannot be used with [point in time]", validationErrors.validationErrors().get(0));
         }
-        {
-            SearchRequest searchRequest = new SearchRequest().source(
-                new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100))
-                    .query(QueryBuilders.termQuery("field", "term"))
-                    .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null)))
-                    .profile(true)
-            );
-            ActionRequestValidationException validationErrors = searchRequest.validate();
-            assertNotNull(validationErrors);
-            assertEquals(1, validationErrors.validationErrors().size());
-            assertEquals("[rank] requires [profile] is [false]", validationErrors.validationErrors().get(0));
-        }
         {
             SearchRequest searchRequest = new SearchRequest("test").source(
                 new SearchSourceBuilder().pointInTimeBuilder(new PointInTimeBuilder(BytesArray.EMPTY))

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java

@@ -191,6 +191,7 @@ private void createIndex(String index, String alias, boolean isTimeSeries) throw
         createIndexWithSettings(client(), index, alias, settings, mapping);
     }
 
+    @TestLogging(value = "org.elasticsearch.xpack.ilm:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/105437")
     public void testRollupIndex() throws Exception {
         createIndex(index, alias, true);
         index(client(), index, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5));

x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml

@@ -0,0 +1,218 @@
+setup:
+  - requires:
+      cluster_features: "gte_v8.15.0"
+      reason: 'profile for rrf was enabled in 8.15'
+      test_runner_features: close_to
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              text:
+                type: text
+              integer:
+                type: integer
+              vector:
+                type: dense_vector
+                dims: 1
+                index: true
+                similarity: l2_norm
+                index_options:
+                  type: hnsw
+                  ef_construction: 100
+                  m: 16
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          text: "term"
+          integer: 1
+          vector: [5]
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          text: "term term"
+          integer: 2
+          vector: [4]
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          text: "term term term"
+          integer: 3
+          vector: [3]
+  - do:
+      index:
+        index: test
+        id: "4"
+        body:
+          text: "term term term term"
+          integer: 3
+
+  - do:
+      index:
+        index: test
+        id: "5"
+        body:
+          integer: 1
+          vector: [0]
+
+  - do:
+      indices.refresh: {}
+
+---
+"profile standard and knn query":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          retriever:
+            rrf:
+              retrievers: [
+                {
+                  standard: {
+                    query: {
+                      term: {
+                        text: "term"
+                      }
+                    }
+                  }
+                },
+                {
+                  standard: {
+                    query: {
+                      knn: {
+                        field: "vector",
+                        query_vector: [ 3 ],
+                        num_candidates: 5
+                      }
+                    }
+                  }
+                }
+              ]
+              rank_window_size: 5
+              rank_constant: 1
+          size: 3
+          profile: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - not_exists: profile.shards.0.dfs
+  - match: { profile.shards.0.searches.0.query.0.type: ConstantScoreQuery }
+  - length: { profile.shards.0.searches.0.query.0.children: 1 }
+  - match: { profile.shards.0.searches.0.query.0.children.0.type: BooleanQuery }
+  - length: { profile.shards.0.searches.0.query.0.children.0.children: 2 }
+  - match: { profile.shards.0.searches.0.query.0.children.0.children.0.type: TermQuery }
+  - match: { profile.shards.0.searches.0.query.0.children.0.children.1.type: DocAndScoreQuery }
+
+---
+"profile standard and knn dfs retrievers":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          retriever:
+            rrf:
+              retrievers: [
+                {
+                  standard: {
+                    query: {
+                      term: {
+                        text: "term"
+                      }
+                    }
+                  }
+                },
+                {
+                  knn: {
+                      field: "vector",
+                      query_vector: [ 3 ],
+                      num_candidates: 5,
+                      k: 5
+                  }
+                }
+              ]
+              rank_window_size: 5
+              rank_constant: 1
+          size: 3
+          profile: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - exists: profile.shards.0.dfs
+  - length: { profile.shards.0.dfs.knn: 1 }
+  - length: { profile.shards.0.dfs.knn.0.query: 1 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.type: DocAndScoreQuery }
+
+  - match: { profile.shards.0.searches.0.query.0.type: ConstantScoreQuery }
+  - length: { profile.shards.0.searches.0.query.0.children: 1 }
+  - match: { profile.shards.0.searches.0.query.0.children.0.type: BooleanQuery }
+  - length: { profile.shards.0.searches.0.query.0.children.0.children: 2 }
+  - match: { profile.shards.0.searches.0.query.0.children.0.children.0.type: TermQuery }
+  - match: { profile.shards.0.searches.0.query.0.children.0.children.1.type: KnnScoreDocQuery }
+
+---
+"using query and dfs knn search":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          query: {
+            term: {
+              text: {
+                value: "term"
+              }
+            }
+          }
+          knn: {
+            field: "vector",
+            query_vector: [ 3 ],
+            num_candidates: 5,
+            k: 5
+          }
+          rank: {
+            rrf: {
+              rank_window_size: 5,
+              rank_constant: 1
+            }
+          }
+          size: 3
+          profile: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - exists: profile.shards.0.dfs
+  - length: { profile.shards.0.dfs.knn: 1 }
+  - length: { profile.shards.0.dfs.knn.0.query: 1 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.type: DocAndScoreQuery }
+
+  - match: { profile.shards.0.searches.0.query.0.type: ConstantScoreQuery }
+  - length: { profile.shards.0.searches.0.query.0.children: 1 }
+  - match: { profile.shards.0.searches.0.query.0.children.0.type: BooleanQuery }
+  - length: { profile.shards.0.searches.0.query.0.children.0.children: 2 }
+  - match: { profile.shards.0.searches.0.query.0.children.0.children.0.type: TermQuery }
+  - match: { profile.shards.0.searches.0.query.0.children.0.children.1.type: KnnScoreDocQuery }