From dcaba064dd7ca49f2786b634ab4e86c490cf6152 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 18 Oct 2023 07:20:01 +0200 Subject: [PATCH 01/23] Remove more explicit SearchResponse references from test code (#100985) Follow-up to #100966 adding more overrides to assertions that consume a request builder. --- .../bucket/TimeSeriesAggregationsIT.java | 18 ++-- .../datastreams/DataStreamIT.java | 18 ++-- .../script/expression/MoreExpressionIT.java | 4 +- .../join/query/ChildQuerySearchIT.java | 49 +++++------ .../elasticsearch/join/query/InnerHitsIT.java | 28 +++--- .../admin/cluster/node/tasks/TasksIT.java | 3 +- .../indices/delete/DeleteIndexBlocksIT.java | 8 +- .../allocation/decider/MockDiskUsagesIT.java | 6 +- .../index/WaitUntilRefreshIT.java | 16 ++-- .../index/mapper/DynamicMappingIT.java | 20 +++-- .../indices/stats/IndexStatsIT.java | 14 +-- .../elasticsearch/recovery/RelocationIT.java | 2 +- .../aggregations/metrics/TopHitsIT.java | 69 +++++++-------- .../search/fetch/subphase/InnerHitsIT.java | 28 +++--- .../highlight/HighlighterSearchIT.java | 19 ++-- .../FunctionScoreFieldValueIT.java | 83 ++++++++++-------- .../functionscore/RandomScoreFunctionIT.java | 3 - .../search/morelikethis/MoreLikeThisIT.java | 86 +++++++++---------- .../search/nested/NestedWithMinScoreIT.java | 6 +- .../elasticsearch/search/query/ExistsIT.java | 7 +- .../search/query/ScriptScoreQueryIT.java | 8 +- .../search/query/SearchQueryIT.java | 38 ++++---- .../scriptfilter/ScriptQuerySearchIT.java | 6 +- .../search/searchafter/SearchAfterIT.java | 2 +- .../search/sort/FieldSortIT.java | 42 ++++----- .../search/sort/GeoDistanceSortBuilderIT.java | 33 +++---- .../search/sort/SimpleSortIT.java | 9 +- .../threadpool/SimpleThreadPoolIT.java | 4 +- .../search/SearchServiceTests.java | 11 +-- .../hamcrest/ElasticsearchAssertions.java | 46 ++++++++-- 30 files changed, 347 insertions(+), 339 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java index a63f7d6f7c548..3100db781172a 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java @@ -57,6 +57,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.search.aggregations.AggregationBuilders.topHits; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; @@ -513,16 +514,15 @@ public void testGetHitsFailure() throws Exception { .get(); QueryBuilder queryBuilder = QueryBuilders.rangeQuery("@timestamp").lte("2021-01-01T00:10:00Z"); - SearchResponse response = client().prepareSearch("test") - .setQuery(queryBuilder) - .setSize(10) - .addSort("key", SortOrder.ASC) - .addSort("@timestamp", SortOrder.ASC) - .get(); - assertSearchResponse(response); - response = client().prepareSearch("test").setQuery(queryBuilder).setSize(10).addAggregation(timeSeries("by_ts")).get(); - assertSearchResponse(response); + assertNoFailures( + client().prepareSearch("test") + .setQuery(queryBuilder) + .setSize(10) + .addSort("key", SortOrder.ASC) + .addSort("@timestamp", SortOrder.ASC) + ); + assertNoFailures(client().prepareSearch("test").setQuery(queryBuilder).setSize(10).addAggregation(timeSeries("by_ts"))); assertAcked(indicesAdmin().delete(new DeleteIndexRequest("test")).actionGet()); } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 44eb1ea7c47c2..6cef9280707b8 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -754,11 +754,9 @@ public void testDataSteamAliasWithFilter() throws Exception { ); // Searching the data stream directly should return all hits: - SearchResponse searchResponse = client().prepareSearch("logs-foobar").get(); - assertSearchHits(searchResponse, "1", "2"); + assertSearchHits(client().prepareSearch("logs-foobar"), "1", "2"); // Search the alias should only return document 2, because it matches with the defined filter in the alias: - searchResponse = client().prepareSearch("foo").get(); - assertSearchHits(searchResponse, "2"); + assertSearchHits(client().prepareSearch("foo"), "2"); // Update alias: addAction = new AliasActions(AliasActions.Type.ADD).index(dataStreamName) @@ -786,11 +784,9 @@ public void testDataSteamAliasWithFilter() throws Exception { ); // Searching the data stream directly should return all hits: - searchResponse = client().prepareSearch("logs-foobar").get(); - assertSearchHits(searchResponse, "1", "2"); + assertSearchHits(client().prepareSearch("logs-foobar"), "1", "2"); // Search the alias should only return document 1, because it matches with the defined filter in the alias: - searchResponse = client().prepareSearch("foo").get(); - assertSearchHits(searchResponse, "1"); + assertSearchHits(client().prepareSearch("foo"), "1"); } public void testSearchFilteredAndUnfilteredAlias() throws Exception { @@ -833,11 +829,9 @@ public void testSearchFilteredAndUnfilteredAlias() throws Exception { ); // Searching the filtered and unfiltered aliases should return all results (unfiltered): - SearchResponse searchResponse = client().prepareSearch("foo", "bar").get(); - assertSearchHits(searchResponse, "1", "2"); + assertSearchHits(client().prepareSearch("foo", "bar"), "1", "2"); // Searching the data stream name and the filtered alias should return all results (unfiltered): - searchResponse = client().prepareSearch("foo", dataStreamName).get(); - assertSearchHits(searchResponse, "1", "2"); + assertSearchHits(client().prepareSearch("foo", dataStreamName), "1", "2"); } public void testRandomDataSteamAliasesUpdate() throws Exception { diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java index e7d6d127174ec..69b0046e5e207 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java @@ -43,6 +43,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.bucketScript; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; @@ -129,8 +130,7 @@ public void testScore() throws Exception { score = ScoreFunctionBuilders.scriptFunction(new Script(ScriptType.INLINE, "expression", "1 / _score", Collections.emptyMap())); req.addAggregation(AggregationBuilders.max("max_score").script((score).getScript())); req.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); // make sure DF is consistent - rsp = req.get(); - assertSearchResponse(rsp); + assertNoFailures(req); } public void testDateMethods() throws Exception { diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java index bac87ebfbf9c8..baa4a4459f408 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java @@ -247,14 +247,10 @@ public void testCachingBugWithFqueryFilter() throws Exception { for (int i = 1; i <= 10; i++) { logger.info("Round {}", i); - SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(constantScoreQuery(hasChildQuery("child", matchAllQuery(), ScoreMode.Max))) - .get(); - assertNoFailures(searchResponse); - searchResponse = client().prepareSearch("test") - .setQuery(constantScoreQuery(hasParentQuery("parent", matchAllQuery(), true))) - .get(); - assertNoFailures(searchResponse); + assertNoFailures( + client().prepareSearch("test").setQuery(constantScoreQuery(hasChildQuery("child", matchAllQuery(), ScoreMode.Max))) + ); + assertNoFailures(client().prepareSearch("test").setQuery(constantScoreQuery(hasParentQuery("parent", matchAllQuery(), true)))); } } @@ -471,18 +467,17 @@ public void testDfsSearchType() throws Exception { refresh(); - SearchResponse searchResponse = client().prepareSearch("test") - .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) - .setQuery(boolQuery().mustNot(hasChildQuery("child", boolQuery().should(queryStringQuery("c_field:*")), ScoreMode.None))) - .get(); - assertNoFailures(searchResponse); + assertNoFailures( + client().prepareSearch("test") + .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) + .setQuery(boolQuery().mustNot(hasChildQuery("child", boolQuery().should(queryStringQuery("c_field:*")), ScoreMode.None))) + ); - searchResponse = client().prepareSearch("test") - .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) - .setQuery(boolQuery().mustNot(hasParentQuery("parent", boolQuery().should(queryStringQuery("p_field:*")), false))) - .execute() - .actionGet(); - assertNoFailures(searchResponse); + assertNoFailures( + client().prepareSearch("test") + .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) + .setQuery(boolQuery().mustNot(hasParentQuery("parent", boolQuery().should(queryStringQuery("p_field:*")), false))) + ); } public void testHasChildAndHasParentFailWhenSomeSegmentsDontContainAnyParentOrChildDocs() throws Exception { @@ -1711,15 +1706,15 @@ public void testHasParentInnerQueryType() { refresh(); // make sure that when we explicitly set a type, the inner query is executed in the context of the child type instead - SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(hasChildQuery("child-type", new IdsQueryBuilder().addIds("child-id"), ScoreMode.None)) - .get(); - assertSearchHits(searchResponse, "parent-id"); + assertSearchHits( + client().prepareSearch("test").setQuery(hasChildQuery("child-type", new IdsQueryBuilder().addIds("child-id"), ScoreMode.None)), + "parent-id" + ); // make sure that when we explicitly set a type, the inner query is executed in the context of the parent type instead - searchResponse = client().prepareSearch("test") - .setQuery(hasParentQuery("parent-type", new IdsQueryBuilder().addIds("parent-id"), false)) - .get(); - assertSearchHits(searchResponse, "child-id"); + assertSearchHits( + client().prepareSearch("test").setQuery(hasParentQuery("parent-type", new IdsQueryBuilder().addIds("parent-id"), false)), + "child-id" + ); } public void testHighlightersIgnoreParentChild() throws IOException { diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java index 8aabbe31def85..2e647e6ea08e5 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java @@ -676,19 +676,19 @@ public void testTooHighResultWindow() { containsString("the inner hit definition's [_name]'s from + size must be less than or equal to: [100] but was [110]") ); updateIndexSettings(Settings.builder().put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 110), "index1"); - response = client().prepareSearch("index1") - .setQuery( - hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true) - .innerHit(new InnerHitBuilder().setFrom(100).setSize(10).setName("_name")) - ) - .get(); - assertNoFailures(response); - response = client().prepareSearch("index1") - .setQuery( - hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true) - .innerHit(new InnerHitBuilder().setFrom(10).setSize(100).setName("_name")) - ) - .get(); - assertNoFailures(response); + assertNoFailures( + client().prepareSearch("index1") + .setQuery( + hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true) + .innerHit(new InnerHitBuilder().setFrom(100).setSize(10).setName("_name")) + ) + ); + assertNoFailures( + client().prepareSearch("index1") + .setQuery( + hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true) + .innerHit(new InnerHitBuilder().setFrom(10).setSize(100).setName("_name")) + ) + ); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 54c10499b0b3a..e33e7ee148e04 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -81,7 +81,6 @@ import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFutureThrows; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -359,7 +358,7 @@ public void testSearchTaskDescriptions() { headers.put(Task.X_OPAQUE_ID_HTTP_HEADER, "my_id"); headers.put("Foo-Header", "bar"); headers.put("Custom-Task-Header", "my_value"); - assertSearchResponse(client().filterWithHeader(headers).prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).get()); + assertNoFailures(client().filterWithHeader(headers).prepareSearch("test").setQuery(QueryBuilders.matchAllQuery())); // the search operation should produce one main task List mainTask = findEvents(SearchAction.NAME, Tuple::v1); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java index e5690d0f17f5f..b44c49982e31b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java @@ -48,12 +48,12 @@ public void testDeleteIndexOnIndexReadOnlyAllowDeleteSetting() { refresh(); try { updateIndexSettings(Settings.builder().put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, true), "test"); - assertSearchHits(client().prepareSearch().get(), "1"); + assertSearchHits(client().prepareSearch(), "1"); assertBlocked( client().prepareIndex().setIndex("test").setId("2").setSource("foo", "bar"), IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK ); - assertSearchHits(client().prepareSearch().get(), "1"); + assertSearchHits(client().prepareSearch(), "1"); assertAcked(indicesAdmin().prepareDelete("test")); } finally { Settings settings = Settings.builder().putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE).build(); @@ -92,7 +92,7 @@ public void testDeleteIndexOnClusterReadOnlyAllowDeleteSetting() { refresh(); try { updateClusterSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true)); - assertSearchHits(client().prepareSearch().get(), "1"); + assertSearchHits(client().prepareSearch(), "1"); assertBlocked( client().prepareIndex().setIndex("test").setId("2").setSource("foo", "bar"), Metadata.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK @@ -101,7 +101,7 @@ public void testDeleteIndexOnClusterReadOnlyAllowDeleteSetting() { indicesAdmin().prepareUpdateSettings("test").setSettings(Settings.builder().put("index.number_of_replicas", 2)), Metadata.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK ); - assertSearchHits(client().prepareSearch().get(), "1"); + assertSearchHits(client().prepareSearch(), "1"); assertAcked(indicesAdmin().prepareDelete("test")); } finally { updateClusterSettings(Settings.builder().putNull(Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey())); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java index 5ad8d02c15d6e..8b4a82cc36c20 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java @@ -195,7 +195,7 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception { } client().prepareIndex("test").setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); - assertSearchHits(client().prepareSearch("test").get(), "1"); + assertSearchHits(client().prepareSearch("test"), "1"); // Move all nodes above the low watermark so no shard movement can occur, and at least one node above the flood stage watermark so // the index is blocked @@ -221,7 +221,7 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception { client().prepareIndex().setIndex("test").setId("2").setSource("foo", "bar"), IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK ); - assertSearchHits(client().prepareSearch("test").get(), "1"); + assertSearchHits(client().prepareSearch("test"), "1"); logger.info("--> index is confirmed read-only, releasing disk space"); @@ -240,7 +240,7 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception { throw new AssertionError("retrying", e); } }); - assertSearchHits(client().prepareSearch("test").get(), "1", "3"); + assertSearchHits(client().prepareSearch("test"), "1", "3"); } public void testOnlyMovesEnoughShardsToDropBelowHighWatermark() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java index 302d6ce74d65f..6970f73e591fc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java @@ -63,13 +63,13 @@ public void testIndex() { .get(); assertEquals(RestStatus.CREATED, index.status()); assertFalse("request shouldn't have forced a refresh", index.forcedRefresh()); - assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")), "1"); } public void testDelete() throws InterruptedException, ExecutionException { // Index normally indexRandom(true, client().prepareIndex("test").setId("1").setSource("foo", "bar")); - assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")), "1"); // Now delete with blockUntilRefresh DeleteResponse delete = client().prepareDelete("test", "1").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get(); @@ -81,7 +81,7 @@ public void testDelete() throws InterruptedException, ExecutionException { public void testUpdate() throws InterruptedException, ExecutionException { // Index normally indexRandom(true, client().prepareIndex("test").setId("1").setSource("foo", "bar")); - assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")), "1"); // Update with RefreshPolicy.WAIT_UNTIL UpdateResponse update = client().prepareUpdate("test", "1") @@ -90,7 +90,7 @@ public void testUpdate() throws InterruptedException, ExecutionException { .get(); assertEquals(2, update.getVersion()); assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); - assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1"); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")), "1"); // Upsert with RefreshPolicy.WAIT_UNTIL update = client().prepareUpdate("test", "2") @@ -100,7 +100,7 @@ public void testUpdate() throws InterruptedException, ExecutionException { .get(); assertEquals(1, update.getVersion()); assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); - assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "cat")).get(), "2"); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "cat")), "2"); // Update-becomes-delete with RefreshPolicy.WAIT_UNTIL update = client().prepareUpdate("test", "2") @@ -117,13 +117,13 @@ public void testBulk() { BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); bulk.add(client().prepareIndex("test").setId("1").setSource("foo", "bar")); assertBulkSuccess(bulk.get()); - assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")), "1"); // Update by bulk with RefreshPolicy.WAIT_UNTIL bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); bulk.add(client().prepareUpdate("test", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz")); assertBulkSuccess(bulk.get()); - assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1"); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")), "1"); // Delete by bulk with RefreshPolicy.WAIT_UNTIL bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); @@ -153,7 +153,7 @@ public void testNoRefreshInterval() throws InterruptedException, ExecutionExcept } assertEquals(RestStatus.CREATED, index.get().status()); assertFalse("request shouldn't have forced a refresh", index.get().forcedRefresh()); - assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")), "1"); } private void assertBulkSuccess(BulkResponse response) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index ce23c44cb96cc..be2cd1eeb3581 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -371,14 +371,18 @@ public void testBulkRequestWithDynamicTemplates() throws Exception { final BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertFalse(bulkResponse.hasFailures()); - SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(new GeoBoundingBoxQueryBuilder("location").setCorners(new GeoPoint(42, -72), new GeoPoint(40, -74))) - .get(); - assertSearchHits(searchResponse, "1", "2", "4"); - searchResponse = client().prepareSearch("test") - .setQuery(new GeoBoundingBoxQueryBuilder("address.location").setCorners(new GeoPoint(42, -72), new GeoPoint(40, -74))) - .get(); - assertSearchHits(searchResponse, "3"); + assertSearchHits( + client().prepareSearch("test") + .setQuery(new GeoBoundingBoxQueryBuilder("location").setCorners(new GeoPoint(42, -72), new GeoPoint(40, -74))), + "1", + "2", + "4" + ); + assertSearchHits( + client().prepareSearch("test") + .setQuery(new GeoBoundingBoxQueryBuilder("address.location").setCorners(new GeoPoint(42, -72), new GeoPoint(40, -74))), + "3" + ); } public void testBulkRequestWithNotFoundDynamicTemplate() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 53df5a1d3c834..494d9c4b0a6f3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -82,7 +82,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.emptyCollectionOf; import static org.hamcrest.Matchers.equalTo; @@ -1163,8 +1163,8 @@ public void testFilterCacheStats() throws Exception { // the query cache has an optimization that disables it automatically if there is contention, // so we run it in an assertBusy block which should eventually succeed assertBusy(() -> { - assertSearchResponse( - client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get() + assertNoFailures( + client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))) ); IndicesStatsResponse stats = indicesAdmin().prepareStats("index").setQueryCache(true).get(); assertCumulativeQueryCacheStats(stats); @@ -1174,8 +1174,8 @@ public void testFilterCacheStats() throws Exception { }); assertBusy(() -> { - assertSearchResponse( - client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get() + assertNoFailures( + client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))) ); IndicesStatsResponse stats = indicesAdmin().prepareStats("index").setQueryCache(true).get(); assertCumulativeQueryCacheStats(stats); @@ -1224,8 +1224,8 @@ public void testFilterCacheStats() throws Exception { ); assertBusy(() -> { - assertSearchResponse( - client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get() + assertNoFailures( + client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))) ); IndicesStatsResponse stats = indicesAdmin().prepareStats("index").setQueryCache(true).get(); assertCumulativeQueryCacheStats(stats); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java index 95f65e23b0ab4..a18015da0737a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java @@ -485,7 +485,7 @@ public void testIndexSearchAndRelocateConcurrently() throws Exception { for (int i = 0; i < searchThreads.length; i++) { searchThreads[i] = new Thread(() -> { while (stopped.get() == false) { - assertNoFailures(client().prepareSearch("test").setRequestCache(false).get()); + assertNoFailures(client().prepareSearch("test").setRequestCache(false)); } }); searchThreads[i].start(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index af475c11ea43e..5829f75d45edb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -995,28 +995,29 @@ public void testUseMaxDocInsteadOfSize() throws Exception { Settings.builder().put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), ArrayUtil.MAX_ARRAY_LENGTH), "idx" ); - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()) - .field(TERMS_AGGS_FIELD) - .subAggregation( - topHits("hits").size(ArrayUtil.MAX_ARRAY_LENGTH - 1).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC)) - ) - ) - .get(); - assertNoFailures(response); + assertNoFailures( + client().prepareSearch("idx") + .addAggregation( + terms("terms").executionHint(randomExecutionHint()) + .field(TERMS_AGGS_FIELD) + .subAggregation( + topHits("hits").size(ArrayUtil.MAX_ARRAY_LENGTH - 1) + .sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC)) + ) + ) + ); updateIndexSettings(Settings.builder().putNull(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey()), "idx"); } public void testTooHighResultWindow() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()) - .field(TERMS_AGGS_FIELD) - .subAggregation(topHits("hits").from(50).size(10).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))) - ) - .get(); - assertNoFailures(response); + assertNoFailures( + client().prepareSearch("idx") + .addAggregation( + terms("terms").executionHint(randomExecutionHint()) + .field(TERMS_AGGS_FIELD) + .subAggregation(topHits("hits").from(50).size(10).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))) + ) + ); Exception e = expectThrows( SearchPhaseExecutionException.class, @@ -1048,22 +1049,22 @@ public void testTooHighResultWindow() throws Exception { ); updateIndexSettings(Settings.builder().put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 110), "idx"); - response = client().prepareSearch("idx") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()) - .field(TERMS_AGGS_FIELD) - .subAggregation(topHits("hits").from(100).size(10).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))) - ) - .get(); - assertNoFailures(response); - response = client().prepareSearch("idx") - .addAggregation( - terms("terms").executionHint(randomExecutionHint()) - .field(TERMS_AGGS_FIELD) - .subAggregation(topHits("hits").from(10).size(100).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))) - ) - .get(); - assertNoFailures(response); + assertNoFailures( + client().prepareSearch("idx") + .addAggregation( + terms("terms").executionHint(randomExecutionHint()) + .field(TERMS_AGGS_FIELD) + .subAggregation(topHits("hits").from(100).size(10).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))) + ) + ); + assertNoFailures( + client().prepareSearch("idx") + .addAggregation( + terms("terms").executionHint(randomExecutionHint()) + .field(TERMS_AGGS_FIELD) + .subAggregation(topHits("hits").from(10).size(100).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))) + ) + ); updateIndexSettings(Settings.builder().putNull(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey()), "idx"); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java index e786513a1e228..4041bbc431f75 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java @@ -1006,22 +1006,22 @@ public void testTooHighResultWindow() throws Exception { ); updateIndexSettings(Settings.builder().put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 110), "index2"); - response = client().prepareSearch("index2") - .setQuery( - nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg).innerHit( - new InnerHitBuilder().setFrom(100).setSize(10).setName("_name") + assertNoFailures( + client().prepareSearch("index2") + .setQuery( + nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg).innerHit( + new InnerHitBuilder().setFrom(100).setSize(10).setName("_name") + ) ) - ) - .get(); - assertNoFailures(response); - response = client().prepareSearch("index2") - .setQuery( - nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg).innerHit( - new InnerHitBuilder().setFrom(10).setSize(100).setName("_name") + ); + assertNoFailures( + client().prepareSearch("index2") + .setQuery( + nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg).innerHit( + new InnerHitBuilder().setFrom(10).setSize(100).setName("_name") + ) ) - ) - .get(); - assertNoFailures(response); + ); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index 83bc6004a22b4..9da65214599e1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -1539,11 +1539,11 @@ public void testFastVectorHighlighterShouldFailIfNoTermVectors() throws Exceptio } indexRandom(true, indexRequestBuilders); - SearchResponse search = client().prepareSearch() - .setQuery(matchPhraseQuery("title", "this is a test")) - .highlighter(new HighlightBuilder().field("title", 50, 1, 10)) - .get(); - assertNoFailures(search); + assertNoFailures( + client().prepareSearch() + .setQuery(matchPhraseQuery("title", "this is a test")) + .highlighter(new HighlightBuilder().field("title", 50, 1, 10)) + ); assertFailures( client().prepareSearch() @@ -1560,7 +1560,6 @@ public void testFastVectorHighlighterShouldFailIfNoTermVectors() throws Exceptio client().prepareSearch() .setQuery(matchPhraseQuery("title", "this is a test")) .highlighter(new HighlightBuilder().field("tit*", 50, 1, 10).highlighterType("fvh")) - .get() ); } @@ -2656,11 +2655,9 @@ public void testPostingsHighlighterShouldFailIfNoOffsets() throws Exception { } indexRandom(true, indexRequestBuilders); - SearchResponse search = client().prepareSearch() - .setQuery(matchQuery("title", "this is a test")) - .highlighter(new HighlightBuilder().field("title")) - .get(); - assertNoFailures(search); + assertNoFailures( + client().prepareSearch().setQuery(matchQuery("title", "this is a test")).highlighter(new HighlightBuilder().field("title")) + ); } public void testPostingsHighlighterBoostingQuery() throws IOException { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java index 3e135274852d9..7d006c2fa754c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java @@ -53,39 +53,45 @@ public void testFieldValueFactor() throws IOException { refresh(); // document 2 scores higher because 17 > 5 - SearchResponse response = client().prepareSearch("test") - .setExplain(randomBoolean()) - .setQuery(functionScoreQuery(simpleQueryStringQuery("foo"), fieldValueFactorFunction("test"))) - .get(); - assertOrderedSearchHits(response, "2", "1"); + assertOrderedSearchHits( + client().prepareSearch("test") + .setExplain(randomBoolean()) + .setQuery(functionScoreQuery(simpleQueryStringQuery("foo"), fieldValueFactorFunction("test"))), + "2", + "1" + ); // try again, but this time explicitly use the do-nothing modifier - response = client().prepareSearch("test") - .setExplain(randomBoolean()) - .setQuery( - functionScoreQuery( - simpleQueryStringQuery("foo"), - fieldValueFactorFunction("test").modifier(FieldValueFactorFunction.Modifier.NONE) - ) - ) - .get(); - assertOrderedSearchHits(response, "2", "1"); + assertOrderedSearchHits( + client().prepareSearch("test") + .setExplain(randomBoolean()) + .setQuery( + functionScoreQuery( + simpleQueryStringQuery("foo"), + fieldValueFactorFunction("test").modifier(FieldValueFactorFunction.Modifier.NONE) + ) + ), + "2", + "1" + ); // document 1 scores higher because 1/5 > 1/17 - response = client().prepareSearch("test") - .setExplain(randomBoolean()) - .setQuery( - functionScoreQuery( - simpleQueryStringQuery("foo"), - fieldValueFactorFunction("test").modifier(FieldValueFactorFunction.Modifier.RECIPROCAL) - ) - ) - .get(); - assertOrderedSearchHits(response, "1", "2"); + assertOrderedSearchHits( + client().prepareSearch("test") + .setExplain(randomBoolean()) + .setQuery( + functionScoreQuery( + simpleQueryStringQuery("foo"), + fieldValueFactorFunction("test").modifier(FieldValueFactorFunction.Modifier.RECIPROCAL) + ) + ), + "1", + "2" + ); // doc 3 doesn't have a "test" field, so an exception will be thrown try { - response = client().prepareSearch("test") + SearchResponse response = client().prepareSearch("test") .setExplain(randomBoolean()) .setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("test"))) .get(); @@ -95,19 +101,22 @@ public void testFieldValueFactor() throws IOException { } // doc 3 doesn't have a "test" field but we're defaulting it to 100 so it should be last - response = client().prepareSearch("test") - .setExplain(randomBoolean()) - .setQuery( - functionScoreQuery( - matchAllQuery(), - fieldValueFactorFunction("test").modifier(FieldValueFactorFunction.Modifier.RECIPROCAL).missing(100) - ) - ) - .get(); - assertOrderedSearchHits(response, "1", "2", "3"); + assertOrderedSearchHits( + client().prepareSearch("test") + .setExplain(randomBoolean()) + .setQuery( + functionScoreQuery( + matchAllQuery(), + fieldValueFactorFunction("test").modifier(FieldValueFactorFunction.Modifier.RECIPROCAL).missing(100) + ) + ), + "1", + "2", + "3" + ); // field is not mapped but we're defaulting it to 100 so all documents should have the same score - response = client().prepareSearch("test") + SearchResponse response = client().prepareSearch("test") .setExplain(randomBoolean()) .setQuery( functionScoreQuery( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java index b02a04e2ebd06..ff1db506e73d8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java @@ -317,14 +317,12 @@ public void testSeeds() throws Exception { client().prepareSearch() .setSize(docCount) // get all docs otherwise we are prone to tie-breaking .setQuery(functionScoreQuery(matchAllQuery(), randomFunction().seed(randomInt()).setField(SeqNoFieldMapper.NAME))) - .get() ); assertNoFailures( client().prepareSearch() .setSize(docCount) // get all docs otherwise we are prone to tie-breaking .setQuery(functionScoreQuery(matchAllQuery(), randomFunction().seed(randomLong()).setField(SeqNoFieldMapper.NAME))) - .get() ); assertNoFailures( @@ -336,7 +334,6 @@ public void testSeeds() throws Exception { randomFunction().seed(randomRealisticUnicodeOfLengthBetween(10, 20)).setField(SeqNoFieldMapper.NAME) ) ) - .get() ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java index 528e5785f7050..5a8b506b99296 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java @@ -47,7 +47,6 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.notNullValue; public class MoreLikeThisIT extends ESIntegTestCase { @@ -289,14 +288,8 @@ public void testMoreLikeThisIssue2197() throws Exception { indicesAdmin().prepareRefresh("foo").get(); assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN)); - SearchResponse response = client().prepareSearch() - .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1") })) - .get(); - assertNoFailures(response); - assertThat(response, notNullValue()); - response = client().prepareSearch().setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1") })).get(); - assertNoFailures(response); - assertThat(response, notNullValue()); + assertNoFailures(client().prepareSearch().setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1") }))); + assertNoFailures(client().prepareSearch().setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1") }))); } // Issue #2489 @@ -311,11 +304,9 @@ public void testMoreLikeWithCustomRouting() throws Exception { .get(); indicesAdmin().prepareRefresh("foo").get(); - SearchResponse response = client().prepareSearch() - .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1").routing("2") })) - .get(); - assertNoFailures(response); - assertThat(response, notNullValue()); + assertNoFailures( + client().prepareSearch().setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1").routing("2") })) + ); } // Issue #3039 @@ -329,11 +320,9 @@ public void testMoreLikeThisIssueRoutingNotSerialized() throws Exception { .setRouting("4000") .get(); indicesAdmin().prepareRefresh("foo").get(); - SearchResponse response = client().prepareSearch() - .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1").routing("4000") })) - .get(); - assertNoFailures(response); - assertThat(response, notNullValue()); + assertNoFailures( + client().prepareSearch().setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1").routing("4000") })) + ); } // Issue #3252 @@ -520,35 +509,40 @@ public void testSimpleMoreLikeInclude() throws Exception { indicesAdmin().refresh(new RefreshRequest()).actionGet(); logger.info("Running More Like This with include true"); - SearchResponse response = client().prepareSearch() - .setQuery( - new MoreLikeThisQueryBuilder(null, new Item[] { new Item("test", "1") }).minTermFreq(1) - .minDocFreq(1) - .include(true) - .minimumShouldMatch("0%") - ) - .get(); - assertOrderedSearchHits(response, "1", "2"); - - response = client().prepareSearch() - .setQuery( - new MoreLikeThisQueryBuilder(null, new Item[] { new Item("test", "2") }).minTermFreq(1) - .minDocFreq(1) - .include(true) - .minimumShouldMatch("0%") - ) - .get(); - assertOrderedSearchHits(response, "2", "1"); + assertOrderedSearchHits( + client().prepareSearch() + .setQuery( + new MoreLikeThisQueryBuilder(null, new Item[] { new Item("test", "1") }).minTermFreq(1) + .minDocFreq(1) + .include(true) + .minimumShouldMatch("0%") + ), + "1", + "2" + ); + + assertOrderedSearchHits( + client().prepareSearch() + .setQuery( + new MoreLikeThisQueryBuilder(null, new Item[] { new Item("test", "2") }).minTermFreq(1) + .minDocFreq(1) + .include(true) + .minimumShouldMatch("0%") + ), + "2", + "1" + ); logger.info("Running More Like This with include false"); - response = client().prepareSearch() - .setQuery( - new MoreLikeThisQueryBuilder(null, new Item[] { new Item("test", "1") }).minTermFreq(1) - .minDocFreq(1) - .minimumShouldMatch("0%") - ) - .get(); - assertSearchHits(response, "2"); + assertSearchHits( + client().prepareSearch() + .setQuery( + new MoreLikeThisQueryBuilder(null, new Item[] { new Item("test", "1") }).minTermFreq(1) + .minDocFreq(1) + .minimumShouldMatch("0%") + ), + "2" + ); } public void testSimpleMoreLikeThisIds() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java index 45b460fc8cc69..ce1b4bf30d498 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java @@ -9,8 +9,6 @@ package org.elasticsearch.search.nested; import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -113,8 +111,6 @@ public void testNestedWithMinScore() throws Exception { if (randomBoolean()) { source.trackTotalHitsUpTo(randomBoolean() ? Integer.MAX_VALUE : randomIntBetween(1, 1000)); } - SearchRequest searchRequest = new SearchRequest("test").source(source); - final SearchResponse searchResponse = client().search(searchRequest).actionGet(); - ElasticsearchAssertions.assertSearchHits(searchResponse, "d1"); + ElasticsearchAssertions.assertSearchHits(client().prepareSearch("test").setSource(source), "d1"); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java index 56ecd865cd707..dd3e160054f6b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java @@ -30,6 +30,7 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; public class ExistsIT extends ESIntegTestCase { @@ -37,10 +38,8 @@ public class ExistsIT extends ESIntegTestCase { // TODO: move this to a unit test somewhere... public void testEmptyIndex() throws Exception { createIndex("test"); - SearchResponse resp = client().prepareSearch("test").setQuery(QueryBuilders.existsQuery("foo")).get(); - assertSearchResponse(resp); - resp = client().prepareSearch("test").setQuery(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("foo"))).get(); - assertSearchResponse(resp); + assertNoFailures(client().prepareSearch("test").setQuery(QueryBuilders.existsQuery("foo"))); + assertNoFailures(client().prepareSearch("test").setQuery(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("foo")))); } public void testExists() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java index dc59982e8016e..40b8000b30b7e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java @@ -136,10 +136,7 @@ public void testDisallowExpensiveQueries() { // Execute with search.allow_expensive_queries = null => default value = true => success Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['field2'].value * param1", Map.of("param1", 0.1)); - SearchResponse resp = client().prepareSearch("test-index") - .setQuery(scriptScoreQuery(matchQuery("field1", "text0"), script)) - .get(); - assertNoFailures(resp); + assertNoFailures(client().prepareSearch("test-index").setQuery(scriptScoreQuery(matchQuery("field1", "text0"), script))); // Set search.allow_expensive_queries to "false" => assert failure updateClusterSettings(Settings.builder().put("search.allow_expensive_queries", false)); @@ -155,8 +152,7 @@ public void testDisallowExpensiveQueries() { // Set search.allow_expensive_queries to "true" => success updateClusterSettings(Settings.builder().put("search.allow_expensive_queries", true)); - resp = client().prepareSearch("test-index").setQuery(scriptScoreQuery(matchQuery("field1", "text0"), script)).get(); - assertNoFailures(resp); + assertNoFailures(client().prepareSearch("test-index").setQuery(scriptScoreQuery(matchQuery("field1", "text0"), script))); } finally { updateClusterSettings(Settings.builder().put("search.allow_expensive_queries", (String) null)); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java index c8bfd52e61d4a..18a5ad78995da 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -1474,25 +1474,27 @@ public void testSimpleDFSQuery() throws IOException { .get(); refresh(); - SearchResponse response = client().prepareSearch("test") - .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) - .setQuery( - boolQuery().must(termQuery("online", true)) - .must( - boolQuery().should( - boolQuery().must(rangeQuery("ts").lt(System.currentTimeMillis() - (15 * 1000))).must(termQuery("type", "bs")) - ) - .should( - boolQuery().must(rangeQuery("ts").lt(System.currentTimeMillis() - (15 * 1000))).must(termQuery("type", "s")) + assertNoFailures( + client().prepareSearch("test") + .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) + .setQuery( + boolQuery().must(termQuery("online", true)) + .must( + boolQuery().should( + boolQuery().must(rangeQuery("ts").lt(System.currentTimeMillis() - (15 * 1000))) + .must(termQuery("type", "bs")) ) - ) - ) - .setVersion(true) - .setFrom(0) - .setSize(100) - .setExplain(true) - .get(); - assertNoFailures(response); + .should( + boolQuery().must(rangeQuery("ts").lt(System.currentTimeMillis() - (15 * 1000))) + .must(termQuery("type", "s")) + ) + ) + ) + .setVersion(true) + .setFrom(0) + .setSize(100) + .setExplain(true) + ); } public void testMultiFieldQueryString() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java index 83804ec526e4b..499aba8fd57d3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java @@ -224,8 +224,7 @@ public void testDisallowExpensiveQueries() { // Execute with search.allow_expensive_queries = null => default value = false => success Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value > 1", Collections.emptyMap()); - SearchResponse resp = client().prepareSearch("test-index").setQuery(scriptQuery(script)).get(); - assertNoFailures(resp); + assertNoFailures(client().prepareSearch("test-index").setQuery(scriptQuery(script))); updateClusterSettings(Settings.builder().put("search.allow_expensive_queries", false)); @@ -241,8 +240,7 @@ public void testDisallowExpensiveQueries() { // Set search.allow_expensive_queries to "true" => success updateClusterSettings(Settings.builder().put("search.allow_expensive_queries", true)); - resp = client().prepareSearch("test-index").setQuery(scriptQuery(script)).get(); - assertNoFailures(resp); + assertNoFailures(client().prepareSearch("test-index").setQuery(scriptQuery(script))); } finally { updateClusterSettings(Settings.builder().put("search.allow_expensive_queries", (String) null)); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java index 1185fd0a0485f..87a500691aab5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java @@ -269,7 +269,7 @@ public void testWithCustomFormatSortValueOfDateField() throws Exception { .addSort(SortBuilders.fieldSort("end_date").setFormat("epoch_millis")) .searchAfter(new Object[] { "21/02/2016", 1748390400000L }) .setSize(2); - assertNoFailures(searchRequest.get()); + assertNoFailures(searchRequest); searchRequest = client().prepareSearch("test") .addSort(SortBuilders.fieldSort("start_date").setFormat("dd/MM/yyyy")) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java index 966bed24bbe66..3b6f29fd1ec90 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -1071,33 +1071,29 @@ public void testIgnoreUnmapped() throws Exception { } } - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("kkk").unmappedType("keyword")) - .get(); - assertNoFailures(searchResponse); + assertNoFailures(client().prepareSearch().setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("kkk").unmappedType("keyword"))); // nested field - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort( - SortBuilders.fieldSort("nested.foo") - .unmappedType("keyword") - .setNestedSort(new NestedSortBuilder("nested").setNestedSort(new NestedSortBuilder("nested.foo"))) - ) - .get(); - assertNoFailures(searchResponse); + assertNoFailures( + client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort( + SortBuilders.fieldSort("nested.foo") + .unmappedType("keyword") + .setNestedSort(new NestedSortBuilder("nested").setNestedSort(new NestedSortBuilder("nested.foo"))) + ) + ); // nestedQuery - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort( - SortBuilders.fieldSort("nested.foo") - .unmappedType("keyword") - .setNestedSort(new NestedSortBuilder("nested").setFilter(QueryBuilders.termQuery("nested.foo", "abc"))) - ) - .get(); - assertNoFailures(searchResponse); + assertNoFailures( + client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort( + SortBuilders.fieldSort("nested.foo") + .unmappedType("keyword") + .setNestedSort(new NestedSortBuilder("nested").setFilter(QueryBuilders.termQuery("nested.foo", "abc"))) + ) + ); } public void testSortMVField() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java index 338b7807404cf..0862d919843db 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java @@ -368,25 +368,26 @@ public void testCrossIndexIgnoreUnmapped() throws Exception { client().prepareIndex("test2").setSource() ); - SearchResponse resp = client().prepareSearch("test1", "test2") - .addSort(fieldSort("str_field").order(SortOrder.ASC).unmappedType("keyword")) - .addSort(fieldSort("str_field2").order(SortOrder.DESC).unmappedType("keyword")) - .get(); - - assertSortValues(resp, new Object[] { "bcd", null }, new Object[] { null, null }); + assertSortValues( + client().prepareSearch("test1", "test2") + .addSort(fieldSort("str_field").order(SortOrder.ASC).unmappedType("keyword")) + .addSort(fieldSort("str_field2").order(SortOrder.DESC).unmappedType("keyword")), + new Object[] { "bcd", null }, + new Object[] { null, null } + ); - resp = client().prepareSearch("test1", "test2") - .addSort(fieldSort("long_field").order(SortOrder.ASC).unmappedType("long")) - .addSort(fieldSort("long_field2").order(SortOrder.DESC).unmappedType("long")) - .get(); - assertSortValues(resp, new Object[] { 3L, Long.MIN_VALUE }, new Object[] { Long.MAX_VALUE, Long.MIN_VALUE }); + assertSortValues( + client().prepareSearch("test1", "test2") + .addSort(fieldSort("long_field").order(SortOrder.ASC).unmappedType("long")) + .addSort(fieldSort("long_field2").order(SortOrder.DESC).unmappedType("long")), + new Object[] { 3L, Long.MIN_VALUE }, + new Object[] { Long.MAX_VALUE, Long.MIN_VALUE } + ); - resp = client().prepareSearch("test1", "test2") - .addSort(fieldSort("double_field").order(SortOrder.ASC).unmappedType("double")) - .addSort(fieldSort("double_field2").order(SortOrder.DESC).unmappedType("double")) - .get(); assertSortValues( - resp, + client().prepareSearch("test1", "test2") + .addSort(fieldSort("double_field").order(SortOrder.ASC).unmappedType("double")) + .addSort(fieldSort("double_field2").order(SortOrder.DESC).unmappedType("double")), new Object[] { 0.65, Double.NEGATIVE_INFINITY }, new Object[] { Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY } ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java index 170b11f40ad49..d5654c8458229 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java @@ -443,11 +443,8 @@ public void test2920() throws IOException { refresh(); Script sortScript = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "\u0027\u0027", Collections.emptyMap()); - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(scriptSort(sortScript, ScriptSortType.STRING)) - .setSize(10) - .get(); - assertNoFailures(searchResponse); + assertNoFailures( + client().prepareSearch().setQuery(matchAllQuery()).addSort(scriptSort(sortScript, ScriptSortType.STRING)).setSize(10) + ); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java index 0afcc102bbc37..d319eab2b192b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java @@ -63,8 +63,8 @@ public void testThreadNames() throws Exception { indexRandom(true, builders); int numSearches = randomIntBetween(2, 100); for (int i = 0; i < numSearches; i++) { - assertNoFailures(client().prepareSearch("idx").setQuery(QueryBuilders.termQuery("str_value", "s" + i)).get()); - assertNoFailures(client().prepareSearch("idx").setQuery(QueryBuilders.termQuery("l_value", i)).get()); + assertNoFailures(client().prepareSearch("idx").setQuery(QueryBuilders.termQuery("str_value", "s" + i))); + assertNoFailures(client().prepareSearch("idx").setQuery(QueryBuilders.termQuery("l_value", i))); } Set threadNames = new HashSet<>(); for (long l : threadBean.getAllThreadIds()) { diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index abc95e8e89e44..e12163f5cb7e9 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -1108,11 +1108,12 @@ public void testSetSearchThrottled() { Index index = resolveIndex("throttled_threadpool_index"); assertTrue(service.getIndicesService().indexServiceSafe(index).getIndexSettings().isSearchThrottled()); client().prepareIndex("throttled_threadpool_index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); - SearchResponse searchResponse = client().prepareSearch("throttled_threadpool_index") - .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED) - .setSize(1) - .get(); - assertSearchHits(searchResponse, "1"); + assertSearchHits( + client().prepareSearch("throttled_threadpool_index") + .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED) + .setSize(1), + "1" + ); // we add a search action listener in a plugin above to assert that this is actually used client().execute( InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.INSTANCE, diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index 016fbb1efb772..7868043ac61b3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -227,6 +227,15 @@ public static void assertNoSearchHits(SearchResponse searchResponse) { assertThat(searchResponse.getHits().getHits(), emptyArray()); } + public static void assertSearchHits(SearchRequestBuilder searchRequestBuilder, String... ids) { + var res = searchRequestBuilder.get(); + try { + assertSearchHits(res, ids); + } finally { + res.decRef(); + } + } + public static void assertSearchHits(SearchResponse searchResponse, String... ids) { assertThat( "Incorrect SearchHit ids. " + formatShardStatus(searchResponse), @@ -235,13 +244,27 @@ public static void assertSearchHits(SearchResponse searchResponse, String... ids ); } - public static void assertSortValues(SearchResponse searchResponse, Object[]... sortValues) { - assertSearchResponse(searchResponse); - SearchHit[] hits = searchResponse.getHits().getHits(); - assertEquals(sortValues.length, hits.length); - for (int i = 0; i < sortValues.length; ++i) { - final Object[] hitsSortValues = hits[i].getSortValues(); - assertArrayEquals("Offset " + i + ", id " + hits[i].getId(), sortValues[i], hitsSortValues); + public static void assertSortValues(SearchRequestBuilder searchRequestBuilder, Object[]... sortValues) { + var searchResponse = searchRequestBuilder.get(); + try { + assertSearchResponse(searchResponse); + SearchHit[] hits = searchResponse.getHits().getHits(); + assertEquals(sortValues.length, hits.length); + for (int i = 0; i < sortValues.length; ++i) { + final Object[] hitsSortValues = hits[i].getSortValues(); + assertArrayEquals("Offset " + i + ", id " + hits[i].getId(), sortValues[i], hitsSortValues); + } + } finally { + searchResponse.decRef(); + } + } + + public static void assertOrderedSearchHits(SearchRequestBuilder searchRequestBuilder, String... ids) { + var res = searchRequestBuilder.get(); + try { + assertOrderedSearchHits(res, ids); + } finally { + res.decRef(); } } @@ -296,6 +319,15 @@ public static void assertSearchHit(SearchResponse searchResponse, int number, Ma assertThat(searchResponse.getHits().getAt(number - 1), matcher); } + public static void assertNoFailures(SearchRequestBuilder searchRequestBuilder) { + var res = searchRequestBuilder.get(); + try { + assertNoFailures(res); + } finally { + res.decRef(); + } + } + public static void assertNoFailures(SearchResponse searchResponse) { assertThat( "Unexpected ShardFailures: " + Arrays.toString(searchResponse.getShardFailures()), From 5697fcf594433bff8cc2fe97de71e52b49f0ac63 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 18 Oct 2023 10:00:09 +0300 Subject: [PATCH 02/23] `WaitForSnapshotStep` verifies if the index belongs to the latest snapshot of that SLM policy (#100911) The `WaitForSnapshotStep` used to check if the SLM policy has been executed after the index has entered the delete phase, but it did not check if the SLM policy included this index. The result of this is that if the user used an SLM policy that did not include this index, when the index would enter the `WaitForSnapshotStep`, it would wait for a snapshot to be taken, a snapshot that would not include the index, and then ILM would delete the index. See the exact reproduction path: https://github.com/elastic/elasticsearch/issues/57809 **Solution** This PR, after it finds a successful SLM run, it verifies if the snapshot taken by SLM contains this index. If not it throws an error, otherwise it proceeds. ILM explain will report: ``` "step_info": { "type": "illegal_state_exception", "reason": "the last successful snapshot of policy 'hourly-snapshots' does not include index '.ds-my-other-stream-2023.10.16-000001'" } ``` **Backwards compatibility concerns** In this PR, the `WaitForSnapshotStep` changed from `ClusterStateWaitStep` to `AsyncWaitStep`. We do not think this is gonna cause an issue. This was tested manually by the following steps: - Run a master node with the old version. - When ILM is executing `wait-for-snapshot`, we shutdown the node - We start the node again with the new version os ES - ES was able to pick up the step and continue with the new code. We believe that this covers bwc concerns. Fixes: https://github.com/elastic/elasticsearch/issues/57809 --- docs/changelog/100911.yaml | 6 + .../xpack/core/ilm/WaitForSnapshotAction.java | 2 +- .../xpack/core/ilm/WaitForSnapshotStep.java | 62 ++++- .../core/ilm/WaitForSnapshotStepTests.java | 259 +++++++++++++++--- 4 files changed, 282 insertions(+), 47 deletions(-) create mode 100644 docs/changelog/100911.yaml diff --git a/docs/changelog/100911.yaml b/docs/changelog/100911.yaml new file mode 100644 index 0000000000000..baab6f2482a76 --- /dev/null +++ b/docs/changelog/100911.yaml @@ -0,0 +1,6 @@ +pr: 100911 +summary: '`WaitForSnapshotStep` verifies if the index belongs to the latest snapshot + of that SLM policy' +area: ILM+SLM +type: bug +issues: [] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotAction.java index c824c74f48fb9..08a884f0b8f3c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotAction.java @@ -62,7 +62,7 @@ public String getPolicy() { @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey waitForSnapshotKey = new StepKey(phase, NAME, WaitForSnapshotStep.NAME); - return Collections.singletonList(new WaitForSnapshotStep(waitForSnapshotKey, nextStepKey, policy)); + return Collections.singletonList(new WaitForSnapshotStep(waitForSnapshotKey, nextStepKey, client, policy)); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStep.java index 82604e0e5a9b4..9c24324f706ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStep.java @@ -8,10 +8,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xpack.core.ilm.step.info.EmptyInfo; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata; @@ -20,11 +25,12 @@ import java.util.Objects; /*** - * A step that waits for snapshot to be taken by SLM to ensure we have backup before we delete the index. - * It will signal error if it can't get data needed to do the check (action time from ILM and SLM metadata) - * and will only return success if execution of SLM policy took place after index entered the wait for snapshot action. + * A step that waits for snapshot to be taken by SLM that includes the index in question to ensure we have backup + * before we delete the index. It will signal error if it can't get data needed to do the check (action time from ILM + * and SLM metadata) and will only return success if execution of SLM policy took place after index entered the wait + * for snapshot action and the latest successful snapshot includes the index. */ -public class WaitForSnapshotStep extends ClusterStateWaitStep { +public class WaitForSnapshotStep extends AsyncWaitStep { static final String NAME = "wait-for-snapshot"; private static final Logger logger = LogManager.getLogger(WaitForSnapshotStep.class); @@ -32,32 +38,40 @@ public class WaitForSnapshotStep extends ClusterStateWaitStep { private static final String MESSAGE_FIELD = "message"; private static final String POLICY_NOT_EXECUTED_MESSAGE = "waiting for policy '%s' to be executed since %s"; private static final String POLICY_NOT_FOUND_MESSAGE = "configured policy '%s' not found"; + private static final String INDEX_NOT_INCLUDED_IN_SNAPSHOT_MESSAGE = + "the last successful snapshot of policy '%s' does not include index '%s'"; + + private static final String UNEXPECTED_SNAPSHOT_STATE_MESSAGE = + "unexpected number of snapshots retrieved for repository '%s' and snapshot '%s' (expected 1, found %d)"; private static final String NO_INDEX_METADATA_MESSAGE = "no index metadata found for index '%s'"; private static final String NO_ACTION_TIME_MESSAGE = "no information about ILM action start in index metadata for index '%s'"; private final String policy; - WaitForSnapshotStep(StepKey key, StepKey nextStepKey, String policy) { - super(key, nextStepKey); + WaitForSnapshotStep(StepKey key, StepKey nextStepKey, Client client, String policy) { + super(key, nextStepKey, client); this.policy = policy; } @Override - public Result isConditionMet(Index index, ClusterState clusterState) { - IndexMetadata indexMetadata = clusterState.metadata().index(index); + public void evaluateCondition(Metadata metadata, Index index, Listener listener, TimeValue masterTimeout) { + IndexMetadata indexMetadata = metadata.index(index); if (indexMetadata == null) { - throw error(NO_INDEX_METADATA_MESSAGE, index.getName()); + listener.onFailure(error(NO_INDEX_METADATA_MESSAGE, index.getName())); + return; } Long actionTime = indexMetadata.getLifecycleExecutionState().actionTime(); if (actionTime == null) { - throw error(NO_ACTION_TIME_MESSAGE, index.getName()); + listener.onFailure(error(NO_ACTION_TIME_MESSAGE, index.getName())); + return; } - SnapshotLifecycleMetadata snapMeta = clusterState.metadata().custom(SnapshotLifecycleMetadata.TYPE); + SnapshotLifecycleMetadata snapMeta = metadata.custom(SnapshotLifecycleMetadata.TYPE); if (snapMeta == null || snapMeta.getSnapshotConfigurations().containsKey(policy) == false) { - throw error(POLICY_NOT_FOUND_MESSAGE, policy); + listener.onFailure(error(POLICY_NOT_FOUND_MESSAGE, policy)); + return; } SnapshotLifecyclePolicyMetadata snapPolicyMeta = snapMeta.getSnapshotConfigurations().get(policy); if (snapPolicyMeta.getLastSuccess() == null @@ -79,7 +93,8 @@ public Result isConditionMet(Index index, ClusterState clusterState) { snapPolicyMeta.getLastSuccess().getSnapshotFinishTimestamp() ); } - return new Result(false, notExecutedMessage(actionTime)); + listener.onResponse(false, notExecutedMessage(actionTime)); + return; } logger.debug( "executing policy because snapshot start time {} is after action time {}, snapshot timestamp is {}", @@ -87,7 +102,24 @@ public Result isConditionMet(Index index, ClusterState clusterState) { actionTime, snapPolicyMeta.getLastSuccess().getSnapshotFinishTimestamp() ); - return new Result(true, null); + String snapshotName = snapPolicyMeta.getLastSuccess().getSnapshotName(); + String repositoryName = snapPolicyMeta.getPolicy().getRepository(); + GetSnapshotsRequest request = new GetSnapshotsRequest().repositories(repositoryName) + .snapshots(new String[] { snapshotName }) + .includeIndexNames(true) + .verbose(false); + getClient().admin().cluster().getSnapshots(request, ActionListener.wrap(response -> { + if (response.getSnapshots().size() != 1) { + listener.onFailure(error(UNEXPECTED_SNAPSHOT_STATE_MESSAGE, repositoryName, snapshotName, response.getSnapshots().size())); + } else { + if (response.getSnapshots().get(0).indices().contains(index.getName())) { + listener.onResponse(true, EmptyInfo.INSTANCE); + } else { + listener.onFailure(error(INDEX_NOT_INCLUDED_IN_SNAPSHOT_MESSAGE, policy, index.getName())); + } + } + }, listener::onFailure)); + } public String getPolicy() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java index df7d351f2c1af..ed1cb477c30ef 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotStepTests.java @@ -6,6 +6,11 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; +import org.elasticsearch.client.internal.ClusterAdminClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -13,19 +18,41 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xpack.core.ilm.step.info.EmptyInfo; import org.elasticsearch.xpack.core.slm.SnapshotInvocationRecord; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy; import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata; +import org.junit.Before; +import org.mockito.Mockito; import java.io.IOException; +import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; + public class WaitForSnapshotStepTests extends AbstractStepTestCase { + private final ClusterAdminClient clusterAdminClient = mock(ClusterAdminClient.class); + + @Before + public void setupClusterClient() { + Mockito.when(adminClient.cluster()).thenReturn(clusterAdminClient); + } + @Override protected WaitForSnapshotStep createRandomInstance() { - return new WaitForSnapshotStep(randomStepKey(), randomStepKey(), randomAlphaOfLengthBetween(1, 10)); + return new WaitForSnapshotStep(randomStepKey(), randomStepKey(), client, randomAlphaOfLengthBetween(1, 10)); } @Override @@ -41,12 +68,12 @@ protected WaitForSnapshotStep mutateInstance(WaitForSnapshotStep instance) { default -> throw new AssertionError("Illegal randomisation branch"); } - return new WaitForSnapshotStep(key, nextKey, policy); + return new WaitForSnapshotStep(key, nextKey, client, policy); } @Override protected WaitForSnapshotStep copyInstance(WaitForSnapshotStep instance) { - return new WaitForSnapshotStep(instance.getKey(), instance.getNextStepKey(), instance.getPolicy()); + return new WaitForSnapshotStep(instance.getKey(), instance.getNextStepKey(), client, instance.getPolicy()); } public void testNoSlmPolicies() { @@ -60,14 +87,24 @@ public void testNoSlmPolicies() { Metadata.Builder meta = Metadata.builder().indices(indices); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(meta).build(); WaitForSnapshotStep instance = createRandomInstance(); - IllegalStateException e = expectThrows( - IllegalStateException.class, - () -> instance.isConditionMet(indexMetadata.getIndex(), clusterState) - ); - assertTrue(e.getMessage().contains(instance.getPolicy())); + SetOnce error = new SetOnce<>(); + instance.evaluateCondition(clusterState.metadata(), indexMetadata.getIndex(), new AsyncWaitStep.Listener() { + @Override + public void onResponse(boolean conditionMet, ToXContentObject info) { + logger.warn("expected an error got unexpected response {}", conditionMet); + throw new AssertionError("unexpected method call"); + } + + @Override + public void onFailure(Exception e) { + error.set(e); + } + }, MASTER_TIMEOUT); + + assertThat(error.get().getMessage(), containsString("'" + instance.getPolicy() + "' not found")); } - public void testSlmPolicyNotExecuted() throws IOException { + public void testSlmPolicyNotExecuted() { WaitForSnapshotStep instance = createRandomInstance(); SnapshotLifecyclePolicyMetadata slmPolicy = SnapshotLifecyclePolicyMetadata.builder() .setModifiedDate(randomLong()) @@ -88,9 +125,23 @@ public void testSlmPolicyNotExecuted() throws IOException { Map indices = Map.of(indexMetadata.getIndex().getName(), indexMetadata); Metadata.Builder meta = Metadata.builder().indices(indices).putCustom(SnapshotLifecycleMetadata.TYPE, smlMetadata); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(meta).build(); - ClusterStateWaitStep.Result result = instance.isConditionMet(indexMetadata.getIndex(), clusterState); - assertFalse(result.isComplete()); - assertTrue(getMessage(result).contains("to be executed")); + SetOnce isConditionMet = new SetOnce<>(); + SetOnce informationContext = new SetOnce<>(); + instance.evaluateCondition(clusterState.metadata(), indexMetadata.getIndex(), new AsyncWaitStep.Listener() { + @Override + public void onResponse(boolean conditionMet, ToXContentObject info) { + isConditionMet.set(conditionMet); + informationContext.set(info); + } + + @Override + public void onFailure(Exception e) { + logger.warn("unexpected onFailure call", e); + throw new AssertionError("unexpected method call"); + } + }, MASTER_TIMEOUT); + assertThat(isConditionMet.get(), is(false)); + assertTrue(toString(informationContext.get()).contains("to be executed")); } public void testSlmPolicyExecutedBeforeStep() throws IOException { @@ -98,9 +149,36 @@ public void testSlmPolicyExecutedBeforeStep() throws IOException { assertSlmPolicyExecuted(false, false); } - public void testSlmPolicyExecutedAfterStep() throws IOException { + public void testSlmPolicyExecutedAfterStep() { + String repoName = randomAlphaOfLength(10); + String snapshotName = randomAlphaOfLength(10); + String indexName = randomAlphaOfLength(10); // The snapshot was started and finished after the phase time, so we do expect the step to finish: - assertSlmPolicyExecuted(true, true); + GetSnapshotsResponse response = new GetSnapshotsResponse( + List.of( + new SnapshotInfo( + new Snapshot(randomAlphaOfLength(10), new SnapshotId(snapshotName, randomAlphaOfLength(10))), + List.of(indexName), + List.of(), + List.of(), + SnapshotState.SUCCESS + ) + ), + Map.of(), + null, + 0, + 0 + ); + Mockito.doAnswer(invocationOnMock -> { + GetSnapshotsRequest request = (GetSnapshotsRequest) invocationOnMock.getArguments()[0]; + assertGetSnapshotRequest(repoName, snapshotName, request); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(response); + return null; + }).when(clusterAdminClient).getSnapshots(any(), any()); + + assertSlmPolicyExecuted(repoName, snapshotName, indexName, true, true); } public void testSlmPolicyNotExecutedWhenStartIsBeforePhaseTime() throws IOException { @@ -108,16 +186,102 @@ public void testSlmPolicyNotExecutedWhenStartIsBeforePhaseTime() throws IOExcept assertSlmPolicyExecuted(false, true); } - private void assertSlmPolicyExecuted(boolean startTimeAfterPhaseTime, boolean finishTimeAfterPhaseTime) throws IOException { + public void testIndexNotBackedUpYet() { + String repoName = randomAlphaOfLength(10); + String snapshotName = randomAlphaOfLength(10); + String indexName = randomAlphaOfLength(10); + + // The latest snapshot does not contain the index we are interested in + GetSnapshotsResponse response = new GetSnapshotsResponse( + List.of( + new SnapshotInfo( + new Snapshot(randomAlphaOfLength(10), new SnapshotId(snapshotName, randomAlphaOfLength(10))), + List.of(), + List.of(), + List.of(), + SnapshotState.SUCCESS + ) + ), + Map.of(), + null, + 0, + 0 + ); + Mockito.doAnswer(invocationOnMock -> { + GetSnapshotsRequest request = (GetSnapshotsRequest) invocationOnMock.getArguments()[0]; + assertGetSnapshotRequest(repoName, snapshotName, request); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(response); + return null; + }).when(clusterAdminClient).getSnapshots(any(), any()); + + long phaseTime = randomLongBetween(100, 100000); + long actionTime = phaseTime + randomLongBetween(100, 100000); + WaitForSnapshotStep instance = createRandomInstance(); + SnapshotLifecyclePolicyMetadata slmPolicy = SnapshotLifecyclePolicyMetadata.builder() + .setModifiedDate(randomLong()) + .setPolicy(new SnapshotLifecyclePolicy("", "", "", repoName, null, null)) + .setLastSuccess(new SnapshotInvocationRecord(snapshotName, actionTime + 10, actionTime + 100, "")) + .build(); + SnapshotLifecycleMetadata smlMetadata = new SnapshotLifecycleMetadata( + Map.of(instance.getPolicy(), slmPolicy), + OperationMode.RUNNING, + null + ); + + IndexMetadata indexMetadata = IndexMetadata.builder(indexName) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Map.of("action_time", Long.toString(actionTime))) + .settings(settings(IndexVersion.current())) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)) + .build(); + Map indices = Map.of(indexMetadata.getIndex().getName(), indexMetadata); + Metadata.Builder meta = Metadata.builder().indices(indices).putCustom(SnapshotLifecycleMetadata.TYPE, smlMetadata); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(meta).build(); + SetOnce error = new SetOnce<>(); + instance.evaluateCondition(clusterState.metadata(), indexMetadata.getIndex(), new AsyncWaitStep.Listener() { + @Override + public void onResponse(boolean conditionMet, ToXContentObject info) { + logger.warn("expected an error got unexpected response {}", conditionMet); + throw new AssertionError("unexpected method call"); + } + + @Override + public void onFailure(Exception e) { + error.set(e); + } + }, MASTER_TIMEOUT); + + assertThat(error.get().getMessage(), containsString("does not include index '" + indexName + "'")); + } + + private void assertSlmPolicyExecuted(boolean startTimeAfterPhaseTime, boolean finishTimeAfterPhaseTime) { + assertSlmPolicyExecuted( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + startTimeAfterPhaseTime, + finishTimeAfterPhaseTime + ); + } + + private void assertSlmPolicyExecuted( + String repoName, + String snapshotName, + String indexName, + boolean startTimeAfterPhaseTime, + boolean finishTimeAfterPhaseTime + ) { long phaseTime = randomLong(); WaitForSnapshotStep instance = createRandomInstance(); SnapshotLifecyclePolicyMetadata slmPolicy = SnapshotLifecyclePolicyMetadata.builder() .setModifiedDate(randomLong()) - .setPolicy(new SnapshotLifecyclePolicy("", "", "", "", null, null)) + .setPolicy(new SnapshotLifecyclePolicy("", "", "", repoName, null, null)) .setLastSuccess( new SnapshotInvocationRecord( - "", + snapshotName, phaseTime + (startTimeAfterPhaseTime ? 10 : -100), phaseTime + (finishTimeAfterPhaseTime ? 100 : -10), "" @@ -130,7 +294,7 @@ private void assertSlmPolicyExecuted(boolean startTimeAfterPhaseTime, boolean fi null ); - IndexMetadata indexMetadata = IndexMetadata.builder(randomAlphaOfLength(10)) + IndexMetadata indexMetadata = IndexMetadata.builder(indexName) .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, Map.of("action_time", Long.toString(phaseTime))) .settings(settings(IndexVersion.current())) .numberOfShards(randomIntBetween(1, 5)) @@ -139,17 +303,40 @@ private void assertSlmPolicyExecuted(boolean startTimeAfterPhaseTime, boolean fi Map indices = Map.of(indexMetadata.getIndex().getName(), indexMetadata); Metadata.Builder meta = Metadata.builder().indices(indices).putCustom(SnapshotLifecycleMetadata.TYPE, smlMetadata); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(meta).build(); - ClusterStateWaitStep.Result result = instance.isConditionMet(indexMetadata.getIndex(), clusterState); + SetOnce isConditionMet = new SetOnce<>(); + SetOnce informationContext = new SetOnce<>(); + instance.evaluateCondition(clusterState.metadata(), indexMetadata.getIndex(), new AsyncWaitStep.Listener() { + @Override + public void onResponse(boolean conditionMet, ToXContentObject info) { + isConditionMet.set(conditionMet); + informationContext.set(info); + } + + @Override + public void onFailure(Exception e) { + logger.warn("unexpected onFailure call", e); + throw new AssertionError("unexpected method call"); + } + }, MASTER_TIMEOUT); if (startTimeAfterPhaseTime) { - assertTrue(result.isComplete()); - assertNull(result.getInfomationContext()); + assertThat(isConditionMet.get(), is(true)); + assertThat(informationContext.get(), is(EmptyInfo.INSTANCE)); } else { - assertFalse(result.isComplete()); - assertTrue(getMessage(result).contains("to be executed")); + assertThat(isConditionMet.get(), is(false)); + assertThat(toString(informationContext.get()), containsString("to be executed")); } } - public void testNullStartTime() throws IOException { + private void assertGetSnapshotRequest(String repoName, String snapshotName, GetSnapshotsRequest request) { + assertThat(request.repositories().length, is(1)); + assertThat(request.repositories()[0], equalTo(repoName)); + assertThat(request.snapshots().length, is(1)); + assertThat(request.snapshots()[0], equalTo(snapshotName)); + assertThat(request.includeIndexNames(), is(true)); + assertThat(request.verbose(), is(false)); + } + + public void testNullStartTime() { long phaseTime = randomLong(); WaitForSnapshotStep instance = createRandomInstance(); @@ -173,14 +360,24 @@ public void testNullStartTime() throws IOException { Map indices = Map.of(indexMetadata.getIndex().getName(), indexMetadata); Metadata.Builder meta = Metadata.builder().indices(indices).putCustom(SnapshotLifecycleMetadata.TYPE, smlMetadata); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(meta).build(); - IllegalStateException e = expectThrows( - IllegalStateException.class, - () -> instance.isConditionMet(indexMetadata.getIndex(), clusterState) - ); - assertTrue(e.getMessage().contains("no information about ILM action start")); + SetOnce error = new SetOnce<>(); + instance.evaluateCondition(clusterState.metadata(), indexMetadata.getIndex(), new AsyncWaitStep.Listener() { + @Override + public void onResponse(boolean conditionMet, ToXContentObject info) { + logger.warn("expected an error got unexpected response {}", conditionMet); + throw new AssertionError("unexpected method call"); + } + + @Override + public void onFailure(Exception e) { + error.set(e); + } + }, MASTER_TIMEOUT); + + assertThat(error.get().getMessage(), containsString("no information about ILM action start")); } - private String getMessage(ClusterStateWaitStep.Result result) throws IOException { - return Strings.toString(result.getInfomationContext()); + private String toString(ToXContentObject info) { + return Strings.toString(info); } } From 76488e25257a8983ae1fa148223b9e3fb79ebf67 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 18 Oct 2023 08:06:42 +0100 Subject: [PATCH 03/23] Add/fix known issue docs for 8.10.4 (#101013) 8.10.4 includes a partial mitigation for the snapshots downgrades bug introduced in 8.10.0. This commit adds known-issue docs for 8.10.4, and adjusts the known-issue docs for earlier 8.10.x issues. --- docs/reference/release-notes/8.10.0.asciidoc | 8 ++++--- docs/reference/release-notes/8.10.4.asciidoc | 24 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/reference/release-notes/8.10.0.asciidoc b/docs/reference/release-notes/8.10.0.asciidoc index b8a57c702b8e7..9fbe7a2b1d099 100644 --- a/docs/reference/release-notes/8.10.0.asciidoc +++ b/docs/reference/release-notes/8.10.0.asciidoc @@ -28,9 +28,11 @@ first. If you cannot repair the repository in this way, first delete all the snapshots in the repository taken with version 8.9.0 or later. To do this will require using a cluster running version 8.10.0 or later. + -If you wish to downgrade to a version in the 8.9 series, you must take or -delete a snapshot using a cluster running version 8.11.0 or later to repair the -repository format first. +If you wish to downgrade to a version in the 8.9 series, you must take or delete +a snapshot using a cluster running version 8.11.0 or later to repair the +repository format first. If you cannot repair the repository in this way, first +delete all the snapshots in the repository taken with version 8.10.0 or later +using a cluster running version 8.10.4. // end::repositorydata-format-change[] [[breaking-8.10.0]] diff --git a/docs/reference/release-notes/8.10.4.asciidoc b/docs/reference/release-notes/8.10.4.asciidoc index 92221b89f5d0e..f2e95af71afcb 100644 --- a/docs/reference/release-notes/8.10.4.asciidoc +++ b/docs/reference/release-notes/8.10.4.asciidoc @@ -1,6 +1,30 @@ [[release-notes-8.10.4]] == {es} version 8.10.4 +[[known-issues-8.10.4]] +[float] +=== Known issues + +* Snapshot-based downgrades ++ +The snapshot repository format changed in a manner that prevents earlier +versions of Elasticsearch from reading the repository contents if it contains +snapshots from this version and the last cluster to write to this repository was +in the 8.10 series. This will prevent you from reverting an upgrade to the 8.10 +series by restoring a snapshot taken before the upgrade. ++ +Snapshot repositories written by clusters running versions 8.11.0 and later are +compatible with all earlier versions. Moreover, clusters running version 8.11.0 +or later will also automatically repair the repository format the first time +they write to the repository to take or delete a snapshot, making it so that all +earlier versions can read its contents again. ++ +If you wish to downgrade to a version prior to 8.10.0, take or delete a snapshot +using a cluster running version 8.11.0 or later to repair the repository format +first. If you cannot repair the repository in this way, first delete all the +snapshots in the repository taken with version 8.10.0 or later using a cluster +running version 8.10.4. + Also see <>. [[bug-8.10.4]] From 86ed7800f702d222c22e887447316ff931827258 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 18 Oct 2023 09:47:55 +0200 Subject: [PATCH 04/23] Fix geo tile bounding boxes to be consistent with arithmetic method (#100826) Added some logic to GeoTileUtils#toBoundingBox that make sure the generated bounding boxes are consistent with the arithmetic solution and hence we can use both approaches indistintable. --- docs/changelog/100826.yaml | 7 +++ .../elasticsearch/common/geo/GeoUtils.java | 15 +++++ .../geogrid/GeoTileBoundedPredicate.java | 11 ++-- .../bucket/geogrid/GeoTileUtils.java | 38 +++++++++++-- .../bucket/geogrid/GeoTileUtilsTests.java | 57 ++++++++++++++++++- .../GeoGridAggAndQueryConsistencyIT.java | 11 +--- .../index/query/GeoGridQueryBuilderTests.java | 18 +++--- 7 files changed, 128 insertions(+), 29 deletions(-) create mode 100644 docs/changelog/100826.yaml diff --git a/docs/changelog/100826.yaml b/docs/changelog/100826.yaml new file mode 100644 index 0000000000000..1b1729d1491ea --- /dev/null +++ b/docs/changelog/100826.yaml @@ -0,0 +1,7 @@ +pr: 100826 +summary: Fix geo tile bounding boxes to be consistent with arithmetic method +area: Geo +type: bug +issues: + - 92611 + - 95574 diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index d65169148635c..39d5d90cc6c04 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -8,6 +8,7 @@ package org.elasticsearch.common.geo; +import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.util.SloppyMath; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.unit.DistanceUnit; @@ -566,5 +567,19 @@ public boolean advanceExact(int target) throws IOException { } } + /** + * Transforms the provided longitude to the equivalent in lucene quantize space. + */ + public static double quantizeLon(double lon) { + return GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); + } + + /** + * Transforms the provided latitude to the equivalent in lucene quantize space. + */ + public static double quantizeLat(double lat) { + return GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); + } + private GeoUtils() {} } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileBoundedPredicate.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileBoundedPredicate.java index ac64e8f24627a..65e90637ac473 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileBoundedPredicate.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileBoundedPredicate.java @@ -11,6 +11,9 @@ import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.geometry.Rectangle; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude; + /** * Filters out tiles using the provided bounds at the provided precision. In order to be efficient it works on the X/Y coordinates of the * geotile scheme. @@ -35,15 +38,15 @@ public GeoTileBoundedPredicate(int precision, GeoBoundingBox bbox) { final int minY = GeoTileUtils.getYTile(bbox.top(), tiles); final Rectangle minTile = GeoTileUtils.toBoundingBox(minX, minY, precision); // touching tiles are excluded, they need to share at least one interior point - this.leftX = minTile.getMaxX() == bbox.left() ? minX + 1 : minX; - this.minY = minTile.getMinY() == bbox.top() ? minY + 1 : minY; + this.leftX = encodeLongitude(minTile.getMaxX()) == encodeLongitude(bbox.left()) ? minX + 1 : minX; + this.minY = encodeLatitude(minTile.getMinY()) == encodeLatitude(bbox.top()) ? minY + 1 : minY; // compute maxX, maxY final int maxX = GeoTileUtils.getXTile(bbox.right(), tiles); final int maxY = GeoTileUtils.getYTile(bbox.bottom(), tiles); final Rectangle maxTile = GeoTileUtils.toBoundingBox(maxX, maxY, precision); // touching tiles are excluded, they need to share at least one interior point - this.rightX = maxTile.getMinX() == bbox.right() ? maxX : maxX + 1; - this.maxY = maxTile.getMaxY() == bbox.bottom() ? maxY : maxY + 1; + this.rightX = encodeLongitude(maxTile.getMinX()) == encodeLongitude(bbox.right()) ? maxX : maxX + 1; + this.maxY = encodeLatitude(maxTile.getMaxY()) == encodeLatitude(bbox.bottom()) ? maxY : maxY + 1; if (crossesDateline) { this.maxTiles = ((long) tiles + this.rightX - this.leftX) * (this.maxY - this.minY); } else { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index a8abfd258f380..e5fd0aa10ced2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -22,6 +22,7 @@ import static org.elasticsearch.common.geo.GeoUtils.normalizeLat; import static org.elasticsearch.common.geo.GeoUtils.normalizeLon; +import static org.elasticsearch.common.geo.GeoUtils.quantizeLat; /** * Implements geotile key hashing, same as used by many map tile implementations. @@ -41,6 +42,12 @@ private GeoTileUtils() {} private static final double PI_TIMES_4 = Math.PI * 4.0; + // precision up to geometry and arithmetic solution are consistent + private static final int MAX_TILES_FULL_PRECISION = 1 << 20; + + // lucene latitude resolution + static final double LUCENE_LAT_RES = 180.0D / (0x1L << 32); + /** * Largest number of tiles (precision) to use. * This value cannot be more than (64-5)/2 = 29, because 5 bits are used for zoom level itself (0-31) @@ -229,10 +236,12 @@ public static Rectangle toBoundingBox(String hash) { } /** - * Decode a bucket key to a bounding box of the tile corners + * Decode a bucket key to a bounding box of the tile corners. The points belonging + * to the max latitude and min longitude belong to the tile while the points + * belonging to the min latitude and max longitude belong to the next tile. */ public static Rectangle toBoundingBox(int xTile, int yTile, int precision) { - final double tiles = validateZXY(precision, xTile, yTile); + final int tiles = validateZXY(precision, xTile, yTile); return new Rectangle( tileToLon(xTile, tiles), // minLon tileToLon(xTile + 1, tiles), // maxLon @@ -256,10 +265,29 @@ private static double tileToLon(double xTile, double tiles) { * Decode a yTile into its latitude value */ public static double tileToLat(int yTile, int tiles) { - return tileToLat(yTile, (double) tiles); + final double lat = tileToLat((double) yTile, tiles); + if (tiles < MAX_TILES_FULL_PRECISION || yTile == 0 || yTile == tiles) { + return lat; // precise case, don't need to do more work + } + // Maybe adjust latitude due to numerical errors + final double qLat = quantizeLat(lat); + final int computedYTile = getYTile(qLat, tiles); + // the idea here is that the latitude returned belongs to the tile and the next latitude up belongs to the next tile + // therefore we can be in the current tile and we need to find the point up just before the next tile, + // or we are in the other tile and we need to find the first point down that belong to this tile. + return findBoundaryPoint(qLat, computedYTile, tiles, computedYTile == yTile ? LUCENE_LAT_RES : -LUCENE_LAT_RES); + } + + private static double findBoundaryPoint(double qLat, int yTile, int tiles, double step) { + final double nextQLat = qLat + step; + final int nextYTile = getYTile(nextQLat, tiles); + if (yTile != nextYTile) { + return step > 0 ? qLat : nextQLat; + } + return findBoundaryPoint(nextQLat, nextYTile, tiles, step); } - private static double tileToLat(double yTile, double tiles) { + private static double tileToLat(double yTile, int tiles) { final double n = Math.PI - (PI_TIMES_2 * yTile) / tiles; return Math.toDegrees(ESSloppyMath.atan(ESSloppyMath.sinh(n))); } @@ -281,7 +309,7 @@ private static int validateZXY(int zoom, int xTile, int yTile) { * Converts zoom/x/y integers into a GeoPoint. */ private static GeoPoint zxyToGeoPoint(int zoom, int xTile, int yTile) { - final double tiles = validateZXY(zoom, xTile, yTile); + final int tiles = validateZXY(zoom, xTile, yTile); return new GeoPoint(tileToLat(yTile + 0.5, tiles), tileToLon(xTile + 0.5, tiles)); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index 0ba1095937af1..e6de1e3aa5db8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -8,9 +8,12 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.checkPrecisionRange; @@ -235,9 +238,61 @@ public void testPointToTile() { assertThat(GeoTileUtils.getYTile(rectangle.getMinY(), tiles), anyOf(equalTo(yTile + 1), equalTo(yTile))); // check point inside double x = randomDoubleBetween(rectangle.getMinX(), rectangle.getMaxX(), false); - double y = randomDoubleBetween(rectangle.getMinY(), rectangle.getMaxY(), false); + double y = randomDoubleBetween(rectangle.getMinY() + GeoTileUtils.LUCENE_LAT_RES, rectangle.getMaxY(), false); assertThat(GeoTileUtils.getXTile(x, tiles), equalTo(xTile)); assertThat(GeoTileUtils.getYTile(y, tiles), equalTo(yTile)); } + + public void testEncodingLuceneLonConsistency() { + final double qLon = GeoEncodingUtils.decodeLongitude(randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE)); + for (int zoom = 0; zoom <= MAX_ZOOM; zoom++) { + final int tiles = 1 << zoom; + final int x = GeoTileUtils.getXTile(qLon, tiles); + final Rectangle rectangle = GeoTileUtils.toBoundingBox(x, randomIntBetween(0, tiles - 1), zoom); + // max longitude belongs to the next tile except the last one + assertThat( + GeoTileUtils.getXTile(GeoUtils.quantizeLon(rectangle.getMaxX()), tiles), + Matchers.anyOf(equalTo(x + 1), equalTo(tiles - 1)) + ); + // next encoded value down belongs to the tile + assertThat(GeoTileUtils.getXTile(quantizeLonDown(rectangle.getMaxX()), tiles), equalTo(x)); + // min longitude belongs to the tile + assertThat(GeoTileUtils.getXTile(GeoUtils.quantizeLon(rectangle.getMinX()), tiles), equalTo(x)); + if (x != 0) { + // next encoded value down belongs to the previous tile + assertThat(GeoTileUtils.getXTile(quantizeLonDown(rectangle.getMinX()), tiles), equalTo(x - 1)); + } + } + } + + public void testEncodingLuceneLatConsistency() { + final double qLat = GeoEncodingUtils.decodeLatitude(randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE)); + for (int zoom = 0; zoom <= MAX_ZOOM; zoom++) { + final int tiles = 1 << zoom; + final int y = GeoTileUtils.getYTile(qLat, tiles); + final Rectangle rectangle = GeoTileUtils.toBoundingBox(randomIntBetween(0, tiles - 1), y, zoom); + // max latitude belongs to the tile + assertThat(GeoTileUtils.getYTile(GeoUtils.quantizeLat(rectangle.getMaxLat()), tiles), equalTo(y)); + if (y != 0) { + // next encoded value up belongs to the previous tile + assertThat(GeoTileUtils.getYTile(quantizeLatUp(rectangle.getMaxLat()), tiles), equalTo(y - 1)); + } + // min latitude belongs to the next tile except the last one + assertThat( + GeoTileUtils.getYTile(GeoUtils.quantizeLat(rectangle.getMinLat()), tiles), + Matchers.anyOf(equalTo(y + 1), equalTo(tiles - 1)) + ); + // next encoded value up belongs to the tile + assertThat(GeoTileUtils.getYTile(quantizeLatUp(rectangle.getMinLat()), tiles), equalTo(y)); + } + } + + private static double quantizeLonDown(double lon) { + return GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon) - 1); + } + + private static double quantizeLatUp(double lat) { + return GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat) + 1); + } } diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java index 81b4c110602a8..e70fc324064ad 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java @@ -63,7 +63,6 @@ public void testGeoPointGeoHash() throws IOException { public void testGeoPointGeoTile() throws IOException { doTestGeotileGrid( GeoPointFieldMapper.CONTENT_TYPE, - GeoTileUtils.MAX_ZOOM - 4, // levels 26 and above have some rounding errors, but this is past the index resolution // just generate points on bounds () -> randomValueOtherThanMany( p -> p.getLat() > GeoTileUtils.NORMALIZED_LATITUDE_MASK || p.getLat() < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK, @@ -81,11 +80,7 @@ public void testGeoShapeGeoHash() throws IOException { } public void testGeoShapeGeoTile() throws IOException { - doTestGeotileGrid( - GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE, - GeoTileUtils.MAX_ZOOM - 1, - () -> GeometryTestUtils.randomGeometryWithoutCircle(0, false) - ); + doTestGeotileGrid(GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE, () -> GeometryTestUtils.randomGeometryWithoutCircle(0, false)); } public void testGeoShapeGeoHex() throws IOException { @@ -198,10 +193,10 @@ private void doTestGeohashGrid(String fieldType, Supplier randomGeomet ); } - private void doTestGeotileGrid(String fieldType, int maxPrecision, Supplier randomGeometriesSupplier) throws IOException { + private void doTestGeotileGrid(String fieldType, Supplier randomGeometriesSupplier) throws IOException { doTestGrid( 0, - maxPrecision, + GeoTileUtils.MAX_ZOOM, fieldType, (precision, point) -> GeoTileUtils.stringEncode(GeoTileUtils.longEncode(point.getLon(), point.getLat(), precision)), tile -> toPoints(GeoTileUtils.toBoundingBox(tile)), diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java index fc782adc5c5c1..83eed8042e4de 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.spatial.index.query; -import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; @@ -34,6 +33,8 @@ import java.util.Arrays; import java.util.Collection; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.stringEncode; @@ -256,10 +257,8 @@ public void testIgnoreUnmapped() throws IOException { } public void testGeohashBoundingBox() { - double lat = randomDoubleBetween(-90d, 90d, true); - double lon = randomDoubleBetween(-180d, 180d, true); - double qLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); - double qLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); + double qLat = decodeLatitude(randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE)); + double qLon = decodeLongitude(randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE)); for (int zoom = 1; zoom <= Geohash.PRECISION; zoom++) { String hash = Geohash.stringEncode(qLon, qLat, zoom); Rectangle qRect = GeoGridQueryBuilder.getQueryHash(hash); @@ -278,13 +277,10 @@ private void assertBoundingBox(String hash, int precision, double lon, double la ); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/92611") public void testBoundingBoxQuantize() { - double lat = randomDoubleBetween(-GeoTileUtils.LATITUDE_MASK, GeoTileUtils.LATITUDE_MASK, true); - double lon = randomDoubleBetween(-180d, 180d, true); - double qLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); - double qLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); - for (int zoom = 0; zoom < MAX_ZOOM; zoom++) { + double qLat = decodeLatitude(randomIntBetween(GeoTileUtils.ENCODED_LATITUDE_MASK, GeoTileUtils.ENCODED_LATITUDE_MASK)); + double qLon = decodeLongitude(randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE)); + for (int zoom = 0; zoom <= MAX_ZOOM; zoom++) { long tile = GeoTileUtils.longEncode(qLon, qLat, zoom); Rectangle qRect = GeoGridQueryBuilder.getQueryTile(stringEncode(tile)); assertBoundingBox(tile, zoom, qLon, qLat, qRect); From 48fcb2e7d4651b95a8e22612e2ec001ad4c4bdc4 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 18 Oct 2023 09:11:23 +0100 Subject: [PATCH 05/23] [ML] Add "fake" availability zone mapper (#100956) To solve an issue in serverless where multiple trained model allocations are assigned to a single node, this PR introduces the concept of "fake" availability zones, where each node is treated as being in its own availability zone. --- .../ml/DefaultMachineLearningExtension.java | 8 + .../xpack/ml/MachineLearning.java | 9 +- .../xpack/ml/MachineLearningExtension.java | 7 + .../AbstractNodeAvailabilityZoneMapper.java | 93 ++++++ .../NodeAvailabilityZoneMapper.java | 173 +---------- .../NodeFakeAvailabilityZoneMapper.java | 76 +++++ .../NodeRealAvailabilityZoneMapper.java | 124 ++++++++ .../xpack/ml/MachineLearningTests.java | 8 + .../MlAutoscalingDeciderServiceTests.java | 6 +- .../MlMemoryAutoscalingDeciderTests.java | 12 +- .../NodeFakeAvailabilityZoneMapperTests.java | 284 ++++++++++++++++++ ... NodeRealAvailabilityZoneMapperTests.java} | 94 +++--- ...nedModelAssignmentClusterServiceTests.java | 22 +- 13 files changed, 694 insertions(+), 222 deletions(-) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/AbstractNodeAvailabilityZoneMapper.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeFakeAvailabilityZoneMapper.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeRealAvailabilityZoneMapper.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/NodeFakeAvailabilityZoneMapperTests.java rename x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/{NodeAvailabilityZoneMapperTests.java => NodeRealAvailabilityZoneMapperTests.java} (70%) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java index fdc2c91cea3de..cddbe721d3d56 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java @@ -8,7 +8,11 @@ package org.elasticsearch.xpack.ml; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.xpack.ml.autoscaling.AbstractNodeAvailabilityZoneMapper; +import org.elasticsearch.xpack.ml.autoscaling.NodeRealAvailabilityZoneMapper; public class DefaultMachineLearningExtension implements MachineLearningExtension { @@ -51,4 +55,8 @@ public boolean isNlpEnabled() { public String[] getAnalyticsDestIndexAllowedSettings() { return ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS; } + + public AbstractNodeAvailabilityZoneMapper getNodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings) { + return new NodeRealAvailabilityZoneMapper(settings, clusterSettings); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index a7bbbbcffb8b3..cb38e23c7f8eb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -303,9 +303,9 @@ import org.elasticsearch.xpack.ml.aggs.kstest.BucketCountKSTestAggregationBuilder; import org.elasticsearch.xpack.ml.aggs.kstest.InternalKSTestAggregation; import org.elasticsearch.xpack.ml.annotations.AnnotationPersister; +import org.elasticsearch.xpack.ml.autoscaling.AbstractNodeAvailabilityZoneMapper; import org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingDeciderService; import org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingNamedWritableProvider; -import org.elasticsearch.xpack.ml.autoscaling.NodeAvailabilityZoneMapper; import org.elasticsearch.xpack.ml.datafeed.DatafeedConfigAutoUpdater; import org.elasticsearch.xpack.ml.datafeed.DatafeedContextProvider; import org.elasticsearch.xpack.ml.datafeed.DatafeedJobBuilder; @@ -1197,10 +1197,9 @@ public Collection createComponents( // Perform node startup operations nativeStorageProvider.cleanupLocalTmpStorageInCaseOfUncleanShutdown(); - NodeAvailabilityZoneMapper nodeAvailabilityZoneMapper = new NodeAvailabilityZoneMapper( - settings, - clusterService.getClusterSettings() - ); + AbstractNodeAvailabilityZoneMapper nodeAvailabilityZoneMapper = machineLearningExtension.get() + .getNodeAvailabilityZoneMapper(settings, clusterService.getClusterSettings()); + clusterService.addListener(nodeAvailabilityZoneMapper); // allocation service objects diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java index 24b93301f37ac..b0e65d524036c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java @@ -7,7 +7,10 @@ package org.elasticsearch.xpack.ml; +import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.ml.autoscaling.AbstractNodeAvailabilityZoneMapper; +import org.elasticsearch.xpack.ml.autoscaling.NodeRealAvailabilityZoneMapper; public interface MachineLearningExtension { @@ -26,4 +29,8 @@ default void configure(Settings settings) {} default String[] getAnalyticsDestIndexAllowedSettings() { return DefaultMachineLearningExtension.ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS; } + + default AbstractNodeAvailabilityZoneMapper getNodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings) { + return new NodeRealAvailabilityZoneMapper(settings, clusterSettings); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/AbstractNodeAvailabilityZoneMapper.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/AbstractNodeAvailabilityZoneMapper.java new file mode 100644 index 0000000000000..a4d6ec1662bb4 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/AbstractNodeAvailabilityZoneMapper.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.autoscaling; + +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; + +public abstract class AbstractNodeAvailabilityZoneMapper implements ClusterStateListener, NodeAvailabilityZoneMapper { + private volatile DiscoveryNodes lastDiscoveryNodes; + private volatile Map, Collection> allNodesByAvailabilityZone; + private volatile Map, Collection> mlNodesByAvailabilityZone; + + public AbstractNodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings) { + this(settings, clusterSettings, null); + } + + public AbstractNodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings, DiscoveryNodes discoveryNodes) { + lastDiscoveryNodes = discoveryNodes; + } + + /** + * @return A map whose keys are lists of attributes that together define an availability zone, and whose values are + * collections of nodes that have that combination of attributes. If availability zones + * are not configured then the map will contain one entry mapping an empty list to a collection of all nodes. If + * it is too early in the lifecycle of the node to know the answer then an empty map will be returned. + */ + public Map, Collection> getAllNodesByAvailabilityZone() { + return allNodesByAvailabilityZone; + } + + /** + * @return The number of availability zones in the cluster. If availability zones are not configured this will be 1. + * If it is too early in the lifecycle of the node to know the answer then {@link OptionalInt#empty()} will be returned. + */ + public OptionalInt getNumAvailabilityZones() { + return (lastDiscoveryNodes == null) ? OptionalInt.empty() : OptionalInt.of(allNodesByAvailabilityZone.size()); + } + + /** + * @return A map whose keys are lists of attributes that together define an availability zone, and whose values are + * collections of nodes that have that combination of attributes. If availability zones + * are not configured then the map will contain one entry mapping an empty list to a collection of all nodes. If + * it is too early in the lifecycle of the node to know the answer then an empty map will be returned. An empty + * map will also be returned if there are no ML nodes in the cluster. (These two empty map return scenarios can be + * distinguished by calling one of the other methods.) + */ + public Map, Collection> getMlNodesByAvailabilityZone() { + return mlNodesByAvailabilityZone; + } + + /** + * @return The number of availability zones in the cluster. If availability zones are not configured this will be 1. + * If it is too early in the lifecycle of the node to know the answer then {@link OptionalInt#empty()} will be returned. + * Unlike {@link #getNumAvailabilityZones()}, it is possible this method will return 0, as it is possible + * for a cluster to have no ML nodes. + */ + public OptionalInt getNumMlAvailabilityZones() { + return (lastDiscoveryNodes == null) ? OptionalInt.empty() : OptionalInt.of(mlNodesByAvailabilityZone.size()); + } + + @Override + public synchronized void clusterChanged(ClusterChangedEvent event) { + if (lastDiscoveryNodes == null || event.nodesChanged()) { + lastDiscoveryNodes = event.state().nodes(); + updateNodesByAvailabilityZone(); + } + } + + synchronized void updateNodesByAvailabilityZone() { + if (lastDiscoveryNodes == null) { + allNodesByAvailabilityZone = Map.of(); + mlNodesByAvailabilityZone = allNodesByAvailabilityZone; + return; + } + NodesByAvailabilityZone nodesByAvailabilityZone = buildNodesByAvailabilityZone(lastDiscoveryNodes); + this.allNodesByAvailabilityZone = nodesByAvailabilityZone.allNodes(); + this.mlNodesByAvailabilityZone = nodesByAvailabilityZone.mlNodes(); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeAvailabilityZoneMapper.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeAvailabilityZoneMapper.java index 2d99f9c2faab7..ceeaa9da75f31 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeAvailabilityZoneMapper.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeAvailabilityZoneMapper.java @@ -7,183 +7,30 @@ package org.elasticsearch.xpack.ml.autoscaling; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Settings; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.OptionalInt; -/** - * Maintains a list of the nodes in each availability zone, where an availability zone - * means a unique combination of values of the attributes listed in the cluster setting - * cluster.routing.allocation.awareness.attributes. - */ -public class NodeAvailabilityZoneMapper implements ClusterStateListener { - - private static final Logger logger = LogManager.getLogger(NodeAvailabilityZoneMapper.class); - - private volatile List awarenessAttributes; - private volatile DiscoveryNodes lastDiscoveryNodes; - private volatile Map, Collection> allNodesByAvailabilityZone; - private volatile Map, Collection> mlNodesByAvailabilityZone; - - public NodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings) { - this(settings, clusterSettings, null); - } - - public NodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings, DiscoveryNodes discoveryNodes) { - awarenessAttributes = AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.get(settings); - lastDiscoveryNodes = discoveryNodes; - updateNodesByAvailabilityZone(); - clusterSettings.addSettingsUpdateConsumer( - AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING, - this::setAwarenessAttributes - ); - } - - private synchronized void setAwarenessAttributes(List awarenessAttributes) { - this.awarenessAttributes = List.copyOf(awarenessAttributes); - updateNodesByAvailabilityZone(); - } - - public List getAwarenessAttributes() { - return awarenessAttributes; - } - +public interface NodeAvailabilityZoneMapper { /** - * @return A map whose keys are lists of awareness attribute values in the same order as the configured awareness attribute - * names, and whose values are collections of nodes that have that combination of attributes. If availability zones - * are not configured then the map will contain one entry mapping an empty list to a collection of all nodes. If - * it is too early in the lifecycle of the node to know the answer then an empty map will be returned. + * @param clusterState The specific cluster state whose nodes will be used to detect ML nodes by availability zone. + * @return A map whose keys are conceptually lists of availability zone attributes, and whose values are collections + * of nodes corresponding to the availability zone attributes. + * An empty map will be returned if there are no ML nodes in the cluster. */ - public Map, Collection> getAllNodesByAvailabilityZone() { - return allNodesByAvailabilityZone; - } + Map, Collection> buildMlNodesByAvailabilityZone(ClusterState clusterState); - /** - * @return The number of availability zones in the cluster. If availability zones are not configured this will be 1. - * If it is too early in the lifecycle of the node to know the answer then {@link OptionalInt#empty()} will be returned. - */ - public OptionalInt getNumAvailabilityZones() { - return (lastDiscoveryNodes == null) ? OptionalInt.empty() : OptionalInt.of(allNodesByAvailabilityZone.size()); - } + OptionalInt getNumMlAvailabilityZones(); - /** - * @return A map whose keys are lists of awareness attribute values in the same order as the configured awareness attribute - * names, and whose values are collections of nodes that have that combination of attributes. If availability zones - * are not configured then the map will contain one entry mapping an empty list to a collection of all nodes. If - * it is too early in the lifecycle of the node to know the answer then an empty map will be returned. An empty - * map will also be returned if there are no ML nodes in the cluster. (These two empty map return scenarios can be - * distinguished by calling one of the other methods.) - */ - public Map, Collection> getMlNodesByAvailabilityZone() { - return mlNodesByAvailabilityZone; - } - - /** - * @return The number of availability zones in the cluster. If availability zones are not configured this will be 1. - * If it is too early in the lifecycle of the node to know the answer then {@link OptionalInt#empty()} will be returned. - * Unlike {@link #getNumAvailabilityZones()}, it is possible this method will return 0, as it is possible - * for a cluster to have no ML nodes. - */ - public OptionalInt getNumMlAvailabilityZones() { - return (lastDiscoveryNodes == null) ? OptionalInt.empty() : OptionalInt.of(mlNodesByAvailabilityZone.size()); - } - - @Override - public synchronized void clusterChanged(ClusterChangedEvent event) { - if (lastDiscoveryNodes == null || event.nodesChanged()) { - lastDiscoveryNodes = event.state().nodes(); - updateNodesByAvailabilityZone(); - } - } - - private synchronized void updateNodesByAvailabilityZone() { - if (lastDiscoveryNodes == null) { - allNodesByAvailabilityZone = Map.of(); - mlNodesByAvailabilityZone = allNodesByAvailabilityZone; - return; - } - NodesByAvailabilityZone nodesByAvailabilityZone = buildNodesByAvailabilityZone(lastDiscoveryNodes, awarenessAttributes); - this.allNodesByAvailabilityZone = nodesByAvailabilityZone.allNodes; - this.mlNodesByAvailabilityZone = nodesByAvailabilityZone.mlNodes; - } - - private static NodesByAvailabilityZone buildNodesByAvailabilityZone(DiscoveryNodes discoveryNodes, List awarenessAttributes) { - Collection nodes = discoveryNodes.getNodes().values(); - - if (awarenessAttributes.isEmpty()) { - return new NodesByAvailabilityZone( - Map.of(List.of(), nodes), - Map.of(List.of(), nodes.stream().filter(n -> n.getRoles().contains(DiscoveryNodeRole.ML_ROLE)).toList()) - ); - } - - Map, Collection> allNodesByAvailabilityZone = new HashMap<>(); - Map, Collection> mlNodesByAvailabilityZone = new HashMap<>(); - for (DiscoveryNode node : nodes) { - List orderedNodeAttributeValues = awarenessAttributes.stream().map(a -> { - String v = node.getAttributes().get(a); - if (v == null) { - // This will never happen for a Cloud cluster, but for self-managed it's possible. - // We reuse the same allocation attributes that are used for shards, but self-managed - // users might not have bothered to set them on their ML nodes if they are not spread - // across availability zones. - logger.debug( - "Node [{}] does not have all configured awareness attributes {} - missing [{}]", - node, - awarenessAttributes, - a - ); - return ""; - } - return v; - }).toList(); - allNodesByAvailabilityZone.computeIfAbsent(orderedNodeAttributeValues, k -> new ArrayList<>()).add(node); - if (node.getRoles().contains(DiscoveryNodeRole.ML_ROLE)) { - mlNodesByAvailabilityZone.compute(orderedNodeAttributeValues, (k, v) -> { - if (v == null) { - v = new ArrayList<>(); - } - v.add(node); - return v; - }); - } - } - return new NodesByAvailabilityZone(Map.copyOf(allNodesByAvailabilityZone), Map.copyOf(mlNodesByAvailabilityZone)); - } - - /** - * This is different to {@link #getMlNodesByAvailabilityZone()} in that the latter returns the ML nodes by availability zone - * of the latest cluster state, while this method does the same for a specific cluster state. - * - * @param clusterState The cluster state whose nodes will be used to detect ML nodes by availability zone. - * @return A map whose keys are lists of awareness attribute values in the same order as the configured awareness attribute - * names, and whose values are collections of nodes that have that combination of attributes. If availability zones - * are not configured then the map will contain one entry mapping an empty list to a collection of all nodes. If - * it is too early in the lifecycle of the node to know the answer then an empty map will be returned. An empty - * map will also be returned if there are no ML nodes in the cluster. (These two empty map return scenarios can be - * distinguished by calling one of the other methods.) - */ - public Map, Collection> buildMlNodesByAvailabilityZone(ClusterState clusterState) { - return buildNodesByAvailabilityZone(clusterState.nodes(), awarenessAttributes).mlNodes; - } + NodesByAvailabilityZone buildNodesByAvailabilityZone(DiscoveryNodes discoveryNodes); - private record NodesByAvailabilityZone( + record NodesByAvailabilityZone( Map, Collection> allNodes, Map, Collection> mlNodes - ) {}; + ) {} } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeFakeAvailabilityZoneMapper.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeFakeAvailabilityZoneMapper.java new file mode 100644 index 0000000000000..618db1da71ab8 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeFakeAvailabilityZoneMapper.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.autoscaling; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Maintains a list of the nodes in each fake availability zone, where each availability zone + * corresponds to a single node. + */ +public class NodeFakeAvailabilityZoneMapper extends AbstractNodeAvailabilityZoneMapper { + + private static final Logger logger = LogManager.getLogger(NodeFakeAvailabilityZoneMapper.class); + + public NodeFakeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings) { + this(settings, clusterSettings, null); + } + + public NodeFakeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings, DiscoveryNodes discoveryNodes) { + super(settings, clusterSettings, discoveryNodes); + updateNodesByAvailabilityZone(); + } + + /** + * @param discoveryNodes The current set of all nodes in the cluster. + * @return A map whose keys are single item lists of node id values, and whose values are single item collections + * of nodes corresponding to the node ids. An empty map will be returned if there are no ML nodes in the + * cluster. + */ + public NodesByAvailabilityZone buildNodesByAvailabilityZone(DiscoveryNodes discoveryNodes) { + Collection nodes = discoveryNodes.getNodes().values(); + + Map, Collection> allNodesByAvailabilityZone = new HashMap<>(); + Map, Collection> mlNodesByAvailabilityZone = new HashMap<>(); + for (DiscoveryNode node : nodes) { + List nodeIdValues = List.of(node.getId()); + List nodeList = List.of(node); + allNodesByAvailabilityZone.put(nodeIdValues, nodeList); + if (node.getRoles().contains(DiscoveryNodeRole.ML_ROLE)) { + mlNodesByAvailabilityZone.put(nodeIdValues, nodeList); + } + } + return new NodesByAvailabilityZone(Map.copyOf(allNodesByAvailabilityZone), Map.copyOf(mlNodesByAvailabilityZone)); + } + + /** + * This is different to {@link #getMlNodesByAvailabilityZone()} in that the latter returns the ML nodes by (fake) availability zone + * of the latest cluster state, while this method does the same for a specific cluster state. + * + * @param clusterState The cluster state whose nodes will be used to detect ML nodes by fake availability zone. + * @return A map whose keys are single item lists of node id values, and whose values are single item collections + * of nodes corresponding to the node ids. An empty map will be returned if there are no ML nodes in the + * cluster. + */ + @Override + public Map, Collection> buildMlNodesByAvailabilityZone(ClusterState clusterState) { + return buildNodesByAvailabilityZone(clusterState.nodes()).mlNodes(); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeRealAvailabilityZoneMapper.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeRealAvailabilityZoneMapper.java new file mode 100644 index 0000000000000..dfac0948087df --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NodeRealAvailabilityZoneMapper.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.autoscaling; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Maintains a list of the nodes in each availability zone, where an availability zone + * means a unique combination of values of the attributes listed in the cluster setting + * cluster.routing.allocation.awareness.attributes. + */ +public class NodeRealAvailabilityZoneMapper extends AbstractNodeAvailabilityZoneMapper { + + private static final Logger logger = LogManager.getLogger(NodeRealAvailabilityZoneMapper.class); + + private volatile List awarenessAttributes; + + public NodeRealAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings) { + this(settings, clusterSettings, null); + } + + public NodeRealAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings, DiscoveryNodes discoveryNodes) { + super(settings, clusterSettings, discoveryNodes); + awarenessAttributes = AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.get(settings); + updateNodesByAvailabilityZone(); + clusterSettings.addSettingsUpdateConsumer( + AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING, + this::setAwarenessAttributes + ); + } + + private synchronized void setAwarenessAttributes(List awarenessAttributes) { + this.awarenessAttributes = List.copyOf(awarenessAttributes); + updateNodesByAvailabilityZone(); + } + + public List getAwarenessAttributes() { + return awarenessAttributes; + } + + public NodesByAvailabilityZone buildNodesByAvailabilityZone(DiscoveryNodes discoveryNodes) { + return buildNodesByAvailabilityZone(discoveryNodes, awarenessAttributes); + } + + private static NodesByAvailabilityZone buildNodesByAvailabilityZone(DiscoveryNodes discoveryNodes, List awarenessAttributes) { + Collection nodes = discoveryNodes.getNodes().values(); + + if (awarenessAttributes.isEmpty()) { + return new NodesByAvailabilityZone( + Map.of(List.of(), nodes), + Map.of(List.of(), nodes.stream().filter(n -> n.getRoles().contains(DiscoveryNodeRole.ML_ROLE)).toList()) + ); + } + + Map, Collection> allNodesByAvailabilityZone = new HashMap<>(); + Map, Collection> mlNodesByAvailabilityZone = new HashMap<>(); + for (DiscoveryNode node : nodes) { + List orderedNodeAttributeValues = awarenessAttributes.stream().map(a -> { + String v = node.getAttributes().get(a); + if (v == null) { + // This will never happen for a Cloud cluster, but for self-managed it's possible. + // We reuse the same allocation attributes that are used for shards, but self-managed + // users might not have bothered to set them on their ML nodes if they are not spread + // across availability zones. + logger.debug( + "Node [{}] does not have all configured awareness attributes {} - missing [{}]", + node, + awarenessAttributes, + a + ); + return ""; + } + return v; + }).toList(); + allNodesByAvailabilityZone.computeIfAbsent(orderedNodeAttributeValues, k -> new ArrayList<>()).add(node); + if (node.getRoles().contains(DiscoveryNodeRole.ML_ROLE)) { + mlNodesByAvailabilityZone.compute(orderedNodeAttributeValues, (k, v) -> { + if (v == null) { + v = new ArrayList<>(); + } + v.add(node); + return v; + }); + } + } + return new NodesByAvailabilityZone(Map.copyOf(allNodesByAvailabilityZone), Map.copyOf(mlNodesByAvailabilityZone)); + } + + /** + * This is different to {@link #getMlNodesByAvailabilityZone()} in that the latter returns the ML nodes by availability zone + * of the latest cluster state, while this method does the same for a specific cluster state. + * + * @param clusterState The cluster state whose nodes will be used to detect ML nodes by availability zone. + * @return A map whose keys are lists of awareness attribute values in the same order as the configured awareness attribute + * names, and whose values are collections of nodes that have that combination of attributes. If availability zones + * are not configured then the map will contain one entry mapping an empty list to a collection of all nodes. If + * it is too early in the lifecycle of the node to know the answer then an empty map will be returned. An empty + * map will also be returned if there are no ML nodes in the cluster. (These two empty map return scenarios can be + * distinguished by calling one of the other methods.) + */ + @Override + public Map, Collection> buildMlNodesByAvailabilityZone(ClusterState clusterState) { + return buildNodesByAvailabilityZone(clusterState.nodes(), awarenessAttributes).mlNodes(); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java index d4b58e0da2fbe..84cef907cd093 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; @@ -31,6 +32,8 @@ import org.elasticsearch.xpack.core.ml.action.MlInfoAction; import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; +import org.elasticsearch.xpack.ml.autoscaling.AbstractNodeAvailabilityZoneMapper; +import org.elasticsearch.xpack.ml.autoscaling.NodeRealAvailabilityZoneMapper; import org.elasticsearch.xpack.ml.rest.RestMlInfoAction; import org.elasticsearch.xpack.ml.rest.dataframe.RestGetDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.rest.inference.RestGetTrainedModelsAction; @@ -332,6 +335,11 @@ public boolean isNlpEnabled() { public String[] getAnalyticsDestIndexAllowedSettings() { return ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS; } + + @Override + public AbstractNodeAvailabilityZoneMapper getNodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings) { + return new NodeRealAvailabilityZoneMapper(settings, clusterSettings); + } } public static class MlTestExtensionLoader implements ExtensiblePlugin.ExtensionLoader { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java index 63db389f25487..632730bc7f141 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java @@ -102,7 +102,7 @@ private static long mlOnlyNodeJvmBytes(long systemMemoryBytes) { private static final long TEST_JOB_SIZE = ByteSizeValue.ofMb(200).getBytes(); private NodeLoadDetector nodeLoadDetector; - private NodeAvailabilityZoneMapper nodeAvailabilityZoneMapper; + private NodeRealAvailabilityZoneMapper nodeRealAvailabilityZoneMapper; private ClusterService clusterService; private Settings settings; private TimeMachine timeSupplier; @@ -122,7 +122,7 @@ public void setup() { when(nodeLoadDetector.detectNodeLoad(any(), any(), anyInt(), anyInt(), anyBoolean())).thenReturn( NodeLoad.builder("any").setUseMemory(true).incAssignedAnomalyDetectorMemory(ByteSizeValue.ofGb(1).getBytes()).build() ); - nodeAvailabilityZoneMapper = mock(NodeAvailabilityZoneMapper.class); + nodeRealAvailabilityZoneMapper = mock(NodeRealAvailabilityZoneMapper.class); clusterService = mock(ClusterService.class); settings = Settings.EMPTY; timeSupplier = new TimeMachine(); @@ -279,7 +279,7 @@ private DiscoveryNode buildNode(String id, ByteSizeValue machineMemory, int allo } private MlAutoscalingDeciderService buildService() { - return new MlAutoscalingDeciderService(nodeLoadDetector, settings, nodeAvailabilityZoneMapper, clusterService, timeSupplier); + return new MlAutoscalingDeciderService(nodeLoadDetector, settings, nodeRealAvailabilityZoneMapper, clusterService, timeSupplier); } static class DeciderContext implements AutoscalingDeciderContext { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java index 00a9f8cb30110..2d9e19cbb3830 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlMemoryAutoscalingDeciderTests.java @@ -132,7 +132,7 @@ public static long mlOnlyNodeJvmBytes(long systemMemoryBytes) { private static final long PER_NODE_OVERHEAD = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(); private NodeLoadDetector nodeLoadDetector; - private NodeAvailabilityZoneMapper nodeAvailabilityZoneMapper; + private NodeRealAvailabilityZoneMapper nodeRealAvailabilityZoneMapper; private ClusterService clusterService; private Settings settings; private LongSupplier timeSupplier; @@ -152,7 +152,7 @@ public void setup() { when(nodeLoadDetector.detectNodeLoad(any(), any(), anyInt(), anyInt(), anyBoolean())).thenReturn( NodeLoad.builder("any").setUseMemory(true).incAssignedAnomalyDetectorMemory(ByteSizeValue.ofGb(1).getBytes()).build() ); - nodeAvailabilityZoneMapper = mock(NodeAvailabilityZoneMapper.class); + nodeRealAvailabilityZoneMapper = mock(NodeRealAvailabilityZoneMapper.class); clusterService = mock(ClusterService.class); settings = Settings.EMPTY; timeSupplier = System::currentTimeMillis; @@ -580,7 +580,7 @@ public void testScaleUp_withWaitingJobsAndRoomInNodes() { List jobTasks = List.of("waiting_job", "waiting_job_2"); List analytics = List.of("analytics_waiting"); // Two small nodes in cluster, so simulate two availability zones - when(nodeAvailabilityZoneMapper.getNumMlAvailabilityZones()).thenReturn(OptionalInt.of(2)); + when(nodeRealAvailabilityZoneMapper.getNumMlAvailabilityZones()).thenReturn(OptionalInt.of(2)); List nodesWithRoom = List.of( NodeLoad.builder("partially_filled") .setMaxMemory(2 * TEST_JOB_SIZE + PER_NODE_OVERHEAD) @@ -909,7 +909,7 @@ public void testScaleUp_withWaitingModelAndAutoMemoryAndNoRoomInNodes() { public void testScaleUp_withWaitingModelsAndRoomInNodes() { // Two small nodes in cluster, so simulate two availability zones - when(nodeAvailabilityZoneMapper.getNumMlAvailabilityZones()).thenReturn(OptionalInt.of(2)); + when(nodeRealAvailabilityZoneMapper.getNumMlAvailabilityZones()).thenReturn(OptionalInt.of(2)); List nodesWithRoom = List.of( NodeLoad.builder("partially_filled") .setMaxMemory(2 * TEST_JOB_SIZE + PER_NODE_OVERHEAD) @@ -945,7 +945,7 @@ public void testScaleUp_withWaitingModelsAndRoomInNodes() { } public void testScaleDown() { - when(nodeAvailabilityZoneMapper.getNumMlAvailabilityZones()).thenReturn(OptionalInt.of(3)); + when(nodeRealAvailabilityZoneMapper.getNumMlAvailabilityZones()).thenReturn(OptionalInt.of(3)); MlMemoryAutoscalingDecider decider = buildDecider(); decider.setMaxMachineMemoryPercent(25); { // Current capacity allows for smaller node @@ -1297,7 +1297,7 @@ private MlMemoryAutoscalingDecider buildDecider() { return new MlMemoryAutoscalingDecider( settings, clusterService, - nodeAvailabilityZoneMapper, + nodeRealAvailabilityZoneMapper, nodeLoadDetector, new ScaleTimer(timeSupplier) ); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/NodeFakeAvailabilityZoneMapperTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/NodeFakeAvailabilityZoneMapperTests.java new file mode 100644 index 0000000000000..3a3a78126e4da --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/NodeFakeAvailabilityZoneMapperTests.java @@ -0,0 +1,284 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.autoscaling; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.ESTestCase; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; +import java.util.Set; + +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_ROLE; +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.MASTER_ROLE; +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.ML_ROLE; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; + +public class NodeFakeAvailabilityZoneMapperTests extends ESTestCase { + + public void testBeforeClusterReady() { + Settings settings = Settings.builder().build(); + ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings(settings); + + NodeFakeAvailabilityZoneMapper nodeFakeAvailabilityZoneMapper = new NodeFakeAvailabilityZoneMapper(settings, clusterSettings); + + assertThat(nodeFakeAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), anEmptyMap()); + assertThat(nodeFakeAvailabilityZoneMapper.getNumAvailabilityZones(), is(OptionalInt.empty())); + } + + public void testAvailabilityZonesAttributesNotConfiguredMultiRoleNodes() { + // Fake availability zones are populated completely independently of cluster settings, + // attributes etc. Here we test that even in the absence of zone attributes that the + // fake availability zone mapper still assigns each node to a single, unique availability + // zone + Settings settings = Settings.EMPTY; + ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings(settings); + + DiscoveryNode node1 = DiscoveryNodeUtils.create( + "node-1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + Map.of(), + Set.of(MASTER_ROLE, DATA_ROLE, ML_ROLE) + ); + + DiscoveryNode node2 = DiscoveryNodeUtils.create( + "node-2", + new TransportAddress(InetAddress.getLoopbackAddress(), 9301), + Map.of(), + Set.of(MASTER_ROLE, DATA_ROLE, ML_ROLE) + ); + DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(node1).add(node2).build(); + + NodeFakeAvailabilityZoneMapper nodeFakeAvailabilityZoneMapper = new NodeFakeAvailabilityZoneMapper( + settings, + clusterSettings, + discoveryNodes + ); + + DiscoveryNodes expectedDiscoveryNodes = DiscoveryNodes.builder().add(node1).add(node2).build(); + assertThat(nodeFakeAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(2)); + for (DiscoveryNode node : expectedDiscoveryNodes) { + assertThat( + nodeFakeAvailabilityZoneMapper.getAllNodesByAvailabilityZone().get(List.of(node.getId())), + equalTo(new ArrayList(List.of(node))) + ); + } + assertThat(nodeFakeAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(2)); + + DiscoveryNodes expectedMlDiscoveryNodes = DiscoveryNodes.builder().add(node1).add(node2).build(); + assertThat(nodeFakeAvailabilityZoneMapper.getMlNodesByAvailabilityZone(), aMapWithSize(2)); + for (DiscoveryNode node : expectedMlDiscoveryNodes) { + assertThat( + nodeFakeAvailabilityZoneMapper.getMlNodesByAvailabilityZone().get(List.of(node.getId())), + equalTo(new ArrayList(List.of(node))) + ); + } + assertThat(nodeFakeAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(2)); + } + + public void testAvailabilityZonesAttributesNotConfiguredDedicatedNodes() { + // Fake availability zones are populated completely independently of cluster settings, + // attributes etc. Here we test that even in the absence of zone attributes that the + // fake availability zone mapper still assigns each node to a single, unique availability + // zone, and that dedicated ML nodes are identified and allocated to ML availability zones. + Settings settings = Settings.EMPTY; + ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings(settings); + + DiscoveryNode mlNode = DiscoveryNodeUtils.create( + "node-1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + Map.of(), + Set.of(ML_ROLE) + ); + DiscoveryNode node1 = DiscoveryNodeUtils.create( + "node-2", + new TransportAddress(InetAddress.getLoopbackAddress(), 9301), + Map.of(), + Set.of(MASTER_ROLE) + ); + DiscoveryNode node2 = DiscoveryNodeUtils.create( + "node-3", + new TransportAddress(InetAddress.getLoopbackAddress(), 9202), + Map.of(), + Set.of(DATA_ROLE) + ); + DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(mlNode).add(node1).add(node2).build(); + NodeFakeAvailabilityZoneMapper nodeFakeAvailabilityZoneMapper = new NodeFakeAvailabilityZoneMapper( + settings, + clusterSettings, + discoveryNodes + ); + + DiscoveryNodes expectedDiscoveryNodes = DiscoveryNodes.builder().add(mlNode).add(node1).add(node2).build(); + assertThat(nodeFakeAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(3)); + for (DiscoveryNode node : expectedDiscoveryNodes) { + assertThat( + nodeFakeAvailabilityZoneMapper.getAllNodesByAvailabilityZone().get(List.of(node.getId())), + equalTo(new ArrayList(List.of(node))) + ); + } + assertThat(nodeFakeAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(3)); + + DiscoveryNodes expectedMlDiscoveryNodes = DiscoveryNodes.builder().add(mlNode).build(); + + assertThat(nodeFakeAvailabilityZoneMapper.getMlNodesByAvailabilityZone(), aMapWithSize(1)); + assertThat(nodeFakeAvailabilityZoneMapper.getMlNodesByAvailabilityZone().get(List.of("node-1")), contains(mlNode)); + assertThat(nodeFakeAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(1)); + } + + public void testAvailabilityZonesAttributesConfiguredMultiRoleNodes() { + // Fake availability zones are populated completely independently of cluster settings, + // attributes etc. Here we test that the fake availability zone mapper assigns each node + // to a single, unique availability zone, even when explicit allocation awareness attributes are + // configured in the cluster settings. + Settings settings = Settings.builder() + .putList( + AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), + List.of("region", "logical_availability_zone") + ) + .build(); + + ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings(settings); + + DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder(); + int numNodes = randomIntBetween(2, 50); + + // For fake availability zones, each node is mapped to a unique zone, i.e. the number of zones is equal to the number of nodes. + // This allocation of zones is performed completely independently of cluster settings, attributes etc. + int numZones = numNodes; + for (int nodeNum = 1; nodeNum <= numNodes; ++nodeNum) { + discoveryNodesBuilder.add( + DiscoveryNodeUtils.create( + "node-" + nodeNum, + new TransportAddress(InetAddress.getLoopbackAddress(), 9299 + nodeNum), + Map.of("region", "unknown-region", "logical_availability_zone", "zone-" + (nodeNum % numZones)), + Set.of(MASTER_ROLE, DATA_ROLE, ML_ROLE) + ) + ); + } + + DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build(); + NodeFakeAvailabilityZoneMapper nodeFakeAvailabilityZoneMapper = new NodeFakeAvailabilityZoneMapper( + settings, + clusterSettings, + discoveryNodes + ); + + assertThat(nodeFakeAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(numNodes)); + int totalNodesMapped = 0; + for (Map.Entry, Collection> entry : nodeFakeAvailabilityZoneMapper.getAllNodesByAvailabilityZone() + .entrySet()) { + List key = entry.getKey(); + assertThat(key, hasSize(1)); + assertThat(entry.getValue().size(), is(1)); + assertThat(key.get(0), equalTo(entry.getValue().iterator().next().getId())); + + ++totalNodesMapped; + } + assertThat(totalNodesMapped, is(numNodes)); + assertThat(nodeFakeAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(numZones)); + totalNodesMapped = 0; + for (Map.Entry, Collection> entry : nodeFakeAvailabilityZoneMapper.getMlNodesByAvailabilityZone() + .entrySet()) { + List key = entry.getKey(); + assertThat(key, hasSize(1)); + assertThat(entry.getValue().size(), is(1)); + assertThat(key.get(0), equalTo(entry.getValue().iterator().next().getId())); + String zoneAttributeValue = key.get(0); + ++totalNodesMapped; + } + assertThat(totalNodesMapped, is(numNodes)); + assertThat(nodeFakeAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(numZones)); + } + + public void testAvailabilityZonesAttributesConfiguredDedicatedNodes() { + // Fake availability zones are populated completely independently of cluster settings, + // attributes etc. Here we test that the fake availability zone mapper assigns each node + // to a single, unique availability zone, even when explicit allocation awareness attributes are + // configured in the cluster settings, and that dedicated ML nodes are identified and allocated + // to ML availability zones. + Settings settings = Settings.builder() + .putList( + AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), + List.of("region", "logical_availability_zone") + ) + .build(); + ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings(settings); + + List mlNodes = new ArrayList<>(); + Set mlZones = new HashSet<>(); + DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder(); + int numNodes = randomIntBetween(10, 50); + int numZones = numNodes; + int numMlZones = randomIntBetween(1, numZones); + for (int nodeNum = 1; nodeNum <= numNodes; ++nodeNum) { + int zone = nodeNum % numZones; + DiscoveryNodeRole role = (zone < numMlZones) ? randomFrom(MASTER_ROLE, DATA_ROLE, ML_ROLE) : randomFrom(MASTER_ROLE, DATA_ROLE); + DiscoveryNode node = DiscoveryNodeUtils.create( + "node-" + nodeNum, + new TransportAddress(InetAddress.getLoopbackAddress(), 9199 + nodeNum), + Map.of("region", "unknown-region", "logical_availability_zone", "zone-" + zone), + Set.of(role) + ); + if (role == ML_ROLE) { + mlNodes.add(node); + mlZones.add(zone); + } + discoveryNodesBuilder.add(node); + } + + DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build(); + NodeFakeAvailabilityZoneMapper nodeFakeAvailabilityZoneMapper = new NodeFakeAvailabilityZoneMapper( + settings, + clusterSettings, + discoveryNodes + ); + + assertThat(nodeFakeAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(numZones)); + int totalNodesMapped = 0; + for (Map.Entry, Collection> entry : nodeFakeAvailabilityZoneMapper.getAllNodesByAvailabilityZone() + .entrySet()) { + List key = entry.getKey(); + + assertThat(key, hasSize(1)); + assertThat(entry.getValue().size(), is(1)); + assertThat(key.get(0), equalTo(entry.getValue().iterator().next().getId())); + ++totalNodesMapped; + } + assertThat(totalNodesMapped, is(numNodes)); + assertThat(nodeFakeAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(numZones)); + int totalMlNodesMapped = 0; + for (Map.Entry, Collection> entry : nodeFakeAvailabilityZoneMapper.getMlNodesByAvailabilityZone() + .entrySet()) { + List key = entry.getKey(); + assertThat(key, hasSize(1)); + assertThat(entry.getValue().size(), is(1)); + assertThat(key.get(0), equalTo(entry.getValue().iterator().next().getId())); + ++totalMlNodesMapped; + } + assertThat(totalMlNodesMapped, is(mlNodes.size())); + assertThat(nodeFakeAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(mlZones.size())); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/NodeAvailabilityZoneMapperTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/NodeRealAvailabilityZoneMapperTests.java similarity index 70% rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/NodeAvailabilityZoneMapperTests.java rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/NodeRealAvailabilityZoneMapperTests.java index e36337e03051c..7ff2e995bbd7c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/NodeAvailabilityZoneMapperTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/NodeRealAvailabilityZoneMapperTests.java @@ -38,7 +38,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; -public class NodeAvailabilityZoneMapperTests extends ESTestCase { +public class NodeRealAvailabilityZoneMapperTests extends ESTestCase { public void testBeforeClusterReady() { Settings settings = Settings.builder() @@ -52,11 +52,11 @@ public void testBeforeClusterReady() { Set.of(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING) ); - NodeAvailabilityZoneMapper nodeAvailabilityZoneMapper = new NodeAvailabilityZoneMapper(settings, clusterSettings); + NodeRealAvailabilityZoneMapper nodeRealAvailabilityZoneMapper = new NodeRealAvailabilityZoneMapper(settings, clusterSettings); - assertThat(nodeAvailabilityZoneMapper.getAwarenessAttributes(), equalTo(List.of("region", "logical_availability_zone"))); - assertThat(nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), anEmptyMap()); - assertThat(nodeAvailabilityZoneMapper.getNumAvailabilityZones(), is(OptionalInt.empty())); + assertThat(nodeRealAvailabilityZoneMapper.getAwarenessAttributes(), equalTo(List.of("region", "logical_availability_zone"))); + assertThat(nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), anEmptyMap()); + assertThat(nodeRealAvailabilityZoneMapper.getNumAvailabilityZones(), is(OptionalInt.empty())); } public void testAvailabilityZonesNotConfiguredMultiRoleNodes() { @@ -84,23 +84,27 @@ public void testAvailabilityZonesNotConfiguredMultiRoleNodes() { ) ) .build(); - NodeAvailabilityZoneMapper nodeAvailabilityZoneMapper = new NodeAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes); + NodeRealAvailabilityZoneMapper nodeRealAvailabilityZoneMapper = new NodeRealAvailabilityZoneMapper( + settings, + clusterSettings, + discoveryNodes + ); - assertThat(nodeAvailabilityZoneMapper.getAwarenessAttributes(), empty()); - assertThat(nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(1)); - assertThat(nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone().keySet().iterator().next(), empty()); + assertThat(nodeRealAvailabilityZoneMapper.getAwarenessAttributes(), empty()); + assertThat(nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(1)); + assertThat(nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone().keySet().iterator().next(), empty()); assertThat( - nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone().get(List.of()), + nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone().get(List.of()), containsInAnyOrder(discoveryNodes.getNodes().values().toArray()) ); - assertThat(nodeAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(1)); - assertThat(nodeAvailabilityZoneMapper.getMlNodesByAvailabilityZone(), aMapWithSize(1)); - assertThat(nodeAvailabilityZoneMapper.getMlNodesByAvailabilityZone().keySet().iterator().next(), empty()); + assertThat(nodeRealAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(1)); + assertThat(nodeRealAvailabilityZoneMapper.getMlNodesByAvailabilityZone(), aMapWithSize(1)); + assertThat(nodeRealAvailabilityZoneMapper.getMlNodesByAvailabilityZone().keySet().iterator().next(), empty()); assertThat( - nodeAvailabilityZoneMapper.getMlNodesByAvailabilityZone().get(List.of()), + nodeRealAvailabilityZoneMapper.getMlNodesByAvailabilityZone().get(List.of()), containsInAnyOrder(discoveryNodes.getNodes().values().toArray()) ); - assertThat(nodeAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(1)); + assertThat(nodeRealAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(1)); } public void testAvailabilityZonesNotConfiguredDedicatedNodes() { @@ -135,20 +139,24 @@ public void testAvailabilityZonesNotConfiguredDedicatedNodes() { ) ) .build(); - NodeAvailabilityZoneMapper nodeAvailabilityZoneMapper = new NodeAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes); + NodeRealAvailabilityZoneMapper nodeRealAvailabilityZoneMapper = new NodeRealAvailabilityZoneMapper( + settings, + clusterSettings, + discoveryNodes + ); - assertThat(nodeAvailabilityZoneMapper.getAwarenessAttributes(), empty()); - assertThat(nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(1)); - assertThat(nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone().keySet().iterator().next(), empty()); + assertThat(nodeRealAvailabilityZoneMapper.getAwarenessAttributes(), empty()); + assertThat(nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(1)); + assertThat(nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone().keySet().iterator().next(), empty()); assertThat( - nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone().get(List.of()), + nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone().get(List.of()), containsInAnyOrder(discoveryNodes.getNodes().values().toArray()) ); - assertThat(nodeAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(1)); - assertThat(nodeAvailabilityZoneMapper.getMlNodesByAvailabilityZone(), aMapWithSize(1)); - assertThat(nodeAvailabilityZoneMapper.getMlNodesByAvailabilityZone().keySet().iterator().next(), empty()); - assertThat(nodeAvailabilityZoneMapper.getMlNodesByAvailabilityZone().get(List.of()), contains(mlNode)); - assertThat(nodeAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(1)); + assertThat(nodeRealAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(1)); + assertThat(nodeRealAvailabilityZoneMapper.getMlNodesByAvailabilityZone(), aMapWithSize(1)); + assertThat(nodeRealAvailabilityZoneMapper.getMlNodesByAvailabilityZone().keySet().iterator().next(), empty()); + assertThat(nodeRealAvailabilityZoneMapper.getMlNodesByAvailabilityZone().get(List.of()), contains(mlNode)); + assertThat(nodeRealAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(1)); } public void testAvailabilityZonesConfiguredMultiRoleNodes() { @@ -178,12 +186,16 @@ public void testAvailabilityZonesConfiguredMultiRoleNodes() { } DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build(); - NodeAvailabilityZoneMapper nodeAvailabilityZoneMapper = new NodeAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes); + NodeRealAvailabilityZoneMapper nodeRealAvailabilityZoneMapper = new NodeRealAvailabilityZoneMapper( + settings, + clusterSettings, + discoveryNodes + ); - assertThat(nodeAvailabilityZoneMapper.getAwarenessAttributes(), equalTo(List.of("region", "logical_availability_zone"))); - assertThat(nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(numZones)); + assertThat(nodeRealAvailabilityZoneMapper.getAwarenessAttributes(), equalTo(List.of("region", "logical_availability_zone"))); + assertThat(nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(numZones)); int totalNodesMapped = 0; - for (Map.Entry, Collection> entry : nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone() + for (Map.Entry, Collection> entry : nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone() .entrySet()) { List key = entry.getKey(); assertThat(key, hasSize(2)); @@ -195,9 +207,9 @@ public void testAvailabilityZonesConfiguredMultiRoleNodes() { } } assertThat(totalNodesMapped, is(numNodes)); - assertThat(nodeAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(numZones)); + assertThat(nodeRealAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(numZones)); totalNodesMapped = 0; - for (Map.Entry, Collection> entry : nodeAvailabilityZoneMapper.getMlNodesByAvailabilityZone() + for (Map.Entry, Collection> entry : nodeRealAvailabilityZoneMapper.getMlNodesByAvailabilityZone() .entrySet()) { List key = entry.getKey(); assertThat(key, hasSize(2)); @@ -209,7 +221,7 @@ public void testAvailabilityZonesConfiguredMultiRoleNodes() { } } assertThat(totalNodesMapped, is(numNodes)); - assertThat(nodeAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(numZones)); + assertThat(nodeRealAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(numZones)); } public void testAvailabilityZonesConfiguredDedicatedNodes() { @@ -247,12 +259,16 @@ public void testAvailabilityZonesConfiguredDedicatedNodes() { } DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build(); - NodeAvailabilityZoneMapper nodeAvailabilityZoneMapper = new NodeAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes); + NodeRealAvailabilityZoneMapper nodeRealAvailabilityZoneMapper = new NodeRealAvailabilityZoneMapper( + settings, + clusterSettings, + discoveryNodes + ); - assertThat(nodeAvailabilityZoneMapper.getAwarenessAttributes(), equalTo(List.of("region", "logical_availability_zone"))); - assertThat(nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(numZones)); + assertThat(nodeRealAvailabilityZoneMapper.getAwarenessAttributes(), equalTo(List.of("region", "logical_availability_zone"))); + assertThat(nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone(), aMapWithSize(numZones)); int totalNodesMapped = 0; - for (Map.Entry, Collection> entry : nodeAvailabilityZoneMapper.getAllNodesByAvailabilityZone() + for (Map.Entry, Collection> entry : nodeRealAvailabilityZoneMapper.getAllNodesByAvailabilityZone() .entrySet()) { List key = entry.getKey(); assertThat(key, hasSize(2)); @@ -264,9 +280,9 @@ public void testAvailabilityZonesConfiguredDedicatedNodes() { } } assertThat(totalNodesMapped, is(numNodes)); - assertThat(nodeAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(numZones)); + assertThat(nodeRealAvailabilityZoneMapper.getNumAvailabilityZones().getAsInt(), is(numZones)); int totalMlNodesMapped = 0; - for (Map.Entry, Collection> entry : nodeAvailabilityZoneMapper.getMlNodesByAvailabilityZone() + for (Map.Entry, Collection> entry : nodeRealAvailabilityZoneMapper.getMlNodesByAvailabilityZone() .entrySet()) { List key = entry.getKey(); assertThat(key, hasSize(2)); @@ -278,6 +294,6 @@ public void testAvailabilityZonesConfiguredDedicatedNodes() { } } assertThat(totalMlNodesMapped, is(mlNodes.size())); - assertThat(nodeAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(mlZones.size())); + assertThat(nodeRealAvailabilityZoneMapper.getNumMlAvailabilityZones().getAsInt(), is(mlZones.size())); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java index b138d46ac903a..0a54b97cf2f2a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java @@ -61,6 +61,8 @@ import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.autoscaling.NodeAvailabilityZoneMapper; +import org.elasticsearch.xpack.ml.autoscaling.NodeFakeAvailabilityZoneMapper; +import org.elasticsearch.xpack.ml.autoscaling.NodeRealAvailabilityZoneMapper; import org.elasticsearch.xpack.ml.job.NodeLoadDetector; import org.elasticsearch.xpack.ml.job.task.OpenJobPersistentTasksExecutorTests; import org.elasticsearch.xpack.ml.notifications.SystemAuditor; @@ -200,7 +202,7 @@ public void testLogMlNodeHeterogeneity_GivenFailure_ThenError() throws Interrupt } public void testClusterChanged_GivenNodesAdded_ThenLogMlNodeHeterogeneityCalled() { - nodeAvailabilityZoneMapper = mock(NodeAvailabilityZoneMapper.class); + nodeAvailabilityZoneMapper = randomFrom(mock(NodeRealAvailabilityZoneMapper.class), mock(NodeFakeAvailabilityZoneMapper.class)); TrainedModelAssignmentClusterService serviceSpy = spy(createClusterService(randomInt(5))); doNothing().when(serviceSpy).logMlNodeHeterogeneity(); doReturn(false).when(serviceSpy).eventStateHasGlobalBlockStateNotRecoveredBlock(any()); @@ -222,7 +224,7 @@ public void testClusterChanged_GivenNodesAdded_ThenLogMlNodeHeterogeneityCalled( } public void testStopPlatformSpecificModelsInHeterogeneousClusters_GivenMultipleMlNodeArchitectures_ThenCallSetToStopping() { - nodeAvailabilityZoneMapper = mock(NodeAvailabilityZoneMapper.class); + nodeAvailabilityZoneMapper = randomFrom(mock(NodeRealAvailabilityZoneMapper.class), mock(NodeFakeAvailabilityZoneMapper.class)); TrainedModelAssignmentClusterService serviceSpy = spy(createClusterService(randomInt(5))); Set architecturesSet = new HashSet<>(randomList(2, 5, () -> randomAlphaOfLength(10))); @@ -436,7 +438,10 @@ public void testCreateAssignment_GivenModelCannotByFullyAllocated_AndScalingIsPo .add(buildNode("ml-node-shutting-down", true, ByteSizeValue.ofGb(4).getBytes(), 2)) .add(buildOldNode("old-ml-node-with-room", true, ByteSizeValue.ofGb(4).getBytes(), 2)) .build(); - nodeAvailabilityZoneMapper = new NodeAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes); + nodeAvailabilityZoneMapper = randomFrom( + new NodeRealAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes), + new NodeFakeAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes) + ); ClusterState currentState = ClusterState.builder(new ClusterName("testCreateAssignment")) .nodes(discoveryNodes) @@ -477,8 +482,10 @@ public void testCreateAssignment_GivenModelCannotByFullyAllocated_AndScalingIsNo .add(buildNode("ml-node-shutting-down", true, ByteSizeValue.ofGb(4).getBytes(), 2)) .add(buildOldNode("old-ml-node-with-room", true, ByteSizeValue.ofGb(4).getBytes(), 2)) .build(); - nodeAvailabilityZoneMapper = new NodeAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes); - + nodeAvailabilityZoneMapper = randomFrom( + new NodeRealAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes), + new NodeFakeAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes) + ); ClusterState currentState = ClusterState.builder(new ClusterName("testCreateAssignment")) .nodes(discoveryNodes) .metadata(Metadata.builder().putCustom(NodesShutdownMetadata.TYPE, shutdownMetadata("ml-node-shutting-down"))) @@ -505,7 +512,10 @@ public void testCreateAssignmentWhileResetModeIsTrue() throws InterruptedExcepti DiscoveryNodes discoveryNodes = DiscoveryNodes.builder() .add(buildNode("ml-node-with-room", true, ByteSizeValue.ofGb(4).getBytes(), 8)) .build(); - nodeAvailabilityZoneMapper = new NodeAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes); + nodeAvailabilityZoneMapper = randomFrom( + new NodeRealAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes), + new NodeFakeAvailabilityZoneMapper(settings, clusterSettings, discoveryNodes) + ); ClusterState currentState = ClusterState.builder(new ClusterName("testCreateAssignment")) .nodes(discoveryNodes) From dffd8421c0d1d784e14e005f442b1f84dc4b5825 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 18 Oct 2023 09:34:54 +0100 Subject: [PATCH 06/23] [ML] Don't log errors for problems caused by shape of input data (#100996) Logging errors in response to user input can cause enormous volumes of output, which then obscures errors caused by bugs in the software. --- .../deployment/InferencePyTorchAction.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java index 474933236d196..c91efb09d3cae 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java @@ -9,10 +9,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceResults; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.threadpool.ThreadPool; @@ -109,6 +111,17 @@ protected void doRun() throws Exception { } catch (IOException e) { logger.error(() -> "[" + getDeploymentId() + "] error writing to inference process", e); onFailure(ExceptionsHelper.serverError("Error writing to inference process", e)); + } catch (ElasticsearchException e) { + // Don't log problems related to the shape of the input as errors + if (e.status().getStatus() >= RestStatus.INTERNAL_SERVER_ERROR.getStatus()) { + logger.error(() -> "[" + getDeploymentId() + "] internal server error running inference", e); + } else { + logger.debug(() -> "[" + getDeploymentId() + "] error running inference due to input", e); + } + onFailure(e); + } catch (IllegalArgumentException e) { + logger.debug(() -> "[" + getDeploymentId() + "] illegal argument running inference", e); + onFailure(e); } catch (Exception e) { logger.error(() -> "[" + getDeploymentId() + "] error running inference", e); onFailure(e); From 9e77bd7b2ee3811d35d30acf3367f888216bc361 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 18 Oct 2023 10:26:34 +0100 Subject: [PATCH 07/23] AwaitsFix for #100502 --- .../test/search.vectors/60_dense_vector_dynamic_mapping.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml index 78ce73a6db4d7..0215d3499cfde 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml @@ -1,7 +1,9 @@ setup: - skip: - version: ' - 8.10.99' - reason: 'Dynamic mapping of floats to dense_vector was added in 8.11' + version: 'all' + reason: 'AwaitsFix https://github.com/elastic/elasticsearch/issues/100502' + #version: ' - 8.10.99' + #reason: 'Dynamic mapping of floats to dense_vector was added in 8.11' # Additional logging for issue: https://github.com/elastic/elasticsearch/issues/100502 - do: From 9b404099b4d03eea0c52b0f7cdcbf6f6080cec3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Wed, 18 Oct 2023 11:30:38 +0200 Subject: [PATCH 08/23] [DOCS] Adds links to token section in ESLER conceptual. (#101033) --- .../search-your-data/semantic-search-elser.asciidoc | 13 +++++++++---- .../semantic-search/generate-embeddings.asciidoc | 4 +++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc index 2449f26d8dac3..164beb221cd4f 100644 --- a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc @@ -208,7 +208,10 @@ GET my-index/_search The result is the top 10 documents that are closest in meaning to your query text from the `my-index` index sorted by their relevancy. The result also contains the extracted tokens for each of the relevant search results with their -weights. +weights. Tokens are learned associations capturing relevance, they are not +synonyms. To learn more about what tokens are, refer to +{ml-docs}/ml-nlp-elser.html#elser-tokens[this page]. It is possible to exclude +tokens from source, refer to <> to learn more. [source,consol-result] ---- @@ -325,12 +328,14 @@ by using the <> mapping to remove the ELSER terms from the document source. WARNING: Reindex uses the document source to populate the destination index. -Once the ELSER terms have been excluded from the source, they cannot be -recovered through reindexing. Excluding the tokens from the source is a +**Once the ELSER terms have been excluded from the source, they cannot be** +**recovered through reindexing.** Excluding the tokens from the source is a space-saving optimsation that should only be applied if you are certain that reindexing will not be required in the future! It's important to carefully consider this trade-off and make sure that excluding the ELSER terms from the -source aligns with your specific requirements and use case. +source aligns with your specific requirements and use case. Review the +<> and <> sections carefully to learn +more about the possible consequences of excluding the tokens from the `_source`. The mapping that excludes `content_embedding` from the `_source` field can be created by the following API call: diff --git a/docs/reference/tab-widgets/semantic-search/generate-embeddings.asciidoc b/docs/reference/tab-widgets/semantic-search/generate-embeddings.asciidoc index caf6523783b02..2294d3c5598c5 100644 --- a/docs/reference/tab-widgets/semantic-search/generate-embeddings.asciidoc +++ b/docs/reference/tab-widgets/semantic-search/generate-embeddings.asciidoc @@ -39,7 +39,9 @@ and the `output_field` that will contain the {infer} results. To ingest data through the pipeline to generate tokens with ELSER, refer to the <> section of the tutorial. After you successfully ingested documents by using the pipeline, your index will contain the tokens -generated by ELSER. +generated by ELSER. Tokens are learned associations capturing relevance, they +are not synonyms. To learn more about what tokens are, refer to +{ml-docs}/ml-nlp-elser.html#elser-tokens[this page]. // end::elser[] From 22a025e96aed4477cdd04d589ffff146cdedf441 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Wed, 18 Oct 2023 11:33:06 +0200 Subject: [PATCH 09/23] ESQL: Support date and time intervals as input params (#101001) Fixes https://github.com/elastic/elasticsearch/issues/99570 Add support for DATE_PERIOD and TYPE_DURATION values as input parameters, eg. ``` { "query": "row a = 1 | eval x = now() + ?", "params": [{"type":"time_duration", "value":"5 hours"}] } ``` The values have to be passed as strings and then will be converted to the appropriate type. The original issue also pointed to similar problems for Version type, so the PR also includes a test for this case. --- docs/changelog/101001.yaml | 6 + .../xpack/esql/parser/ExpressionBuilder.java | 27 +-- .../esql/type/EsqlDataTypeConverter.java | 184 ++++++++++++++++++ .../xpack/esql/type/EsqlDataTypeRegistry.java | 7 +- .../esql/parser/StatementParserTests.java | 69 ++++++- 5 files changed, 267 insertions(+), 26 deletions(-) create mode 100644 docs/changelog/101001.yaml create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java diff --git a/docs/changelog/101001.yaml b/docs/changelog/101001.yaml new file mode 100644 index 0000000000000..3ebcefc2c8045 --- /dev/null +++ b/docs/changelog/101001.yaml @@ -0,0 +1,6 @@ +pr: 101001 +summary: "ESQL: Support date and time intervals as input params" +area: ES|QL +type: bug +issues: + - 99570 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index a7c8d6dd49cc7..5d3108a785f59 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Alias; @@ -48,15 +49,14 @@ import org.elasticsearch.xpack.ql.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.ql.type.DataTypeConverter; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.type.DateUtils; import org.elasticsearch.xpack.ql.util.StringUtils; import java.math.BigInteger; import java.time.Duration; -import java.time.Period; import java.time.ZoneId; +import java.time.temporal.TemporalAmount; import java.util.List; import java.util.Locale; import java.util.Map; @@ -64,18 +64,17 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.parseTemporalAmout; import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.DATE_PERIOD; import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.TIME_DURATION; import static org.elasticsearch.xpack.ql.parser.ParserUtils.source; import static org.elasticsearch.xpack.ql.parser.ParserUtils.typedParsing; import static org.elasticsearch.xpack.ql.parser.ParserUtils.visitList; -import static org.elasticsearch.xpack.ql.type.DataTypeConverter.safeToInt; -import static org.elasticsearch.xpack.ql.type.DataTypeConverter.safeToLong; import static org.elasticsearch.xpack.ql.util.NumericUtils.asLongUnsigned; import static org.elasticsearch.xpack.ql.util.NumericUtils.unsignedLongAsNumber; import static org.elasticsearch.xpack.ql.util.StringUtils.WILDCARD; -abstract class ExpressionBuilder extends IdentifierBuilder { +public abstract class ExpressionBuilder extends IdentifierBuilder { private final Map params; @@ -224,19 +223,7 @@ public Object visitQualifiedIntegerLiteral(EsqlBaseParser.QualifiedIntegerLitera String qualifier = ctx.UNQUOTED_IDENTIFIER().getText().toLowerCase(Locale.ROOT); try { - Object quantity = switch (qualifier) { - case "millisecond", "milliseconds" -> Duration.ofMillis(safeToLong(value)); - case "second", "seconds" -> Duration.ofSeconds(safeToLong(value)); - case "minute", "minutes" -> Duration.ofMinutes(safeToLong(value)); - case "hour", "hours" -> Duration.ofHours(safeToLong(value)); - - case "day", "days" -> Period.ofDays(safeToInt(safeToLong(value))); - case "week", "weeks" -> Period.ofWeeks(safeToInt(safeToLong(value))); - case "month", "months" -> Period.ofMonths(safeToInt(safeToLong(value))); - case "year", "years" -> Period.ofYears(safeToInt(safeToLong(value))); - - default -> throw new ParsingException(source, "Unexpected time interval qualifier: '{}'", qualifier); - }; + TemporalAmount quantity = parseTemporalAmout(value, qualifier, source); return new Literal(source, quantity, quantity instanceof Duration ? TIME_DURATION : DATE_PERIOD); } catch (QlIllegalArgumentException | ArithmeticException e) { // the range varies by unit: Duration#ofMinutes(), #ofHours() will Math#multiplyExact() to reduce the unit to seconds; @@ -440,7 +427,7 @@ public Object visitInputParam(EsqlBaseParser.InputParamContext ctx) { // otherwise we need to make sure that xcontent-serialized value is converted to the correct type try { - if (DataTypeConverter.canConvert(sourceType, dataType) == false) { + if (EsqlDataTypeConverter.canConvert(sourceType, dataType) == false) { throw new ParsingException( source, "Cannot cast value [{}] of type [{}] to parameter type [{}]", @@ -449,7 +436,7 @@ public Object visitInputParam(EsqlBaseParser.InputParamContext ctx) { dataType ); } - return new Literal(source, DataTypeConverter.converterFor(sourceType, dataType).convert(param.value), dataType); + return new Literal(source, EsqlDataTypeConverter.converterFor(sourceType, dataType).convert(param.value), dataType); } catch (QlIllegalArgumentException ex) { throw new ParsingException(ex, source, "Unexpected actual parameter type [{}] for type [{}]", sourceType, param.type); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java new file mode 100644 index 0000000000000..192d5e43f9366 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.type; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.parser.ParsingException; +import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.Converter; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypeConverter; + +import java.io.IOException; +import java.time.Duration; +import java.time.Period; +import java.time.temporal.TemporalAmount; +import java.util.function.Function; + +import static org.elasticsearch.xpack.ql.type.DataTypeConverter.safeToInt; +import static org.elasticsearch.xpack.ql.type.DataTypeConverter.safeToLong; +import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; +import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; +import static org.elasticsearch.xpack.ql.type.DataTypes.isString; + +public class EsqlDataTypeConverter { + + /** + * Returns true if the from type can be converted to the to type, false - otherwise + */ + public static boolean canConvert(DataType from, DataType to) { + // Special handling for nulls and if conversion is not requires + if (from == to || from == NULL) { + return true; + } + // only primitives are supported so far + return isPrimitive(from) && isPrimitive(to) && converterFor(from, to) != null; + } + + public static Converter converterFor(DataType from, DataType to) { + Converter converter = DataTypeConverter.converterFor(from, to); + if (converter != null) { + return converter; + } + if (isString(from) && to == EsqlDataTypes.TIME_DURATION) { + return EsqlConverter.STRING_TO_TIME_DURATION; + } + if (isString(from) && to == EsqlDataTypes.DATE_PERIOD) { + return EsqlConverter.STRING_TO_DATE_PERIOD; + } + return null; + } + + public static TemporalAmount parseTemporalAmount(Object val, DataType expectedType) { + String errorMessage = "Cannot parse [{}] to {}"; + String str = String.valueOf(val); + if (str == null) { + return null; + } + StringBuilder value = new StringBuilder(); + StringBuilder qualifier = new StringBuilder(); + StringBuilder nextBuffer = value; + boolean lastWasSpace = false; + for (char c : str.trim().toCharArray()) { + if (c == ' ') { + if (lastWasSpace == false) { + nextBuffer = nextBuffer == value ? qualifier : null; + } + lastWasSpace = true; + continue; + } + if (nextBuffer == null) { + throw new ParsingException(Source.EMPTY, errorMessage, val, expectedType); + } + nextBuffer.append(c); + lastWasSpace = false; + } + + if ((value.isEmpty() || qualifier.isEmpty()) == false) { + try { + TemporalAmount result = parseTemporalAmout(Integer.parseInt(value.toString()), qualifier.toString(), Source.EMPTY); + if (EsqlDataTypes.DATE_PERIOD == expectedType && result instanceof Period + || EsqlDataTypes.TIME_DURATION == expectedType && result instanceof Duration) { + return result; + } + if (result instanceof Period && expectedType == EsqlDataTypes.TIME_DURATION) { + errorMessage += ", did you mean " + EsqlDataTypes.DATE_PERIOD + "?"; + } + if (result instanceof Duration && expectedType == EsqlDataTypes.DATE_PERIOD) { + errorMessage += ", did you mean " + EsqlDataTypes.TIME_DURATION + "?"; + } + } catch (NumberFormatException ex) { + // wrong pattern + } + } + + throw new ParsingException(Source.EMPTY, errorMessage, val, expectedType); + } + + /** + * Converts arbitrary object to the desired data type. + *

+ * Throws QlIllegalArgumentException if such conversion is not possible + */ + public static Object convert(Object value, DataType dataType) { + DataType detectedType = EsqlDataTypes.fromJava(value); + if (detectedType == dataType || value == null) { + return value; + } + Converter converter = converterFor(detectedType, dataType); + + if (converter == null) { + throw new QlIllegalArgumentException( + "cannot convert from [{}], type [{}] to [{}]", + value, + detectedType.typeName(), + dataType.typeName() + ); + } + + return converter.convert(value); + } + + public static DataType commonType(DataType left, DataType right) { + return DataTypeConverter.commonType(left, right); + } + + public static TemporalAmount parseTemporalAmout(Number value, String qualifier, Source source) throws QlIllegalArgumentException, + ArithmeticException { + return switch (qualifier) { + case "millisecond", "milliseconds" -> Duration.ofMillis(safeToLong(value)); + case "second", "seconds" -> Duration.ofSeconds(safeToLong(value)); + case "minute", "minutes" -> Duration.ofMinutes(safeToLong(value)); + case "hour", "hours" -> Duration.ofHours(safeToLong(value)); + + case "day", "days" -> Period.ofDays(safeToInt(safeToLong(value))); + case "week", "weeks" -> Period.ofWeeks(safeToInt(safeToLong(value))); + case "month", "months" -> Period.ofMonths(safeToInt(safeToLong(value))); + case "year", "years" -> Period.ofYears(safeToInt(safeToLong(value))); + + default -> throw new ParsingException(source, "Unexpected time interval qualifier: '{}'", qualifier); + }; + } + + public enum EsqlConverter implements Converter { + + STRING_TO_DATE_PERIOD(x -> EsqlDataTypeConverter.parseTemporalAmount(x, EsqlDataTypes.DATE_PERIOD)), + STRING_TO_TIME_DURATION(x -> EsqlDataTypeConverter.parseTemporalAmount(x, EsqlDataTypes.TIME_DURATION)); + + private static final String NAME = "esql-converter"; + private final Function converter; + + EsqlConverter(Function converter) { + this.converter = converter; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(this); + } + + public static Converter read(StreamInput in) throws IOException { + return in.readEnum(EsqlConverter.class); + } + + @Override + public Object convert(Object l) { + if (l == null) { + return null; + } + return converter.apply(l); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java index 83dd0ff4ed1c6..3b5aa5dbecc3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java @@ -9,7 +9,6 @@ import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.ql.type.DataTypeConverter; import org.elasticsearch.xpack.ql.type.DataTypeRegistry; import org.elasticsearch.xpack.ql.type.DataTypes; @@ -52,12 +51,12 @@ public boolean isUnsupported(DataType type) { @Override public boolean canConvert(DataType from, DataType to) { - return DataTypeConverter.canConvert(from, to); + return EsqlDataTypeConverter.canConvert(from, to); } @Override public Object convert(Object value, DataType type) { - return DataTypeConverter.convert(value, type); + return EsqlDataTypeConverter.convert(value, type); } @Override @@ -71,6 +70,6 @@ public DataType commonType(DataType left, DataType right) { if (left == DATE_PERIOD && right == DATE_PERIOD) { return DATE_PERIOD; } - return DataTypeConverter.commonType(left, right); + return EsqlDataTypeConverter.commonType(left, right); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index cb1c48784739f..5047688610120 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -43,8 +43,11 @@ import org.elasticsearch.xpack.ql.plan.logical.OrderBy; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.versionfield.Version; import java.math.BigInteger; +import java.time.Duration; +import java.time.Period; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -709,10 +712,19 @@ public void testUsageOfProject() { } public void testInputParams() { - LogicalPlan stm = statement("row x = ?, y = ?", List.of(new TypedParamValue("integer", 1), new TypedParamValue("keyword", "2"))); + LogicalPlan stm = statement( + "row x = ?, y = ?, a = ?, b = ?, c = ?", + List.of( + new TypedParamValue("integer", 1), + new TypedParamValue("keyword", "2"), + new TypedParamValue("date_period", "2 days"), + new TypedParamValue("time_duration", "4 hours"), + new TypedParamValue("version", "1.2.3") + ) + ); assertThat(stm, instanceOf(Row.class)); Row row = (Row) stm; - assertThat(row.fields().size(), is(2)); + assertThat(row.fields().size(), is(5)); NamedExpression field = row.fields().get(0); assertThat(field.name(), is("x")); @@ -725,6 +737,59 @@ public void testInputParams() { assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; assertThat(alias.child().fold(), is("2")); + + field = row.fields().get(2); + assertThat(field.name(), is("a")); + assertThat(field, instanceOf(Alias.class)); + alias = (Alias) field; + assertThat(alias.child().fold(), is(Period.ofDays(2))); + + field = row.fields().get(3); + assertThat(field.name(), is("b")); + assertThat(field, instanceOf(Alias.class)); + alias = (Alias) field; + assertThat(alias.child().fold(), is(Duration.ofHours(4))); + + field = row.fields().get(4); + assertThat(field.name(), is("c")); + assertThat(field, instanceOf(Alias.class)); + alias = (Alias) field; + assertThat(alias.child().fold().getClass(), is(Version.class)); + assertThat(alias.child().fold().toString(), is("1.2.3")); + } + + public void testWrongIntervalParams() { + expectError("row x = ?", List.of(new TypedParamValue("date_period", "12")), "Cannot parse [12] to DATE_PERIOD"); + expectError("row x = ?", List.of(new TypedParamValue("time_duration", "12")), "Cannot parse [12] to TIME_DURATION"); + expectError( + "row x = ?", + List.of(new TypedParamValue("date_period", "12 months foo")), + "Cannot parse [12 months foo] to DATE_PERIOD" + ); + expectError( + "row x = ?", + List.of(new TypedParamValue("time_duration", "12 minutes bar")), + "Cannot parse [12 minutes bar] to TIME_DURATION" + ); + expectError("row x = ?", List.of(new TypedParamValue("date_period", "12 foo")), "Unexpected time interval qualifier: 'foo'"); + expectError("row x = ?", List.of(new TypedParamValue("time_duration", "12 bar")), "Unexpected time interval qualifier: 'bar'"); + expectError("row x = ?", List.of(new TypedParamValue("date_period", "foo days")), "Cannot parse [foo days] to DATE_PERIOD"); + expectError( + "row x = ?", + List.of(new TypedParamValue("time_duration", "bar seconds")), + "Cannot parse [bar seconds] to TIME_DURATION" + ); + + expectError( + "row x = ?", + List.of(new TypedParamValue("date_period", "2 minutes")), + "Cannot parse [2 minutes] to DATE_PERIOD, did you mean TIME_DURATION?" + ); + expectError( + "row x = ?", + List.of(new TypedParamValue("time_duration", "11 months")), + "Cannot parse [11 months] to TIME_DURATION, did you mean DATE_PERIOD?" + ); } public void testMissingInputParams() { From b5dd2970d4b99a5f1c284a48882ae0e3ed8662ec Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 18 Oct 2023 11:36:09 +0200 Subject: [PATCH 10/23] Mute VectorTileRestIT#testCentroidGridTypeOnPolygon (#101041) relates https://github.com/elastic/elasticsearch/issues/101038 --- .../org/elasticsearch/xpack/vectortile/VectorTileRestIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java index 016bfaabec0ff..1cb77ec45199f 100644 --- a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java +++ b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java @@ -579,6 +579,7 @@ public void testInvalidAggName() { assertThat(ex.getMessage(), Matchers.containsString("Invalid aggregation name [_mvt_name]")); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101038") public void testCentroidGridTypeOnPolygon() throws Exception { final Request mvtRequest = new Request(getHttpMethod(), INDEX_POLYGON + "/_mvt/location/" + (z + 2) + "/" + 4 * x + "/" + 4 * y); mvtRequest.setJsonEntity("{\"size\" : 0, \"grid_type\": \"centroid\", \"grid_precision\": 2}"); From deed5619d05f2870348552325097c3023137c8aa Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:37:14 +0300 Subject: [PATCH 11/23] Throw when wrapping rate agg in DeferableBucketAggregator (#101032) * Throw when wrapping rate agg in DeferableBucketAggregator * Update docs/changelog/101032.yaml --- docs/changelog/101032.yaml | 5 ++ .../rate/TimeSeriesRateAggregator.java | 13 +++++ .../rate/TimeSeriesRateAggregatorTests.java | 40 +++++++++++++ .../rest-api-spec/test/analytics/100_tsdb.yml | 57 +++++++++++++++++++ 4 files changed, 115 insertions(+) create mode 100644 docs/changelog/101032.yaml diff --git a/docs/changelog/101032.yaml b/docs/changelog/101032.yaml new file mode 100644 index 0000000000000..1c69e372704ce --- /dev/null +++ b/docs/changelog/101032.yaml @@ -0,0 +1,5 @@ +pr: 101032 +summary: Throw when wrapping rate agg in `DeferableBucketAggregator` +area: TSDB +type: bug +issues: [] diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java index da8dfb3d480e0..2c2f213cf430b 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.bucket.DeferableBucketAggregator; import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSource; @@ -66,6 +67,18 @@ protected TimeSeriesRateAggregator( this.endTimes = bigArrays().newLongArray(1, true); this.resetCompensations = bigArrays().newDoubleArray(1, true); this.rateUnit = rateUnit; + + Aggregator parentIterator = parent; + while (parentIterator != null) { + if (parentIterator instanceof DeferableBucketAggregator) { + throw new IllegalArgumentException( + "Wrapping a time-series rate aggregation within a DeferableBucketAggregator is not " + + "supported. Consider using an alternative outer aggregation type to avoid this, e.g. date_histogram instead of " + + "auto_date_histogram." + ); + } + parentIterator = parentIterator.parent(); + } } @Override diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 32a337f0b329b..b226f0f23b7ff 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.elasticsearch.aggregations.AggregationsPlugin; +import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; import org.elasticsearch.common.bytes.BytesReference; @@ -110,6 +111,45 @@ public void testNestedWithinDateHistogram() throws IOException { }, verifier, aggTestConfig); } + public void testNestedWithinAutoDateHistogram() throws IOException { + RateAggregationBuilder builder = new RateAggregationBuilder("counter_field").field("counter_field"); + AutoDateHistogramAggregationBuilder dateBuilder = new AutoDateHistogramAggregationBuilder("date"); + dateBuilder.field("@timestamp"); + dateBuilder.subAggregation(builder); + TimeSeriesAggregationBuilder tsBuilder = new TimeSeriesAggregationBuilder("tsid"); + tsBuilder.subAggregation(dateBuilder); + + Consumer verifier = r -> { + assertThat(r.getBuckets(), hasSize(2)); + assertThat(r.getBucketByKey("{dim=1}"), instanceOf(InternalTimeSeries.InternalBucket.class)); + InternalDateHistogram hb = r.getBucketByKey("{dim=1}").getAggregations().get("date"); + { + Rate rate = hb.getBuckets().get(1).getAggregations().get("counter_field"); + assertThat(rate.getValue(), closeTo((60 - 37 + 14) / 2000.0 * MILLIS_IN_SECOND, 0.00001)); + } + { + Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); + assertThat(rate.getValue(), closeTo((37 - 15) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); + } + hb = r.getBucketByKey("{dim=2}").getAggregations().get("date"); + { + Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); + assertThat(rate.getValue(), closeTo((150 - 74) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); + } + { + Rate rate = hb.getBuckets().get(1).getAggregations().get("counter_field"); + assertThat(rate.getValue(), closeTo(90 / 2000.0 * MILLIS_IN_SECOND, 0.00001)); + } + }; + + AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field")) + .withSplitLeavesIntoSeperateAggregators(false); + expectThrows(IllegalArgumentException.class, () -> testCase(iw -> { + iw.addDocuments(docs(2000, "1", 15, 37, 60, /*reset*/ 14)); + iw.addDocuments(docs(2000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40)); + }, verifier, aggTestConfig)); + } + private List docs(long startTimestamp, String dim, long... values) throws IOException { List documents = new ArrayList<>(); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml index 88493a26a01d3..ef34e64ad41d7 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml @@ -84,3 +84,60 @@ aggretate multi_terms: - match: { aggregations.m_terms.buckets.1.doc_count: 3 } - match: { aggregations.m_terms.buckets.2.key_as_string: "{k8s.pod.uid=947e4ced-1786-4e53-9e0c-5c447e959507, metricset=pod}|10.10.55.2" } - match: { aggregations.m_terms.buckets.2.doc_count: 1 } + +--- +"Auto date histogram on counter": + - do: + indices.create: + index: test2 + body: + settings: + mode: time_series + routing_path: [dim] + time_series: + start_time: "2023-01-01T00:00:00.000Z" + mappings: + properties: + dim: + type: keyword + time_series_dimension: true + data: + type: integer + time_series_metric: counter + "@timestamp": + type: date + + - do: + bulk: + index: test2 + refresh: true + body: + - '{ "index": {} }' + - '{ "@timestamp":"2023-01-01T13:03:08.138Z","data":"10", "dim": "A"}' + - '{ "index": {}}' + - '{ "@timestamp":"2023-01-02T13:03:09.138Z","data":"20", "dim": "A"}' + - '{ "index": {}}' + - '{ "@timestamp":"2023-02-01T13:03:10.138Z","data":"30", "dim": "B"}' + + - do: + catch: /Wrapping a time-series rate aggregation within a DeferableBucketAggregator is not supported/ + search: + size: 0 + index: test2 + body: + aggs: + histo: + auto_date_histogram: + field: "@timestamp" + "buckets": 4 + aggs: + ts_data_rate: + time_series: + keyed: false + aggs: + data_rate: + rate: + field: data + unit: day + + From cad7232c39865c310e4e953614c79f50caa029dd Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 18 Oct 2023 20:54:35 +1100 Subject: [PATCH 12/23] [Test] Ensure indices are healthy enough before closing (#101023) Closing unassigned indices is not cleanly handled. This change avoids doing so in the test. Resolves: #99516 --- .../snapshots/SnapshotStressTestsIT.java | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java index a23be834fcc53..cdf76bea1cf04 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ListenableActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -501,21 +502,35 @@ private void restoreSnapshot(SnapshotInfo snapshotInfo, Releasable releasePrevio if (indicesToClose.length > 0) { logger.info( - "--> closing indices {} in preparation for restoring from [{}:{}]", - indicesToRestoreList, - snapshotInfo.repository(), - snapshotInfo.snapshotId().getName() + "--> waiting for yellow health of [{}] before closing", + Strings.arrayToCommaDelimitedString(indicesToClose) ); - indicesAdmin().prepareClose(indicesToClose).execute(mustSucceed(closeIndexResponse -> { + + SubscribableListener.newForked( + l -> prepareClusterHealthRequest(indicesToClose).setWaitForYellowStatus().execute(l) + ).addListener(mustSucceed(clusterHealthResponse -> { + assertFalse( + "timed out waiting for yellow state of " + Strings.arrayToCommaDelimitedString(indicesToClose), + clusterHealthResponse.isTimedOut() + ); + logger.info( - "--> finished closing indices {} in preparation for restoring from [{}:{}]", + "--> closing indices {} in preparation for restoring from [{}:{}]", indicesToRestoreList, snapshotInfo.repository(), snapshotInfo.snapshotId().getName() ); - assertTrue(closeIndexResponse.isAcknowledged()); - assertTrue(closeIndexResponse.isShardsAcknowledged()); - closeIndicesStep.onResponse(null); + indicesAdmin().prepareClose(indicesToClose).execute(mustSucceed(closeIndexResponse -> { + logger.info( + "--> finished closing indices {} in preparation for restoring from [{}:{}]", + indicesToRestoreList, + snapshotInfo.repository(), + snapshotInfo.snapshotId().getName() + ); + assertTrue(closeIndexResponse.isAcknowledged()); + assertTrue(closeIndexResponse.isShardsAcknowledged()); + closeIndicesStep.onResponse(null); + })); })); } else { closeIndicesStep.onResponse(null); From 588fed797b09c2cbad251a35b8376c59511636a9 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 18 Oct 2023 12:14:12 +0200 Subject: [PATCH 13/23] Remove redundant ActionType classes (#101037) Got asked why we have these classes again last night ... figured I'd remove some of them to get us going here. None of these classes are necessary, we can just inline all of these away and make `ActionType` itself final or a record now that the action type consists only in name and reader. See #97721 that made these things redundant I think. --- .../tasks/RestListTasksCancellationIT.java | 6 ++-- .../http/ClusterInfoRestCancellationIT.java | 8 ++--- .../http/NodeStatsRestCancellationIT.java | 4 +-- .../admin/cluster/node/tasks/TasksIT.java | 6 ++-- .../admin/cluster/remote/RemoteInfoIT.java | 8 +++-- .../cluster/ClusterInfoServiceIT.java | 6 ++-- .../elasticsearch/action/ActionModule.java | 20 ++++------- .../org/elasticsearch/action/ActionType.java | 4 +++ .../hotthreads/NodesHotThreadsAction.java | 22 ------------ .../NodesHotThreadsRequestBuilder.java | 3 +- .../TransportNodesHotThreadsAction.java | 5 ++- .../cluster/node/info/NodesInfoAction.java | 22 ------------ .../node/info/NodesInfoRequestBuilder.java | 3 +- .../node/info/TransportNodesInfoAction.java | 4 ++- .../cluster/node/stats/NodesStatsAction.java | 22 ------------ .../node/stats/NodesStatsRequestBuilder.java | 3 +- .../node/stats/TransportNodesStatsAction.java | 4 ++- .../node/tasks/list/ListTasksAction.java | 25 -------------- .../tasks/list/ListTasksRequestBuilder.java | 3 +- .../tasks/list/TransportListTasksAction.java | 7 ++-- .../cluster/node/usage/NodesUsageAction.java | 22 ------------ .../node/usage/TransportNodesUsageAction.java | 4 ++- .../remote/RemoteClusterNodesAction.java | 16 ++++----- .../cluster/remote/RemoteInfoAction.java | 21 ------------ .../remote/TransportRemoteInfoAction.java | 4 ++- .../internal/support/AbstractClient.java | 34 +++++++++---------- .../cluster/coordination/LagDetector.java | 4 +-- .../cluster/RestRemoteClusterInfoAction.java | 4 +-- .../transport/RemoteClusterConnection.java | 2 +- .../transport/SniffConnectionStrategy.java | 2 +- .../action/ActionModuleTests.java | 10 +++--- .../node/tasks/TransportTasksActionTests.java | 4 +-- .../remote/RemoteClusterNodesActionTests.java | 6 ++-- .../RemoteClusterConnectionTests.java | 7 ++-- .../SniffConnectionStrategyTests.java | 4 +-- .../AbstractCoordinatorTestCase.java | 3 +- .../test/AbstractMultiClustersTestCase.java | 4 +-- .../test/rest/ESRestTestCase.java | 8 ++--- .../AutoscalingNodesInfoServiceTests.java | 11 +++--- .../xpack/ccr/RestartIndexFollowingIT.java | 6 ++-- .../xpack/CcrSingleNodeTestCase.java | 6 ++-- .../DataTiersUsageRestCancellationIT.java | 8 ++--- .../ml/utils/MlPlatformArchitecturesUtil.java | 4 +-- .../privilege/ClusterPrivilegeResolver.java | 4 +-- .../authz/store/ReservedRolesStore.java | 4 +-- .../authz/store/ReservedRolesStoreTests.java | 4 +-- .../xpack/ml/MlDailyMaintenanceService.java | 4 +-- .../TransportGetTrainedModelsStatsAction.java | 4 +-- .../ml/MlDailyMaintenanceServiceTests.java | 10 +++--- ...ransportDeleteTrainedModelActionTests.java | 4 +-- .../TransportPutTrainedModelActionTests.java | 4 +-- .../xpack/ml/support/BaseMlIntegTestCase.java | 6 ++-- .../xpack/ml/utils/TaskRetrieverTests.java | 8 ++--- ...lusterSecurityFcActionAuthorizationIT.java | 2 +- ...CrossClusterAccessHeadersForCcsRestIT.java | 4 +-- .../authc/apikey/ApiKeySingleNodeTests.java | 4 +-- .../TransportNodeEnrollmentAction.java | 4 +-- .../InternalEnrollmentTokenGenerator.java | 6 ++-- .../TransportNodeEnrollmentActionTests.java | 4 +-- .../authz/store/CompositeRolesStoreTests.java | 16 +++++++-- ...InternalEnrollmentTokenGeneratorTests.java | 10 +++--- ...ty4ServerTransportAuthenticationTests.java | 2 +- 62 files changed, 191 insertions(+), 292 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsAction.java delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksAction.java delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageAction.java delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java index 29e59af7b9f70..d4d3f6143ff3c 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java @@ -9,7 +9,7 @@ package org.elasticsearch.action.support.tasks; import org.apache.http.client.methods.HttpGet; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; +import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.Cancellable; @@ -49,7 +49,7 @@ public void testListTasksCancellation() throws Exception { final PlainActionFuture tasksFuture = new PlainActionFuture<>(); final Cancellable tasksCancellable = getRestClient().performRequestAsync(tasksRequest, wrapAsRestResponseListener(tasksFuture)); - awaitTaskWithPrefix(ListTasksAction.NAME + "[n]"); + awaitTaskWithPrefix(TransportListTasksAction.TYPE.name() + "[n]"); tasksCancellable.cancel(); @@ -61,7 +61,7 @@ public void testListTasksCancellation() throws Exception { () -> assertFalse( taskManagers.stream() .flatMap(taskManager -> taskManager.getCancellableTasks().values().stream()) - .anyMatch(t -> t.getAction().startsWith(ListTasksAction.NAME)) + .anyMatch(t -> t.getAction().startsWith(TransportListTasksAction.TYPE.name())) ) ); diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java index cbab099f9cba8..b6bd16e12c8ab 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java @@ -9,7 +9,7 @@ package org.elasticsearch.http; import org.apache.http.client.methods.HttpGet; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; +import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -61,7 +61,7 @@ public void testClusterInfoRequestCancellation() throws Exception { ); assertFalse(future.isDone()); - awaitTaskWithPrefix(NodesStatsAction.NAME); + awaitTaskWithPrefix(TransportNodesStatsAction.TYPE.name()); logger.info("--> Checking that all the HttpTransport are waiting..."); safeAwait(cyclicBarrier); @@ -71,12 +71,12 @@ public void testClusterInfoRequestCancellation() throws Exception { assertTrue(future.isDone()); expectThrows(CancellationException.class, future::actionGet); - assertAllCancellableTasksAreCancelled(NodesStatsAction.NAME); + assertAllCancellableTasksAreCancelled(TransportNodesStatsAction.TYPE.name()); logger.info("--> Releasing all the node requests :)"); safeAwait(cyclicBarrier); - assertAllTasksHaveFinished(NodesStatsAction.NAME); + assertAllTasksHaveFinished(TransportNodesStatsAction.TYPE.name()); } @Override diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java index ef0ab68989f66..5800c60b3b8a4 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java @@ -9,11 +9,11 @@ package org.elasticsearch.http; import org.apache.http.client.methods.HttpGet; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; +import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.client.Request; public class NodeStatsRestCancellationIT extends BlockedSearcherRestCancellationTestCase { public void testNodeStatsRestCancellation() throws Exception { - runTest(new Request(HttpGet.METHOD_NAME, "/_nodes/stats"), NodesStatsAction.NAME); + runTest(new Request(HttpGet.METHOD_NAME, "/_nodes/stats"), TransportNodesStatsAction.TYPE.name()); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index e33e7ee148e04..de51066a1f3ba 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -19,8 +19,8 @@ import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest; import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction; @@ -126,7 +126,9 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { public void testTaskCounts() { // Run only on data nodes - ListTasksResponse response = clusterAdmin().prepareListTasks("data:true").setActions(ListTasksAction.NAME + "[n]").get(); + ListTasksResponse response = clusterAdmin().prepareListTasks("data:true") + .setActions(TransportListTasksAction.TYPE.name() + "[n]") + .get(); assertThat(response.getTasks().size(), greaterThanOrEqualTo(cluster().numDataNodes())); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java index 13e2780f844ad..cdbc19611eb24 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java @@ -33,12 +33,14 @@ protected Collection remoteClusterAlias() { public void testRemoteClusterClientRole() { final InternalTestCluster localCluster = cluster(LOCAL_CLUSTER); - localCluster.client().execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest()).actionGet(); + localCluster.client().execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest()).actionGet(); final String nodeWithoutRemoteClientRole = localCluster.startNode(NodeRoles.onlyRoles(Set.of(DiscoveryNodeRole.DATA_ROLE))); final IllegalArgumentException error = expectThrows( IllegalArgumentException.class, - () -> localCluster.client(nodeWithoutRemoteClientRole).execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest()).actionGet() + () -> localCluster.client(nodeWithoutRemoteClientRole) + .execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest()) + .actionGet() ); assertThat( error.getMessage(), @@ -51,7 +53,7 @@ public void testRemoteClusterClientRole() { roles.add(DiscoveryNodeRole.DATA_ROLE); } final String nodeWithRemoteClientRole = cluster(LOCAL_CLUSTER).startNode(NodeRoles.onlyRoles(roles)); - localCluster.client(nodeWithRemoteClientRole).execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest()).actionGet(); + localCluster.client(nodeWithRemoteClientRole).execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest()).actionGet(); } public void testAllowStartingNodeWithUnreachableRemoteCluster() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java index 285d25059c4b4..3846711764ec9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java @@ -11,7 +11,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; +import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; @@ -247,8 +247,8 @@ public void testClusterInfoServiceInformationClearOnError() { final AtomicBoolean timeout = new AtomicBoolean(false); final Set blockedActions = newHashSet( - NodesStatsAction.NAME, - NodesStatsAction.NAME + "[n]", + TransportNodesStatsAction.TYPE.name(), + TransportNodesStatsAction.TYPE.name() + "[n]", IndicesStatsAction.NAME, IndicesStatsAction.NAME + "[n]" ); diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 2d43a2b26b960..6ac451d5bc93b 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -35,27 +35,21 @@ import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; import org.elasticsearch.action.admin.cluster.migration.TransportPostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction; import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsAction; import org.elasticsearch.action.admin.cluster.node.reload.TransportNodesReloadSecureSettingsAction; import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateNodeRemovalAction; import org.elasticsearch.action.admin.cluster.node.shutdown.TransportPrevalidateNodeRemovalAction; import org.elasticsearch.action.admin.cluster.node.shutdown.TransportPrevalidateShardPathAction; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction; import org.elasticsearch.action.admin.cluster.node.tasks.get.TransportGetTaskAction; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; -import org.elasticsearch.action.admin.cluster.node.usage.NodesUsageAction; import org.elasticsearch.action.admin.cluster.node.usage.TransportNodesUsageAction; import org.elasticsearch.action.admin.cluster.remote.RemoteClusterNodesAction; -import org.elasticsearch.action.admin.cluster.remote.RemoteInfoAction; import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryAction; import org.elasticsearch.action.admin.cluster.repositories.cleanup.TransportCleanupRepositoryAction; @@ -670,13 +664,13 @@ public void reg } ActionRegistry actions = new ActionRegistry(); - actions.register(NodesInfoAction.INSTANCE, TransportNodesInfoAction.class); - actions.register(RemoteInfoAction.INSTANCE, TransportRemoteInfoAction.class); - actions.register(RemoteClusterNodesAction.INSTANCE, RemoteClusterNodesAction.TransportAction.class); - actions.register(NodesStatsAction.INSTANCE, TransportNodesStatsAction.class); - actions.register(NodesUsageAction.INSTANCE, TransportNodesUsageAction.class); - actions.register(NodesHotThreadsAction.INSTANCE, TransportNodesHotThreadsAction.class); - actions.register(ListTasksAction.INSTANCE, TransportListTasksAction.class); + actions.register(TransportNodesInfoAction.TYPE, TransportNodesInfoAction.class); + actions.register(TransportRemoteInfoAction.TYPE, TransportRemoteInfoAction.class); + actions.register(RemoteClusterNodesAction.TYPE, RemoteClusterNodesAction.TransportAction.class); + actions.register(TransportNodesStatsAction.TYPE, TransportNodesStatsAction.class); + actions.register(TransportNodesUsageAction.TYPE, TransportNodesUsageAction.class); + actions.register(TransportNodesHotThreadsAction.TYPE, TransportNodesHotThreadsAction.class); + actions.register(TransportListTasksAction.TYPE, TransportListTasksAction.class); actions.register(GetTaskAction.INSTANCE, TransportGetTaskAction.class); actions.register(CancelTasksAction.INSTANCE, TransportCancelTasksAction.class); actions.register(GetHealthAction.INSTANCE, GetHealthAction.LocalAction.class); diff --git a/server/src/main/java/org/elasticsearch/action/ActionType.java b/server/src/main/java/org/elasticsearch/action/ActionType.java index d24abf049ec1e..27820398d10a3 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionType.java +++ b/server/src/main/java/org/elasticsearch/action/ActionType.java @@ -18,6 +18,10 @@ public class ActionType { private final String name; private final Writeable.Reader responseReader; + public static ActionType localOnly(String name) { + return new ActionType<>(name, Writeable.Reader.localOnly()); + } + /** * @param name The name of the action, must be unique across actions. * @param responseReader A reader for the response type diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsAction.java deleted file mode 100644 index d683e6a042939..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsAction.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.admin.cluster.node.hotthreads; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.common.io.stream.Writeable; - -public class NodesHotThreadsAction extends ActionType { - - public static final NodesHotThreadsAction INSTANCE = new NodesHotThreadsAction(); - public static final String NAME = "cluster:monitor/nodes/hot_threads"; - - private NodesHotThreadsAction() { - super(NAME, Writeable.Reader.localOnly()); - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java index 68f859467a010..a1276e164741c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.node.hotthreads; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; import org.elasticsearch.core.TimeValue; @@ -18,7 +19,7 @@ public class NodesHotThreadsRequestBuilder extends NodesOperationRequestBuilder< NodesHotThreadsResponse, NodesHotThreadsRequestBuilder> { - public NodesHotThreadsRequestBuilder(ElasticsearchClient client, NodesHotThreadsAction action) { + public NodesHotThreadsRequestBuilder(ElasticsearchClient client, ActionType action) { super(client, action, new NodesHotThreadsRequest()); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java index 3be934f93dd27..3fabd53299674 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.admin.cluster.node.hotthreads; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.nodes.TransportNodesAction; @@ -32,6 +33,8 @@ public class TransportNodesHotThreadsAction extends TransportNodesAction< TransportNodesHotThreadsAction.NodeRequest, NodeHotThreads> { + public static final ActionType TYPE = ActionType.localOnly("cluster:monitor/nodes/hot_threads"); + @Inject public TransportNodesHotThreadsAction( ThreadPool threadPool, @@ -40,7 +43,7 @@ public TransportNodesHotThreadsAction( ActionFilters actionFilters ) { super( - NodesHotThreadsAction.NAME, + TYPE.name(), clusterService, transportService, actionFilters, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java deleted file mode 100644 index 5cb4a8ef3dbb8..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.admin.cluster.node.info; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.common.io.stream.Writeable; - -public class NodesInfoAction extends ActionType { - - public static final NodesInfoAction INSTANCE = new NodesInfoAction(); - public static final String NAME = "cluster:monitor/nodes/info"; - - private NodesInfoAction() { - super(NAME, Writeable.Reader.localOnly()); - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java index 543804869eaa0..52f3ced207be3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java @@ -8,13 +8,14 @@ package org.elasticsearch.action.admin.cluster.node.info; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; // TODO: This class's interface should match that of NodesInfoRequest public class NodesInfoRequestBuilder extends NodesOperationRequestBuilder { - public NodesInfoRequestBuilder(ElasticsearchClient client, NodesInfoAction action) { + public NodesInfoRequestBuilder(ElasticsearchClient client, ActionType action) { super(client, action, new NodesInfoRequest()); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java index 69ecca8fc4f3a..89753bfa94df8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.node.info; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.nodes.TransportNodesAction; @@ -34,6 +35,7 @@ public class TransportNodesInfoAction extends TransportNodesAction< TransportNodesInfoAction.NodeInfoRequest, NodeInfo> { + public static final ActionType TYPE = ActionType.localOnly("cluster:monitor/nodes/info"); private final NodeService nodeService; @Inject @@ -45,7 +47,7 @@ public TransportNodesInfoAction( ActionFilters actionFilters ) { super( - NodesInfoAction.NAME, + TYPE.name(), clusterService, transportService, actionFilters, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java deleted file mode 100644 index c475088fc434e..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.admin.cluster.node.stats; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.common.io.stream.Writeable; - -public class NodesStatsAction extends ActionType { - - public static final NodesStatsAction INSTANCE = new NodesStatsAction(); - public static final String NAME = "cluster:monitor/nodes/stats"; - - private NodesStatsAction() { - super(NAME, Writeable.Reader.localOnly()); - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java index ae62dc0060737..1cfa92d91e9f1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.node.stats; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; @@ -17,7 +18,7 @@ public class NodesStatsRequestBuilder extends NodesOperationRequestBuilder< NodesStatsResponse, NodesStatsRequestBuilder> { - public NodesStatsRequestBuilder(ElasticsearchClient client, NodesStatsAction action) { + public NodesStatsRequestBuilder(ElasticsearchClient client, ActionType action) { super(client, action, new NodesStatsRequest()); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 10b22694b8a74..b9ab520c4da8d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.node.stats; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.nodes.TransportNodesAction; @@ -36,6 +37,7 @@ public class TransportNodesStatsAction extends TransportNodesAction< TransportNodesStatsAction.NodeStatsRequest, NodeStats> { + public static final ActionType TYPE = ActionType.localOnly("cluster:monitor/nodes/stats"); private final NodeService nodeService; @Inject @@ -47,7 +49,7 @@ public TransportNodesStatsAction( ActionFilters actionFilters ) { super( - NodesStatsAction.NAME, + TYPE.name(), clusterService, transportService, actionFilters, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksAction.java deleted file mode 100644 index 85bbe1e659cb7..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksAction.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.admin.cluster.node.tasks.list; - -import org.elasticsearch.action.ActionType; - -/** - * ActionType for retrieving a list of currently running tasks - */ -public class ListTasksAction extends ActionType { - - public static final ListTasksAction INSTANCE = new ListTasksAction(); - public static final String NAME = "cluster:monitor/tasks/lists"; - - private ListTasksAction() { - super(NAME, ListTasksResponse::new); - } - -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java index 87045ad7ef8b7..cf50f9bca581f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.node.tasks.list; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.tasks.TasksRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; @@ -16,7 +17,7 @@ */ public class ListTasksRequestBuilder extends TasksRequestBuilder { - public ListTasksRequestBuilder(ElasticsearchClient client, ListTasksAction action) { + public ListTasksRequestBuilder(ElasticsearchClient client, ActionType action) { super(client, action, new ListTasksRequest()); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java index 1c02ff5ab524f..f79b0ac9c02b4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.admin.cluster.node.tasks.list; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.action.support.ActionFilters; @@ -39,6 +40,8 @@ import static org.elasticsearch.core.TimeValue.timeValueSeconds; public class TransportListTasksAction extends TransportTasksAction { + public static final ActionType TYPE = new ActionType<>("cluster:monitor/tasks/lists", ListTasksResponse::new); + public static long waitForCompletionTimeout(TimeValue timeout) { if (timeout == null) { timeout = DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT; @@ -51,7 +54,7 @@ public static long waitForCompletionTimeout(TimeValue timeout) { @Inject public TransportListTasksAction(ClusterService clusterService, TransportService transportService, ActionFilters actionFilters) { super( - ListTasksAction.NAME, + TYPE.name(), clusterService, transportService, actionFilters, @@ -117,7 +120,7 @@ protected void processTasks(CancellableTask nodeTask, ListTasksRequest request, ); try { for (final var task : processTasks(request)) { - if (task.getAction().startsWith(ListTasksAction.NAME) == false) { + if (task.getAction().startsWith(TYPE.name()) == false) { // It doesn't make sense to wait for List Tasks and it can cause an infinite loop of the task waiting // for itself or one of its child tasks matchedTasks.add(task); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageAction.java deleted file mode 100644 index 1e9232aae28ba..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageAction.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.admin.cluster.node.usage; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.common.io.stream.Writeable; - -public class NodesUsageAction extends ActionType { - - public static final NodesUsageAction INSTANCE = new NodesUsageAction(); - public static final String NAME = "cluster:monitor/nodes/usage"; - - protected NodesUsageAction() { - super(NAME, Writeable.Reader.localOnly()); - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java index 71802fea0a47c..c661023e79629 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.node.usage; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.nodes.TransportNodesAction; @@ -33,6 +34,7 @@ public class TransportNodesUsageAction extends TransportNodesAction< TransportNodesUsageAction.NodeUsageRequest, NodeUsage> { + public static final ActionType TYPE = ActionType.localOnly("cluster:monitor/nodes/usage"); private final UsageService restUsageService; private final AggregationUsageService aggregationUsageService; private final long sinceTime; @@ -47,7 +49,7 @@ public TransportNodesUsageAction( AggregationUsageService aggregationUsageService ) { super( - NodesUsageAction.NAME, + TYPE.name(), clusterService, transportService, actionFilters, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java index befec9d468f19..a329d07fe5f63 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java @@ -13,9 +13,9 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoMetrics; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.nodes.BaseNodeResponse; @@ -34,14 +34,10 @@ import java.util.List; import java.util.Objects; -public class RemoteClusterNodesAction extends ActionType { +public class RemoteClusterNodesAction { - public static final RemoteClusterNodesAction INSTANCE = new RemoteClusterNodesAction(); public static final String NAME = "cluster:internal/remote_cluster/nodes"; - - public RemoteClusterNodesAction() { - super(NAME, RemoteClusterNodesAction.Response::new); - } + public static final ActionType TYPE = new ActionType<>(NAME, RemoteClusterNodesAction.Response::new); public static class Request extends ActionRequest { public static final Request ALL_NODES = new Request(false); @@ -97,7 +93,7 @@ public static class TransportAction extends HandledTransportAction li if (request.remoteClusterServer) { final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().clear() .addMetrics(NodesInfoMetrics.Metric.REMOTE_CLUSTER_SERVER.metricName()); - client.execute(NodesInfoAction.INSTANCE, nodesInfoRequest, listener.delegateFailureAndWrap((l, response) -> { + client.execute(TransportNodesInfoAction.TYPE, nodesInfoRequest, listener.delegateFailureAndWrap((l, response) -> { l.onResponse(new Response(response.getNodes().stream().map(nodeInfo -> { final RemoteClusterServerInfo remoteClusterServerInfo = nodeInfo.getInfo(RemoteClusterServerInfo.class); if (remoteClusterServerInfo == null) { @@ -121,7 +117,7 @@ protected void doExecute(Task task, Request request, ActionListener li } else { final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().clear(); client.execute( - NodesInfoAction.INSTANCE, + TransportNodesInfoAction.TYPE, nodesInfoRequest, listener.delegateFailureAndWrap( (l, response) -> l.onResponse( diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java deleted file mode 100644 index 494e951b43674..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.admin.cluster.remote; - -import org.elasticsearch.action.ActionType; - -public final class RemoteInfoAction extends ActionType { - - public static final String NAME = "cluster:monitor/remote/info"; - public static final RemoteInfoAction INSTANCE = new RemoteInfoAction(); - - public RemoteInfoAction() { - super(NAME, RemoteInfoResponse::new); - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java index 31b9e46550ffe..6e95a9807c4b2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.admin.cluster.remote; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.search.SearchTransportService; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; @@ -23,6 +24,7 @@ public final class TransportRemoteInfoAction extends HandledTransportAction { + public static final ActionType TYPE = new ActionType<>("cluster:monitor/remote/info", RemoteInfoResponse::new); private final RemoteClusterService remoteClusterService; @Inject @@ -31,7 +33,7 @@ public TransportRemoteInfoAction( ActionFilters actionFilters, SearchTransportService searchTransportService ) { - super(RemoteInfoAction.NAME, transportService, actionFilters, RemoteInfoRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); + super(TYPE.name(), transportService, actionFilters, RemoteInfoRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.remoteClusterService = searchTransportService.getRemoteClusterService(); } diff --git a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java index 5ac1ae6a76611..98bad4d3dd74c 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java @@ -24,20 +24,20 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequestBuilder; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequestBuilder; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsAction; import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequestBuilder; @@ -46,13 +46,13 @@ import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest; import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequestBuilder; import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequestBuilder; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; -import org.elasticsearch.action.admin.cluster.node.usage.NodesUsageAction; +import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.admin.cluster.node.usage.NodesUsageRequest; import org.elasticsearch.action.admin.cluster.node.usage.NodesUsageResponse; +import org.elasticsearch.action.admin.cluster.node.usage.TransportNodesUsageAction; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryAction; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequestBuilder; @@ -750,37 +750,37 @@ public NodesReloadSecureSettingsRequestBuilder prepareReloadSecureSettings() { @Override public ActionFuture nodesInfo(final NodesInfoRequest request) { - return execute(NodesInfoAction.INSTANCE, request); + return execute(TransportNodesInfoAction.TYPE, request); } @Override public void nodesInfo(final NodesInfoRequest request, final ActionListener listener) { - execute(NodesInfoAction.INSTANCE, request, listener); + execute(TransportNodesInfoAction.TYPE, request, listener); } @Override public NodesInfoRequestBuilder prepareNodesInfo(String... nodesIds) { - return new NodesInfoRequestBuilder(this, NodesInfoAction.INSTANCE).setNodesIds(nodesIds); + return new NodesInfoRequestBuilder(this, TransportNodesInfoAction.TYPE).setNodesIds(nodesIds); } @Override public ActionFuture nodesStats(final NodesStatsRequest request) { - return execute(NodesStatsAction.INSTANCE, request); + return execute(TransportNodesStatsAction.TYPE, request); } @Override public void nodesStats(final NodesStatsRequest request, final ActionListener listener) { - execute(NodesStatsAction.INSTANCE, request, listener); + execute(TransportNodesStatsAction.TYPE, request, listener); } @Override public NodesStatsRequestBuilder prepareNodesStats(String... nodesIds) { - return new NodesStatsRequestBuilder(this, NodesStatsAction.INSTANCE).setNodesIds(nodesIds); + return new NodesStatsRequestBuilder(this, TransportNodesStatsAction.TYPE).setNodesIds(nodesIds); } @Override public void nodesUsage(final NodesUsageRequest request, final ActionListener listener) { - execute(NodesUsageAction.INSTANCE, request, listener); + execute(TransportNodesUsageAction.TYPE, request, listener); } @Override @@ -795,27 +795,27 @@ public ClusterStatsRequestBuilder prepareClusterStats() { @Override public void nodesHotThreads(NodesHotThreadsRequest request, ActionListener listener) { - execute(NodesHotThreadsAction.INSTANCE, request, listener); + execute(TransportNodesHotThreadsAction.TYPE, request, listener); } @Override public NodesHotThreadsRequestBuilder prepareNodesHotThreads(String... nodesIds) { - return new NodesHotThreadsRequestBuilder(this, NodesHotThreadsAction.INSTANCE).setNodesIds(nodesIds); + return new NodesHotThreadsRequestBuilder(this, TransportNodesHotThreadsAction.TYPE).setNodesIds(nodesIds); } @Override public ActionFuture listTasks(final ListTasksRequest request) { - return execute(ListTasksAction.INSTANCE, request); + return execute(TransportListTasksAction.TYPE, request); } @Override public void listTasks(final ListTasksRequest request, final ActionListener listener) { - execute(ListTasksAction.INSTANCE, request, listener); + execute(TransportListTasksAction.TYPE, request, listener); } @Override public ListTasksRequestBuilder prepareListTasks(String... nodesIds) { - return new ListTasksRequestBuilder(this, ListTasksAction.INSTANCE).setNodesIds(nodesIds); + return new ListTasksRequestBuilder(this, TransportListTasksAction.TYPE).setNodesIds(nodesIds); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java index f2c45ebc6e748..7a9c7c84d0f00 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java @@ -11,9 +11,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; +import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.ReferenceDocs; @@ -271,7 +271,7 @@ public void onResponse(Releasable releasable) { try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { threadContext.markAsSystemContext(); client.execute( - NodesHotThreadsAction.INSTANCE, + TransportNodesHotThreadsAction.TYPE, new NodesHotThreadsRequest(discoveryNode).threads(500), ActionListener.runBefore(debugListener, () -> Releasables.close(releasable)) ); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java index 61d6c4283ca62..2064f0b2c0fba 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java @@ -8,8 +8,8 @@ package org.elasticsearch.rest.action.admin.cluster; -import org.elasticsearch.action.admin.cluster.remote.RemoteInfoAction; import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest; +import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -33,7 +33,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { - return channel -> client.execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest(), new RestToXContentListener<>(channel)); + return channel -> client.execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest(), new RestToXContentListener<>(channel)); } @Override diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 8499aa03d7808..a055e4122257f 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -124,7 +124,7 @@ void collectNodes(ActionListener> listener) { if (REMOTE_CLUSTER_PROFILE.equals(remoteConnectionManager.getConnectionProfile().getTransportProfile())) { transportService.sendRequest( connection, - RemoteClusterNodesAction.NAME, + RemoteClusterNodesAction.TYPE.name(), RemoteClusterNodesAction.Request.ALL_NODES, TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(contextPreservingActionListener.map(response -> { diff --git a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java index 632828c8df901..869ec2453f411 100644 --- a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java @@ -311,7 +311,7 @@ private void collectRemoteNodes(Iterator> seedNodesSuppl final AbstractSniffResponseHandler sniffResponseHandler; // Use different action to collect nodes information depending on the connection model if (REMOTE_CLUSTER_PROFILE.equals(connectionManager.getConnectionProfile().getTransportProfile())) { - action = RemoteClusterNodesAction.NAME; + action = RemoteClusterNodesAction.TYPE.name(); request = RemoteClusterNodesAction.Request.REMOTE_CLUSTER_SERVER_NODES; sniffResponseHandler = new RemoteClusterNodesSniffResponseHandler(connection, listener, seedNodesSuppliers); } else { diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java index 28b4c8a63df4d..614ede7ec96e4 100644 --- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java +++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.action; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; @@ -59,7 +58,10 @@ public class ActionModuleTests extends ESTestCase { public void testSetupActionsContainsKnownBuiltin() { assertThat( ActionModule.setupActions(emptyList()), - hasEntry(NodesInfoAction.INSTANCE.name(), new ActionHandler<>(NodesInfoAction.INSTANCE, TransportNodesInfoAction.class)) + hasEntry( + TransportNodesInfoAction.TYPE.name(), + new ActionHandler<>(TransportNodesInfoAction.TYPE, TransportNodesInfoAction.class) + ) ); } @@ -67,11 +69,11 @@ public void testPluginCantOverwriteBuiltinAction() { ActionPlugin dupsMainAction = new ActionPlugin() { @Override public List> getActions() { - return singletonList(new ActionHandler<>(NodesInfoAction.INSTANCE, TransportNodesInfoAction.class)); + return singletonList(new ActionHandler<>(TransportNodesInfoAction.TYPE, TransportNodesInfoAction.class)); } }; Exception e = expectThrows(IllegalArgumentException.class, () -> ActionModule.setupActions(singletonList(dupsMainAction))); - assertEquals("action for name [" + NodesInfoAction.NAME + "] already registered", e.getMessage()); + assertEquals("action for name [" + TransportNodesInfoAction.TYPE.name() + "] already registered", e.getMessage()); } public void testPluginCanRegisterAction() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index a650692eea9f4..30f6c2fda8dc1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -14,10 +14,10 @@ import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup; +import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; @@ -922,7 +922,7 @@ public void testTasksToXContentGrouping() throws Exception { // Get the parent task ListTasksRequest listTasksRequest = new ListTasksRequest(); - listTasksRequest.setActions(ListTasksAction.NAME + "*"); + listTasksRequest.setActions(TransportListTasksAction.TYPE.name() + "*"); ListTasksResponse response = ActionTestUtils.executeBlockingWithTask( testNodes[0].transportService.getTaskManager(), testNodes[0].transportService.getLocalNodeConnection(), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java index 00a4ad29f68d3..dc2578b835de2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java @@ -15,10 +15,10 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoMetrics; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.support.AbstractClient; @@ -114,7 +114,7 @@ protected void Request request, ActionListener listener ) { - assertSame(NodesInfoAction.INSTANCE, action); + assertSame(TransportNodesInfoAction.TYPE, action); assertThat( asInstanceOf(NodesInfoRequest.class, request).requestedMetrics(), containsInAnyOrder(NodesInfoMetrics.Metric.REMOTE_CLUSTER_SERVER.metricName()) @@ -191,7 +191,7 @@ protected void Request request, ActionListener listener ) { - assertSame(NodesInfoAction.INSTANCE, action); + assertSame(TransportNodesInfoAction.TYPE, action); assertThat(asInstanceOf(NodesInfoRequest.class, request).requestedMetrics(), empty()); listener.onResponse((Response) nodesInfoResponse); } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 198eff76b172d..44b48b60a9c5e 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -192,7 +192,7 @@ public static MockTransportService startTransport( ); if (RemoteClusterPortSettings.REMOTE_CLUSTER_SERVER_ENABLED.get(s)) { newService.registerRequestHandler( - RemoteClusterNodesAction.NAME, + RemoteClusterNodesAction.TYPE.name(), EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(knownNodes)) @@ -640,7 +640,10 @@ private void doTestCollectNodes(boolean hasClusterCredentials) throws Exception service.acceptIncomingRequests(); service.addSendBehavior((connection, requestId, action, request, options) -> { if (hasClusterCredentials) { - assertThat(action, oneOf(RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, RemoteClusterNodesAction.NAME)); + assertThat( + action, + oneOf(RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, RemoteClusterNodesAction.TYPE.name()) + ); } else { assertThat(action, oneOf(TransportService.HANDSHAKE_ACTION_NAME, ClusterStateAction.NAME)); } diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index ee6a960780803..3f67c0c65d559 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -134,7 +134,7 @@ public MockTransportService startTransport( ); if (hasClusterCredentials) { newService.registerRequestHandler( - RemoteClusterNodesAction.NAME, + RemoteClusterNodesAction.TYPE.name(), EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(knownNodes)) @@ -1188,7 +1188,7 @@ public void testModeSettingsCannotBeUsedWhenInDifferentMode() { private void addSendBehaviour(MockTransportService transportService) { transportService.addSendBehavior((connection, requestId, action, request, options) -> { if (hasClusterCredentials) { - assertThat(action, oneOf(RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, RemoteClusterNodesAction.NAME)); + assertThat(action, oneOf(RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, RemoteClusterNodesAction.TYPE.name())); } else { assertThat(action, oneOf(TransportService.HANDSHAKE_ACTION_NAME, ClusterStateAction.NAME)); } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java index 57523d4a47e0b..0b19f1aa422f3 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -18,7 +18,6 @@ import org.elasticsearch.action.admin.cluster.coordination.ClusterFormationInfoAction; import org.elasticsearch.action.admin.cluster.coordination.CoordinationDiagnosticsAction; import org.elasticsearch.action.admin.cluster.coordination.MasterHistoryAction; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction; import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.client.internal.node.NodeClient; @@ -1176,7 +1175,7 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { ); client.initialize( Map.of( - NodesHotThreadsAction.INSTANCE, + TransportNodesHotThreadsAction.TYPE, new TransportNodesHotThreadsAction(threadPool, clusterService, transportService, new ActionFilters(emptySet())), MasterHistoryAction.INSTANCE, new MasterHistoryAction.TransportAction(transportService, new ActionFilters(Set.of()), masterHistoryService), diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java index 61a1bef41101d..4037ed796011f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java @@ -10,8 +10,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.cluster.remote.RemoteInfoAction; import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest; +import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.network.NetworkModule; @@ -211,7 +211,7 @@ protected void configureRemoteCluster(String clusterAlias, Collection se } assertBusy(() -> { - List remoteConnectionInfos = client().execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest()) + List remoteConnectionInfos = client().execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest()) .actionGet() .getInfos() .stream() diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 444a0c6d752dc..80237927655e0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -22,7 +22,7 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.Build; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; +import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; @@ -494,7 +494,7 @@ public static void waitForPendingTasks(final RestClient restClient, final Predic final StringBuilder tasksListString = new StringBuilder(); while ((line = responseReader.readLine()) != null) { final String taskName = line.split("\\s+")[0]; - if (taskName.startsWith(ListTasksAction.NAME) + if (taskName.startsWith(TransportListTasksAction.TYPE.name()) || taskName.startsWith(HealthNode.TASK_NAME) || taskFilter.test(taskName)) { continue; @@ -1258,8 +1258,8 @@ private static void deleteAllAutoFollowPatterns() throws IOException { private void logIfThereAreRunningTasks() throws IOException { Set runningTasks = runningTasks(adminClient().performRequest(new Request("GET", "/_tasks"))); // Ignore the task list API - it doesn't count against us - runningTasks.remove(ListTasksAction.NAME); - runningTasks.remove(ListTasksAction.NAME + "[n]"); + runningTasks.remove(TransportListTasksAction.TYPE.name()); + runningTasks.remove(TransportListTasksAction.TYPE.name() + "[n]"); if (runningTasks.isEmpty()) { return; } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java index eaade3bbc8eef..8d357a5f050ca 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java @@ -15,13 +15,13 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -522,8 +522,11 @@ protected void Request request, ActionListener listener ) { - assertThat(action, anyOf(Matchers.sameInstance(NodesStatsAction.INSTANCE), Matchers.sameInstance(NodesInfoAction.INSTANCE))); - if (action == NodesStatsAction.INSTANCE) { + assertThat( + action, + anyOf(Matchers.sameInstance(TransportNodesStatsAction.TYPE), Matchers.sameInstance(TransportNodesInfoAction.TYPE)) + ); + if (action == TransportNodesStatsAction.TYPE) { NodesStatsRequest nodesStatsRequest = (NodesStatsRequest) request; assertThat(nodesStatsRequest.timeout(), equalTo(fetchTimeout)); assertThat(responderStats, notNullValue()); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java index cb2c685fae59c..2cb58a9991176 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java @@ -9,8 +9,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.admin.cluster.remote.RemoteInfoAction; import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest; +import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -149,7 +149,7 @@ private void setupRemoteCluster() throws Exception { } private boolean isRemoteConnected() throws Exception { - var infos = followerClient().execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest()).get().getInfos(); + var infos = followerClient().execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest()).get().getInfos(); return infos.size() == 1 && infos.get(0).isConnected(); } @@ -165,7 +165,7 @@ private void cleanRemoteCluster() throws Exception { assertAcked(followerClient().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); assertBusy(() -> { - List infos = followerClient().execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest()) + List infos = followerClient().execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest()) .get() .getInfos(); assertThat(infos.size(), equalTo(0)); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java index a540631dc040e..9b6d81a206f71 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java @@ -7,8 +7,8 @@ package org.elasticsearch.xpack; -import org.elasticsearch.action.admin.cluster.remote.RemoteInfoAction; import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest; +import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.service.ClusterService; @@ -66,7 +66,7 @@ public void setupLocalRemote() throws Exception { updateSettingsRequest.transientSettings(Settings.builder().put("cluster.remote.local.seeds", address)); assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - List infos = client().execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest()).get().getInfos(); + List infos = client().execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest()).get().getInfos(); assertThat(infos.size(), equalTo(1)); assertTrue(infos.get(0).isConnected()); } @@ -89,7 +89,7 @@ public void removeLocalRemote() throws Exception { assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); assertBusy(() -> { - List infos = client().execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest()).get().getInfos(); + List infos = client().execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest()).get().getInfos(); assertThat(infos.size(), equalTo(0)); }); } diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java index e4791dc256335..f669bb8589fd7 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.core.rest.action; import org.apache.http.client.methods.HttpGet; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; +import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; @@ -76,7 +76,7 @@ public void testCancellation() throws Exception { final SubscribableListener nodeStatsRequestsReleaseListener = new SubscribableListener<>(); for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { ((MockTransportService) transportService).addRequestHandlingBehavior( - NodesStatsAction.NAME + "[n]", + TransportNodesStatsAction.TYPE.name() + "[n]", (handler, request, channel, task) -> { tasksBlockedLatch.countDown(); nodeStatsRequestsReleaseListener.addListener( @@ -95,13 +95,13 @@ public void testCancellation() throws Exception { cancellable.cancel(); // NB this test works by blocking node-level stats requests; when #100230 is addressed this will need to target a different action. - assertAllCancellableTasksAreCancelled(NodesStatsAction.NAME); + assertAllCancellableTasksAreCancelled(TransportNodesStatsAction.TYPE.name()); assertAllCancellableTasksAreCancelled(XPackUsageAction.NAME); nodeStatsRequestsReleaseListener.onResponse(null); expectThrows(CancellationException.class, future::actionGet); - assertAllTasksHaveFinished(NodesStatsAction.NAME); + assertAllTasksHaveFinished(TransportNodesStatsAction.TYPE.name()); assertAllTasksHaveFinished(XPackUsageAction.NAME); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlPlatformArchitecturesUtil.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlPlatformArchitecturesUtil.java index 48ded3a4c0c45..9802e06223332 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlPlatformArchitecturesUtil.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlPlatformArchitecturesUtil.java @@ -8,10 +8,10 @@ package org.elasticsearch.xpack.core.ml.utils; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequestBuilder; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.monitor.os.OsInfo; @@ -41,7 +41,7 @@ public static void getMlNodesArchitecturesSet( ); NodesInfoRequest request = MlPlatformArchitecturesUtil.getNodesInfoBuilderWithMlNodeArchitectureInfo(client).request(); - executeAsyncWithOrigin(client, ML_ORIGIN, NodesInfoAction.INSTANCE, request, listener); + executeAsyncWithOrigin(client, ML_ORIGIN, TransportNodesInfoAction.TYPE, request, listener); } static ActionListener getArchitecturesSetFromNodesInfoResponseListener( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java index 68c8491b70528..9230519a85773 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java @@ -165,12 +165,12 @@ public class ClusterPrivilegeResolver { private static final Set CROSS_CLUSTER_SEARCH_PATTERN = Set.of( RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, - RemoteClusterNodesAction.NAME, + RemoteClusterNodesAction.TYPE.name(), XPackInfoAction.NAME ); private static final Set CROSS_CLUSTER_REPLICATION_PATTERN = Set.of( RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, - RemoteClusterNodesAction.NAME, + RemoteClusterNodesAction.TYPE.name(), XPackInfoAction.NAME, ClusterStateAction.NAME ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java index ea9c81efd9342..da8e97b10dc87 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.core.security.authz.store; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.remote.RemoteInfoAction; +import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; @@ -164,7 +164,7 @@ private static Map initializeReservedRoles() { "monitoring_user", new RoleDescriptor( "monitoring_user", - new String[] { "cluster:monitor/main", "cluster:monitor/xpack/info", RemoteInfoAction.NAME }, + new String[] { "cluster:monitor/main", "cluster:monitor/xpack/info", TransportRemoteInfoAction.TYPE.name() }, new RoleDescriptor.IndicesPrivileges[] { RoleDescriptor.IndicesPrivileges.builder() .indices(".monitoring-*") diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 90ef841fb54d4..ea4350262bf41 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -8,7 +8,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.remote.RemoteInfoAction; +import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesAction; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryAction; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteAction; @@ -1491,7 +1491,7 @@ public void testMonitoringUserRole() { ); assertThat(monitoringUserRole.cluster().check(MainAction.NAME, request, authentication), is(true)); assertThat(monitoringUserRole.cluster().check(XPackInfoAction.NAME, request, authentication), is(true)); - assertThat(monitoringUserRole.cluster().check(RemoteInfoAction.NAME, request, authentication), is(true)); + assertThat(monitoringUserRole.cluster().check(TransportRemoteInfoAction.TYPE.name(), request, authentication), is(true)); assertThat(monitoringUserRole.cluster().check(ClusterHealthAction.NAME, request, authentication), is(false)); assertThat(monitoringUserRole.cluster().check(ClusterStateAction.NAME, request, authentication), is(false)); assertThat(monitoringUserRole.cluster().check(ClusterStatsAction.NAME, request, authentication), is(false)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 7a60dff726909..1ccf4906cf333 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -10,9 +10,9 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; @@ -324,7 +324,7 @@ public void triggerDeleteJobsInStateDeletingWithoutDeletionTask(ActionListener(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelActionTests.java index f708ef1fb2959..c9a2482aac343 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelActionTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; @@ -159,7 +159,7 @@ public void testCheckForExistingTaskCallsOnFailureForAnError() { ActionListener actionListener = (ActionListener) invocationOnMock.getArguments()[2]; actionListener.onFailure(new Exception("error")); return Void.TYPE; - }).when(client).execute(same(ListTasksAction.INSTANCE), any(), any()); + }).when(client).execute(same(TransportListTasksAction.TYPE), any(), any()); var responseListener = new PlainActionFuture(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java index 918b660ae7e61..47ed0e9e269ab 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java @@ -9,9 +9,9 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -120,7 +120,7 @@ public abstract class BaseMlIntegTestCase extends ESIntegTestCase { // The ML jobs can trigger many tasks that are not easily tracked. For this reason, here we list // all the tasks that should be excluded from the cleanup jobs because they are not related to the tests. - private static final Set UNRELATED_TASKS = Set.of(ListTasksAction.NAME, HealthNode.TASK_NAME); + private static final Set UNRELATED_TASKS = Set.of(TransportListTasksAction.TYPE.name(), HealthNode.TASK_NAME); @Override protected boolean ignoreExternalCluster() { @@ -456,7 +456,7 @@ public static void waitForPendingTasks(Client client) throws Exception { ListTasksRequest request = new ListTasksRequest().setDetailed(true); assertBusy(() -> { - ListTasksResponse response = client.execute(ListTasksAction.INSTANCE, request).get(); + ListTasksResponse response = client.execute(TransportListTasksAction.TYPE, request).get(); List activeTasks = response.getTasks() .stream() .filter(t -> UNRELATED_TASKS.stream().noneMatch(name -> t.action().startsWith(name))) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/TaskRetrieverTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/TaskRetrieverTests.java index d36532bd0abe5..3e962aa4f0d19 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/TaskRetrieverTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/TaskRetrieverTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequestBuilder; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.AdminClient; import org.elasticsearch.client.internal.Client; @@ -63,7 +63,7 @@ public void testGetExistingTaskInfoCallsOnFailureForAnError() { actionListener.onFailure(new Exception("error")); return Void.TYPE; - }).when(client).execute(same(ListTasksAction.INSTANCE), any(), any()); + }).when(client).execute(same(TransportListTasksAction.TYPE), any(), any()); var listener = new PlainActionFuture(); @@ -123,7 +123,7 @@ public static Client mockClientWithTasksResponse(List taskInfo, Thread actionListener.onResponse(listTasksResponse); return Void.TYPE; - }).when(client).execute(same(ListTasksAction.INSTANCE), any(), any()); + }).when(client).execute(same(TransportListTasksAction.TYPE), any(), any()); return client; } @@ -150,7 +150,7 @@ public static Client mockListTasksClient(ThreadPool threadPool) { public static Client mockListTasksClient(Client client) { var cluster = client.admin().cluster(); - when(cluster.prepareListTasks()).thenReturn(new ListTasksRequestBuilder(client, ListTasksAction.INSTANCE)); + when(cluster.prepareListTasks()).thenReturn(new ListTasksRequestBuilder(client, TransportListTasksAction.TYPE)); return client; } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityFcActionAuthorizationIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityFcActionAuthorizationIT.java index 3db196212a4a6..a0b4222478e59 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityFcActionAuthorizationIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityFcActionAuthorizationIT.java @@ -287,7 +287,7 @@ public void testRestApiKeyIsNotAllowedOnRemoteClusterPort() throws IOException { final ElasticsearchSecurityException e = expectThrows( ElasticsearchSecurityException.class, () -> remoteClusterClient.execute( - RemoteClusterNodesAction.INSTANCE, + RemoteClusterNodesAction.TYPE, RemoteClusterNodesAction.Request.REMOTE_CLUSTER_SERVER_NODES ).actionGet() ); diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java index 985bec3ba8c77..9788042ac5ece 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java @@ -1040,7 +1040,7 @@ private void expectActionsAndHeadersForCluster( expectedActions.add(SearchShardsAction.NAME); } if (false == useProxyMode) { - expectedActions.add(RemoteClusterNodesAction.NAME); + expectedActions.add(RemoteClusterNodesAction.TYPE.name()); } assertThat( actualActionsWithHeaders.stream().map(CapturedActionWithHeaders::action).collect(Collectors.toUnmodifiableSet()), @@ -1121,7 +1121,7 @@ private static MockTransportService startTransport( } ); service.registerRequestHandler( - RemoteClusterNodesAction.NAME, + RemoteClusterNodesAction.TYPE.name(), EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java index e37500dce8a8f..dcc98a8e6df7d 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java @@ -11,8 +11,8 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.get.GetAction; @@ -183,7 +183,7 @@ public void testCreatingApiKeyWithNoAccess() { final ElasticsearchSecurityException e1 = expectThrows( ElasticsearchSecurityException.class, () -> client().filterWithHeader(Map.of("Authorization", "ApiKey " + base64ApiKeyKeyValue)) - .execute(NodesInfoAction.INSTANCE, new NodesInfoRequest()) + .execute(TransportNodesInfoAction.TYPE, new NodesInfoRequest()) .actionGet() ); assertThat(e1.status().getStatus(), equalTo(403)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentAction.java index dca931a17580b..6483b754f8ae9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentAction.java @@ -10,9 +10,9 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoMetrics; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.internal.Client; @@ -145,7 +145,7 @@ protected void doExecute(Task task, NodeEnrollmentRequest request, ActionListene final List nodeList = new ArrayList<>(); final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().addMetric(NodesInfoMetrics.Metric.TRANSPORT.metricName()); - executeAsyncWithOrigin(client, SECURITY_ORIGIN, NodesInfoAction.INSTANCE, nodesInfoRequest, ActionListener.wrap(response -> { + executeAsyncWithOrigin(client, SECURITY_ORIGIN, TransportNodesInfoAction.TYPE, nodesInfoRequest, ActionListener.wrap(response -> { for (NodeInfo nodeInfo : response.getNodes()) { nodeList.add(nodeInfo.getInfo(TransportInfo.class).getAddress().publishAddress().toString()); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java index d0acc61570853..446dfa7e7e310 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java @@ -12,9 +12,9 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoMetrics; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; @@ -68,7 +68,7 @@ public void maybeCreateNodeEnrollmentToken(Consumer consumer, Iterator { + client.execute(TransportNodesInfoAction.TYPE, nodesInfoRequest, ActionListener.wrap(response -> { assert response.getNodes().size() == 1; NodeInfo nodeInfo = response.getNodes().get(0); TransportInfo transportInfo = nodeInfo.getInfo(TransportInfo.class); @@ -135,7 +135,7 @@ public void createKibanaEnrollmentToken(Consumer consumer, Iter // the enrollment token can only be used against the node that generated it final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().nodesIds("_local") .addMetric(NodesInfoMetrics.Metric.HTTP.metricName()); - client.execute(NodesInfoAction.INSTANCE, nodesInfoRequest, ActionListener.wrap(response -> { + client.execute(TransportNodesInfoAction.TYPE, nodesInfoRequest, ActionListener.wrap(response -> { assert response.getNodes().size() == 1; NodeInfo nodeInfo = response.getNodes().get(0); HttpInfo httpInfo = nodeInfo.getInfo(HttpInfo.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java index ca482decd63a4..5c02f29ab7742 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java @@ -11,9 +11,9 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; @@ -129,7 +129,7 @@ public void testDoExecute() throws Exception { ActionListener listener = (ActionListener) invocation.getArguments()[2]; listener.onResponse(new NodesInfoResponse(new ClusterName("cluster"), nodeInfos, List.of())); return null; - }).when(client).execute(same(NodesInfoAction.INSTANCE), any(), any()); + }).when(client).execute(same(TransportNodesInfoAction.TYPE), any(), any()); final TransportService transportService = new TransportService( Settings.EMPTY, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 8a042a5921a45..5c21edd7226c4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; +import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction; @@ -2679,7 +2679,12 @@ public void testAsyncSearchUserCanAccessOnlyAsyncSearchRestrictedIndices() { } public void testAsyncSearchUserHasNoClusterPrivileges() { - for (String action : Arrays.asList(ClusterStateAction.NAME, GetWatchAction.NAME, ClusterStatsAction.NAME, NodesStatsAction.NAME)) { + for (String action : Arrays.asList( + ClusterStateAction.NAME, + GetWatchAction.NAME, + ClusterStatsAction.NAME, + TransportNodesStatsAction.TYPE.name() + )) { assertThat( getAsyncSearchUserRole().cluster().check(action, mock(TransportRequest.class), AuthenticationTestHelper.builder().build()), Matchers.is(false) @@ -2688,7 +2693,12 @@ public void testAsyncSearchUserHasNoClusterPrivileges() { } public void testXpackUserHasClusterPrivileges() { - for (String action : Arrays.asList(ClusterStateAction.NAME, GetWatchAction.NAME, ClusterStatsAction.NAME, NodesStatsAction.NAME)) { + for (String action : Arrays.asList( + ClusterStateAction.NAME, + GetWatchAction.NAME, + ClusterStatsAction.NAME, + TransportNodesStatsAction.TYPE.name() + )) { assertThat( getXPackUserRole().cluster().check(action, mock(TransportRequest.class), AuthenticationTestHelper.builder().build()), Matchers.is(true) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java index 04bb3391f16a8..0b4f349422e8b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java @@ -11,8 +11,8 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; @@ -139,7 +139,7 @@ public void setup() throws Exception { ); return null; }).when(client).execute(eq(CreateApiKeyAction.INSTANCE), any(CreateApiKeyRequest.class), anyActionListener()); - doAnswer(this::answerWithInfo).when(client).execute(eq(NodesInfoAction.INSTANCE), any(), any()); + doAnswer(this::answerWithInfo).when(client).execute(eq(TransportNodesInfoAction.TYPE), any(), any()); } public void testCreationSuccess() { @@ -181,7 +181,7 @@ public void testFailureToGetNodesInfo() { nodeInfoApiCalls += 1; responseActionListener.onFailure(new Exception("error")); return null; - }).when(client).execute(eq(NodesInfoAction.INSTANCE), any(), any()); + }).when(client).execute(eq(TransportNodesInfoAction.TYPE), any(), any()); final SSLService sslService = new TestsSSLService(environment); final InternalEnrollmentTokenGenerator generator = new InternalEnrollmentTokenGenerator(environment, sslService, client); PlainActionFuture future = new PlainActionFuture<>(); @@ -196,7 +196,7 @@ public void testRetryToGetNodesHttpInfo() { doAnswer(this::answerNullHttpInfo).doAnswer(this::answerNullHttpInfo) .doAnswer(this::answerWithInfo) .when(client) - .execute(eq(NodesInfoAction.INSTANCE), any(), any()); + .execute(eq(TransportNodesInfoAction.TYPE), any(), any()); final SSLService sslService = new TestsSSLService(environment); final InternalEnrollmentTokenGenerator generator = new InternalEnrollmentTokenGenerator(environment, sslService, client); PlainActionFuture future = new PlainActionFuture<>(); @@ -212,7 +212,7 @@ public void testRetryToGetNodesHttpInfo() { public void testRetryButFailToGetNodesHttpInfo() { // Answer with null HTTP info every time - doAnswer(this::answerNullHttpInfo).when(client).execute(eq(NodesInfoAction.INSTANCE), any(), any()); + doAnswer(this::answerNullHttpInfo).when(client).execute(eq(TransportNodesInfoAction.TYPE), any(), any()); final SSLService sslService = new TestsSSLService(environment); final InternalEnrollmentTokenGenerator generator = new InternalEnrollmentTokenGenerator(environment, sslService, client); PlainActionFuture future = new PlainActionFuture<>(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java index 99a411ab11a90..981cae74f0530 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java @@ -159,7 +159,7 @@ public TransportRequestHandler interceptHandler( ); DiscoveryNode remoteNode = remoteTransportService.getLocalDiscoNode(); remoteTransportService.registerRequestHandler( - RemoteClusterNodesAction.NAME, + RemoteClusterNodesAction.TYPE.name(), EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(List.of(remoteNode))) From 168a464f4176c58323aa762ef5f1d64ba26e8059 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 18 Oct 2023 21:48:37 +1100 Subject: [PATCH 14/23] More consistent logging messages for snapshot deletion (#101024) This PR makes sure that the start message ("deleting snapshots ...") is logged after the cluster state is processed and any failures before finishing updating the new repositoryData are always logged at warning level. Resolves: #99057 Resolves: #100481 --- docs/changelog/101024.yaml | 5 + .../snapshots/SnapshotsServiceIT.java | 124 ++++++++++++++++++ .../snapshots/SnapshotsService.java | 21 ++- 3 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/101024.yaml create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java diff --git a/docs/changelog/101024.yaml b/docs/changelog/101024.yaml new file mode 100644 index 0000000000000..edbd3d834526c --- /dev/null +++ b/docs/changelog/101024.yaml @@ -0,0 +1,5 @@ +pr: 101024 +summary: More consistent logging messages for snapshot deletion +area: Snapshot/Restore +type: bug +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java new file mode 100644 index 0000000000000..d37c956edbbf5 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.snapshots; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.snapshots.mockstore.MockRepository; +import org.elasticsearch.test.MockLogAppender; + +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +public class SnapshotsServiceIT extends AbstractSnapshotIntegTestCase { + + public void testDeletingSnapshotsIsLoggedAfterClusterStateIsProcessed() throws Exception { + createRepository("test-repo", "fs"); + createIndexWithRandomDocs("test-index", randomIntBetween(1, 42)); + createSnapshot("test-repo", "test-snapshot", List.of("test-index")); + + final MockLogAppender mockLogAppender = new MockLogAppender(); + + try { + mockLogAppender.start(); + Loggers.addAppender(LogManager.getLogger(SnapshotsService.class), mockLogAppender); + + mockLogAppender.addExpectation( + new MockLogAppender.UnseenEventExpectation( + "[does-not-exist]", + SnapshotsService.class.getName(), + Level.INFO, + "deleting snapshots [does-not-exist] from repository [test-repo]" + ) + ); + + mockLogAppender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "[deleting test-snapshot]", + SnapshotsService.class.getName(), + Level.INFO, + "deleting snapshots [test-snapshot] from repository [test-repo]" + ) + ); + + mockLogAppender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "[test-snapshot deleted]", + SnapshotsService.class.getName(), + Level.INFO, + "snapshots [test-snapshot/*] deleted" + ) + ); + + final SnapshotMissingException e = expectThrows( + SnapshotMissingException.class, + () -> startDeleteSnapshot("test-repo", "does-not-exist").actionGet() + ); + assertThat(e.getMessage(), containsString("[test-repo:does-not-exist] is missing")); + assertThat(startDeleteSnapshot("test-repo", "test-snapshot").actionGet().isAcknowledged(), is(true)); + + awaitNoMoreRunningOperations(); // ensure background file deletion is completed + mockLogAppender.assertAllExpectationsMatched(); + } finally { + Loggers.removeAppender(LogManager.getLogger(SnapshotsService.class), mockLogAppender); + mockLogAppender.stop(); + deleteRepository("test-repo"); + } + } + + public void testSnapshotDeletionFailureShouldBeLogged() throws Exception { + createRepository("test-repo", "mock"); + createIndexWithRandomDocs("test-index", randomIntBetween(1, 42)); + createSnapshot("test-repo", "test-snapshot", List.of("test-index")); + + final MockLogAppender mockLogAppender = new MockLogAppender(); + + try { + mockLogAppender.start(); + Loggers.addAppender(LogManager.getLogger(SnapshotsService.class), mockLogAppender); + + mockLogAppender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "[test-snapshot]", + SnapshotsService.class.getName(), + Level.WARN, + "failed to complete snapshot deletion for [test-snapshot] from repository [test-repo]" + ) + ); + + if (randomBoolean()) { + // Failure when listing root blobs + final MockRepository mockRepository = getRepositoryOnMaster("test-repo"); + mockRepository.setRandomControlIOExceptionRate(1.0); + final Exception e = expectThrows(Exception.class, () -> startDeleteSnapshot("test-repo", "test-snapshot").actionGet()); + assertThat(e.getCause().getMessage(), containsString("Random IOException")); + } else { + // Failure when finalizing on index-N file + final ActionFuture deleteFuture; + blockMasterFromFinalizingSnapshotOnIndexFile("test-repo"); + deleteFuture = startDeleteSnapshot("test-repo", "test-snapshot"); + waitForBlock(internalCluster().getMasterName(), "test-repo"); + unblockNode("test-repo", internalCluster().getMasterName()); + final Exception e = expectThrows(Exception.class, deleteFuture::actionGet); + assertThat(e.getCause().getMessage(), containsString("exception after block")); + } + + mockLogAppender.assertAllExpectationsMatched(); + } finally { + Loggers.removeAppender(LogManager.getLogger(SnapshotsService.class), mockLogAppender); + mockLogAppender.stop(); + deleteRepository("test-repo"); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 3f185f8c61088..7f502484f6372 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1887,9 +1887,6 @@ private void failSnapshotCompletionListeners(Snapshot snapshot, Exception e) { public void deleteSnapshots(final DeleteSnapshotRequest request, final ActionListener listener) { final String repositoryName = request.repository(); final String[] snapshotNames = request.snapshots(); - logger.info( - () -> format("deleting snapshots [%s] from repository [%s]", arrayToCommaDelimitedString(snapshotNames), repositoryName) - ); final Repository repository = repositoriesService.repository(repositoryName); executeConsistentStateUpdate(repository, repositoryData -> new ClusterStateUpdateTask(request.masterNodeTimeout()) { @@ -2064,6 +2061,10 @@ public void onFailure(Exception e) { @Override public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { + logger.info( + () -> format("deleting snapshots [%s] from repository [%s]", arrayToCommaDelimitedString(snapshotNames), repositoryName) + ); + if (completedNoCleanup.isEmpty() == false) { logger.info("snapshots {} aborted", completedNoCleanup); } @@ -2355,7 +2356,19 @@ public void onFailure(Exception e) { @Override public void onFailure(Exception e) { - logger.debug(() -> "failed to complete snapshot deletion [" + deleteEntry + "]", e); + logger.warn(() -> { + final StringBuilder sb = new StringBuilder("failed to complete snapshot deletion for ["); + Strings.collectionToDelimitedStringWithLimit( + deleteEntry.getSnapshots().stream().map(SnapshotId::getName).toList(), + ",", + "", + "", + 1024, + sb + ); + sb.append("] from repository [").append(deleteEntry.repository()).append("]"); + return sb; + }, e); submitUnbatchedTask( "remove snapshot deletion metadata after failed delete", new RemoveSnapshotDeletionAndContinueTask(deleteEntry, repositoryData) { From 8d89f913a19885bf890404593e93e5b237c57d42 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Wed, 18 Oct 2023 13:44:33 +0200 Subject: [PATCH 15/23] ESQL: Add warning header when default LIMIT is applied (#100894) Fixes https://github.com/elastic/elasticsearch/issues/100555 ESQL [by default](https://github.com/elastic/elasticsearch/pull/99816) adds an implicit `LIMIT 500` to queries that do not define a limit. Since this can be confusing for the end user, with this PR we also add a warning to the response, making this default clear and explicit. --- docs/reference/esql/esql-rest.asciidoc | 4 +- .../esql/multivalued-fields.asciidoc | 12 ++--- .../xpack/esql/EsqlSecurityIT.java | 5 ++ .../esql/qa/single_node/HeapAttackIT.java | 2 + .../rest-api-spec/test/100_bug_fix.yml | 2 + .../resources/rest-api-spec/test/10_basic.yml | 8 +++ .../resources/rest-api-spec/test/20_aggs.yml | 48 +++++++++++++++++ .../resources/rest-api-spec/test/30_types.yml | 53 +++++++++++++++++++ .../resources/rest-api-spec/test/40_tsdb.yml | 4 ++ .../test/40_unsupported_types.yml | 2 + .../test/45_non_tsdb_counter.yml | 8 +++ .../rest-api-spec/test/50_index_patterns.yml | 25 +++++++++ .../rest-api-spec/test/60_enrich.yml | 8 +++ .../rest-api-spec/test/61_enrich_ip.yml | 4 ++ .../rest-api-spec/test/70_locale.yml | 7 ++- .../resources/rest-api-spec/test/80_text.yml | 39 +++++++++++++- .../rest-api-spec/test/90_non_indexed.yml | 1 + .../xpack/esql/qa/rest/RestEsqlTestCase.java | 17 ++++-- .../xpack/esql/EsqlTestUtils.java | 7 +++ .../xpack/esql/analysis/Analyzer.java | 14 +++-- .../elasticsearch/xpack/esql/CsvTests.java | 6 ++- .../xpack/esql/analysis/AnalyzerTests.java | 7 +++ .../xpack/esql/analysis/VerifierTests.java | 6 +++ .../esql/io/stream/PlanStreamInputTests.java | 6 +++ .../LocalLogicalPlanOptimizerTests.java | 7 +++ .../LocalPhysicalPlanOptimizerTests.java | 6 +++ .../optimizer/LogicalPlanOptimizerTests.java | 5 ++ .../optimizer/PhysicalPlanOptimizerTests.java | 5 ++ .../xpack/esql/planner/FilterTests.java | 7 +++ .../esql/plugin/DataNodeRequestTests.java | 6 +++ .../esql/stats/PlanExecutorMetricsTests.java | 7 +++ .../esql/stats/VerifierMetricsTests.java | 8 +++ 32 files changed, 328 insertions(+), 18 deletions(-) diff --git a/docs/reference/esql/esql-rest.asciidoc b/docs/reference/esql/esql-rest.asciidoc index b2d418450ab3f..28ecfb7eea840 100644 --- a/docs/reference/esql/esql-rest.asciidoc +++ b/docs/reference/esql/esql-rest.asciidoc @@ -107,7 +107,7 @@ s|Description |smile |application/smile -|{wikipedia}/Smile_(data_interchange_format)[Smile] binary data format similar +|{wikipedia}/Smile_(data_interchange_format)[Smile] binary data format similar to CBOR |=== @@ -220,6 +220,7 @@ POST /_query | WHERE page_count > 300 AND author == "Frank Herbert" | STATS count = COUNT(*) by year | WHERE count > 0 + | LIMIT 5 """ } ---- @@ -239,6 +240,7 @@ POST /_query | WHERE page_count > ? AND author == ? | STATS count = COUNT(*) by year | WHERE count > ? + | LIMIT 5 """, "params": [300, "Frank Herbert", 0] } diff --git a/docs/reference/esql/multivalued-fields.asciidoc b/docs/reference/esql/multivalued-fields.asciidoc index 3c990a1cf74a7..5e48eb4ef8af8 100644 --- a/docs/reference/esql/multivalued-fields.asciidoc +++ b/docs/reference/esql/multivalued-fields.asciidoc @@ -17,7 +17,7 @@ POST /mv/_bulk?refresh POST /_query { - "query": "FROM mv" + "query": "FROM mv | LIMIT 2" } ---- @@ -65,7 +65,7 @@ POST /mv/_bulk?refresh POST /_query { - "query": "FROM mv" + "query": "FROM mv | LIMIT 2" } ---- @@ -106,7 +106,7 @@ POST /mv/_bulk?refresh POST /_query { - "query": "FROM mv" + "query": "FROM mv | LIMIT 2" } ---- @@ -148,7 +148,7 @@ POST /mv/_bulk?refresh POST /_query { - "query": "FROM mv | EVAL b=TO_STRING(b)" + "query": "FROM mv | EVAL b=TO_STRING(b) | LIMIT 2" } ---- @@ -183,7 +183,7 @@ POST /mv/_bulk?refresh POST /_query { - "query": "FROM mv | EVAL b + 2, a + b" + "query": "FROM mv | EVAL b + 2, a + b | LIMIT 4" } ---- @@ -217,7 +217,7 @@ Work around this limitation by converting the field to single value with one of: ---- POST /_query { - "query": "FROM mv | EVAL b=MV_MIN(b) | EVAL b + 2, a + b" + "query": "FROM mv | EVAL b=MV_MIN(b) | EVAL b + 2, a + b | LIMIT 4" } ---- // TEST[continued] diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java index 6897994bae3ac..e067cf271478f 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.hamcrest.Matchers.equalTo; @@ -248,6 +249,10 @@ private void removeEnrichPolicy() throws Exception { } private Response runESQLCommand(String user, String command) throws IOException { + if (command.toLowerCase(Locale.ROOT).contains("limit") == false) { + // add a (high) limit to avoid warnings on default limit + command += " | limit 10000000"; + } Settings pragmas = Settings.EMPTY; if (Build.current().isSnapshot()) { Settings.Builder settings = Settings.builder(); diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java index 2f8b64865fa4b..eed96a007a1b9 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentHelper; @@ -279,6 +280,7 @@ private Response query(String query, String filterPath) throws IOException { request.setOptions( RequestOptions.DEFAULT.toBuilder() .setRequestConfig(RequestConfig.custom().setSocketTimeout(Math.toIntExact(TimeValue.timeValueMinutes(5).millis())).build()) + .setWarningsHandler(WarningsHandler.PERMISSIVE) ); return client().performRequest(request); } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml index 3465bf1b17b54..720914b579f36 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml @@ -18,6 +18,7 @@ setup: warnings: - "Line 1:37: evaluation of [to_ip(coalesce(ip1.keyword, \"255.255.255.255\"))] failed, treating result as null. Only first 20 failures recorded." - "Line 1:37: java.lang.IllegalArgumentException: '127.0' is not an IP string literal." + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'FROM test | sort emp_no | eval ip = to_ip(coalesce(ip1.keyword, "255.255.255.255")) | keep emp_no, ip' @@ -36,6 +37,7 @@ setup: warnings: - "Line 1:98: evaluation of [to_ip(x2)] failed, treating result as null. Only first 20 failures recorded." - "Line 1:98: java.lang.IllegalArgumentException: '127.00.1' is not an IP string literal." + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'FROM test | sort emp_no | eval x1 = concat(ip1, ip2), x2 = coalesce(x1, "255.255.255.255"), x3 = to_ip(x2) | keep emp_no, x*' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml index fdd5cf2566961..a3b2de27bcb5b 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -1,5 +1,7 @@ --- setup: + - skip: + features: allowed_warnings_regex - do: indices.create: index: test @@ -109,6 +111,8 @@ setup: --- "Test From": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test' @@ -266,6 +270,8 @@ setup: --- "Test Input Params": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'row a = ? | eval b = ?, c = 1 + ?' @@ -283,6 +289,8 @@ setup: - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | where color == ? and count == ? and time == ? | keep data, count, color' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml index 6e8c0eb120ddd..1087bd5ce06eb 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml @@ -1,5 +1,7 @@ --- setup: + - skip: + features: warnings - do: indices.create: index: test @@ -109,6 +111,8 @@ setup: --- "Test From": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -130,6 +134,8 @@ setup: --- "Test simple grouping avg": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | where color == "red" | stats avg(data) by color' @@ -144,6 +150,8 @@ setup: --- "Test From Stats Avg": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats avg(count)' @@ -156,6 +164,8 @@ setup: --- "Test From Stats Avg With Alias": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats f1 = avg(count)' @@ -168,6 +178,8 @@ setup: --- "Test From Stats Count": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats count(data)' @@ -180,6 +192,8 @@ setup: --- "Test From Stats Count With Alias": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats dataCount = count(data)' @@ -192,6 +206,8 @@ setup: --- "Test From Stats Min": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats min(count)' @@ -204,6 +220,8 @@ setup: --- "Test From Stats Min With Alias": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats minCount=min(count)' @@ -216,6 +234,8 @@ setup: --- "Test From Stats Max": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats max(count)' @@ -228,6 +248,8 @@ setup: --- "Test From Stats Max With Alias": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats maxCount=max(count)' @@ -255,6 +277,8 @@ setup: --- "Test Median On Long": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats med=median(count)' @@ -267,6 +291,8 @@ setup: --- "Test Median On Double": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats med=median(count_d)' @@ -279,6 +305,8 @@ setup: --- "Test Grouping Median On Long": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats med=median(count) by color | sort med' @@ -294,6 +322,8 @@ setup: --- "Test Grouping Median On Double": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats med=median(count_d) by color | sort med' @@ -309,6 +339,8 @@ setup: --- "Test Median Absolute Deviation On Long": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats med=median_absolute_deviation(count)' @@ -321,6 +353,8 @@ setup: --- "Test Median Absolute Deviation On Double": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats med=median_absolute_deviation(count_d)' @@ -333,6 +367,8 @@ setup: --- "Test Grouping Median Absolute Deviation On Long": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats med=median_absolute_deviation(count) by color | sort color' @@ -348,6 +384,8 @@ setup: --- "Test Grouping Median Absolute Deviation On Double": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats med=median_absolute_deviation(count_d) by color | sort color' @@ -363,6 +401,8 @@ setup: --- "Test From Stats Eval": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats avg_count = avg(count) | eval x = avg_count + 7' @@ -374,6 +414,8 @@ setup: --- "Test Stats Where": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats x = avg(count) | where x > 100' @@ -396,6 +438,8 @@ setup: --- "Test Eval Row With Null": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'row a = 1, b = 2, c = null | eval z = c + b + a' @@ -419,6 +463,8 @@ setup: --- "Test Eval With Null And Count": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | eval nullsum = count_d + null | stats count(nullsum)' @@ -433,6 +479,8 @@ setup: --- "Test Eval With Multiple Expressions": - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'row l=1, d=1.0, ln=1 + null, dn=1.0 + null | stats sum(l), sum(d), sum(ln), sum(dn)' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/30_types.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/30_types.yml index 86a4c58147fca..886bb6dc60aca 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/30_types.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/30_types.yml @@ -1,5 +1,12 @@ +--- +setup: + - skip: + features: warnings + --- constant_keyword: + - skip: + features: warnings - do: indices.create: index: test @@ -21,6 +28,8 @@ constant_keyword: - { "color": "red" } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -33,6 +42,8 @@ constant_keyword: - match: { values.0.1: wow such constant } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | eval l=length(kind) | keep l' @@ -62,6 +73,8 @@ multivalued keyword: - { "card": ["jack", "of", "diamonds"] } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -91,6 +104,8 @@ keyword no doc_values: - { "card": ["jack", "of", "diamonds"] } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -119,6 +134,8 @@ wildcard: - { "card": "jack of diamonds" } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -128,6 +145,8 @@ wildcard: - match: {values.0.0: jack of diamonds} - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | eval l=length(card) | keep l' @@ -166,6 +185,8 @@ numbers: - { i: 123, l: -1234567891011121131, d: 1.234567891234568, mv_i: [123456, -123456], mv_l: [1234567891011121131, -1234567891011121131], mv_d: [1.234567891234568, -1.234567891234568] } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -215,6 +236,8 @@ small_numbers: - { b: 1, s: 1245, hf: 12.01, f: 112.0 } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -233,6 +256,8 @@ small_numbers: - match: {values.0.3: 1245} - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | eval sum_d = b + f + hf + s, sum_i = b + s | keep sum_d, sum_i' @@ -245,6 +270,8 @@ small_numbers: - match: {values.0.1: 1246} - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | eval r_f = round(f), r_hf = round(hf) | keep r_f, r_hf' @@ -279,6 +306,8 @@ scaled_float: - { f: 112.01, d: 1.0 } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -291,6 +320,8 @@ scaled_float: - match: {values.0.1: 112.01} - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | eval sum = d + f | keep sum' @@ -319,6 +350,8 @@ multivalued boolean: - { "booleans": [ true, false, false, false ] } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -349,6 +382,8 @@ ip: - { "ip": "127.0.0.1", "keyword": "127.0.0.2" } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -361,6 +396,8 @@ ip: - match: { values.0.1: "127.0.0.2" } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | where keyword == "127.0.0.2" | rename ip as IP | drop keyword' @@ -415,6 +452,8 @@ alias: - { "foo": "def", "level1": {"level2": 50}, "some_long": 15, "some_date": "2015-01-01T12:00:00.000Z" } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | keep foo, bar, level1.level2, level2_alias, some_long, some_long_alias, some_long_alias2, some_date, some_date_alias | sort level2_alias' @@ -457,6 +496,8 @@ alias: - match: { values.1.8: 2015-01-01T12:00:00.000Z } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | where bar == "abc" | keep foo, bar, level1.level2, level2_alias' @@ -475,6 +516,8 @@ alias: - match: { values.0.3: 10 } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | where level2_alias == 10 | keep foo, bar, level1.level2, level2_alias' @@ -493,12 +536,16 @@ alias: - match: { values.0.3: 10 } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | where level2_alias == 20' - length: { values: 0 } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test | stats x = max(level2_alias)' @@ -527,6 +574,8 @@ version: - { "version": [ "1.2.3", "4.5.6-SNOOPY" ] } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' @@ -555,6 +604,8 @@ id: - { "kw": "keyword1" } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test [metadata _id] | keep _id, kw' @@ -584,6 +635,8 @@ unsigned_long: - { "number": [ "1", "9223372036854775808", "0", "18446744073709551615" ] } - do: + warnings: + - "No limit defined, adding default limit of [500]" esql.query: body: query: 'from test' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml index 895a1718b2cbc..33697a789cc26 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml @@ -108,6 +108,7 @@ load everything: - do: allowed_warnings_regex: - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test' @@ -133,6 +134,7 @@ load a document: - do: allowed_warnings_regex: - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | where @timestamp == "2021-04-28T18:50:23.142Z"' @@ -160,6 +162,7 @@ from doc with aggregate_metric_double: - do: allowed_warnings_regex: - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test2' @@ -189,6 +192,7 @@ from index pattern unsupported counter: - do: allowed_warnings_regex: - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'FROM test*' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml index ad0c7b516fde1..c06456f7f127d 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml @@ -104,6 +104,7 @@ unsupported: - do: allowed_warnings_regex: - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test' @@ -269,6 +270,7 @@ unsupported with sort: - do: allowed_warnings_regex: - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | sort some_doc.bar' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml index a4344946aea0d..beb7200f01230 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml @@ -1,4 +1,6 @@ setup: + - skip: + features: allowed_warnings_regex - do: indices.create: index: test @@ -57,6 +59,8 @@ setup: --- load everything: - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test' @@ -80,6 +84,8 @@ load everything: --- load a document: - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | where @timestamp == "2021-04-28T18:50:23.142Z"' @@ -97,6 +103,8 @@ load a document: --- filter on counter: - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | where k8s.pod.network.tx == 1434577921' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml index ff327b2592c88..5fceeee2f6e57 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml @@ -1,3 +1,7 @@ +setup: + - skip: + features: allowed_warnings_regex + --- disjoint_mappings: - do: @@ -40,6 +44,8 @@ disjoint_mappings: - { "message2": 2 } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1' @@ -82,6 +88,8 @@ disjoint_mappings: - match: { values.0.1: null } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1, message2' @@ -147,6 +155,8 @@ disjoint_mappings: - match: { values.1.1: null } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1 nulls first, message2' @@ -165,6 +175,8 @@ disjoint_mappings: - match: { values.3.1: null } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1 nulls first, message2 nulls first' @@ -183,6 +195,8 @@ disjoint_mappings: - match: { values.3.1: null } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1 desc nulls first, message2 desc nulls first' @@ -201,6 +215,8 @@ disjoint_mappings: - match: { values.3.1: null } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 | where message1 == "foo1" | keep message1, message2 | sort message1, message2' @@ -213,6 +229,8 @@ disjoint_mappings: - match: { values.0.1: null } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 | where message1 == "foo1" or message2 == 2 | keep message1, message2 | sort message1, message2' @@ -227,6 +245,8 @@ disjoint_mappings: - match: { values.1.1: 2 } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 | stats x = max(message2)' @@ -236,6 +256,8 @@ disjoint_mappings: - match: { values.0.0: 2 } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 | sort message1, message2 | eval x = message1, y = message2 + 1 | keep message1, message2, x, y' @@ -311,6 +333,7 @@ same_name_different_type: - do: allowed_warnings_regex: - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 ' @@ -360,6 +383,8 @@ same_name_different_type_same_family: - { "message": "foo4" } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test1,test2 | sort message | keep message' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_enrich.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_enrich.yml index 701bd63c3d35d..84d8682508733 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_enrich.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_enrich.yml @@ -1,5 +1,7 @@ --- setup: + - skip: + features: allowed_warnings_regex - do: indices.create: index: cities @@ -65,6 +67,8 @@ setup: --- "Basic": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | enrich cities_policy on city_id | keep name, city, country | sort name' @@ -84,6 +88,8 @@ setup: - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | keep name, city_id | enrich cities_policy on city_id with country | sort name' @@ -103,6 +109,8 @@ setup: - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | keep name, city_id | enrich cities_policy on city_id with country_name = country | sort name' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/61_enrich_ip.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/61_enrich_ip.yml index 1937b425d6a07..bd89af2fd3f79 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/61_enrich_ip.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/61_enrich_ip.yml @@ -1,5 +1,7 @@ --- setup: + - skip: + features: allowed_warnings_regex - do: indices.create: index: networks @@ -73,6 +75,8 @@ setup: "IP strings": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'FROM events | eval ip_str = to_string(ip) | ENRICH networks-policy ON ip_str | sort @timestamp | KEEP ip, name, department, message' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/70_locale.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/70_locale.yml index 91ff5ddc7cbe9..a77e0569668de 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/70_locale.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/70_locale.yml @@ -1,6 +1,7 @@ --- setup: - + - skip: + features: allowed_warnings_regex - do: indices.create: index: events @@ -24,6 +25,8 @@ setup: --- "Date format with default locale": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' @@ -43,6 +46,8 @@ setup: --- "Date format with Italian locale": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml index d05bc372326f8..d6d20fa0a0aee 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml @@ -1,6 +1,7 @@ --- setup: - + - skip: + features: allowed_warnings_regex - do: indices.create: index: test @@ -32,6 +33,8 @@ setup: --- "all": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | sort emp_no' @@ -55,6 +58,8 @@ setup: --- "filter by text": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | where tag == "baz" | keep emp_no, name, job, tag' @@ -74,6 +79,8 @@ setup: --- "like by text": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | where tag LIKE "*az" | keep emp_no, name, job, tag' @@ -93,6 +100,8 @@ setup: --- "rlike by text": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | where tag RLIKE ".*az" | keep emp_no, name, job, tag' @@ -112,6 +121,8 @@ setup: --- "eval and filter text": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | eval x = tag | where x == "baz" | keep emp_no, name, job, x' @@ -131,6 +142,8 @@ setup: --- "filter on text multi-field": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | where job == "IT Director" | keep emp_no, name, job, tag' @@ -150,6 +163,8 @@ setup: --- "like by multi-field text": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | where job LIKE "*Specialist" | keep emp_no, name, job, tag' @@ -169,6 +184,8 @@ setup: --- "rlike by multi-field text": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | where job RLIKE ".*Specialist" | keep emp_no, name, job, tag' @@ -189,6 +206,8 @@ setup: --- "sort by text": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | sort tag | keep emp_no, name, job, tag' @@ -210,6 +229,8 @@ setup: --- "sort by text multi-field": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | sort job | keep emp_no, name, job, tag' @@ -230,6 +251,8 @@ setup: --- "sort by text multi-field desc": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | sort job desc | keep emp_no, name, job, tag' @@ -251,6 +274,8 @@ setup: --- "text in functions": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | sort name | eval description = concat(name, " - ", job) | keep description' @@ -266,6 +291,8 @@ setup: --- "stats text with raw": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | stats jobs = count(job) | keep jobs' @@ -280,6 +307,8 @@ setup: --- "stats text no raw": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | stats tags = count(tag) | keep tags' @@ -294,6 +323,8 @@ setup: --- "stats by text with raw": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | stats names = count(name) by job | keep names' @@ -309,6 +340,8 @@ setup: --- "stats by text no raw": - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | stats names = count(name) by tag | keep names' @@ -351,6 +384,8 @@ setup: - { "emp_no": 20, "name": "John", "job": "Payroll Specialist" } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test2 | sort emp_no | keep job' @@ -392,6 +427,8 @@ setup: - { "emp_no": 20, "name": "John", "job": "Payroll Specialist" } - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test2 | sort emp_no | keep job' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/90_non_indexed.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/90_non_indexed.yml index 53dc5bab6df46..a673fb7a5b88d 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/90_non_indexed.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/90_non_indexed.yml @@ -89,6 +89,7 @@ unsupported: - do: allowed_warnings_regex: - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test' diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 14e645f37a659..4d9a5a259ed03 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import static java.util.Collections.emptySet; import static org.elasticsearch.test.ListMatcher.matchesList; @@ -209,27 +210,27 @@ public void testColumnarMode() throws IOException { public void testTextMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | limit 100").build(); assertEquals(expectedTextBody("txt", count, null), runEsqlAsTextWithFormat(builder, "txt", null)); } public void testCSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | limit 100").build(); assertEquals(expectedTextBody("csv", count, '|'), runEsqlAsTextWithFormat(builder, "csv", '|')); } public void testTSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | limit 100").build(); assertEquals(expectedTextBody("tsv", count, null), runEsqlAsTextWithFormat(builder, "tsv", null)); } public void testCSVNoHeaderMode() throws IOException { bulkLoadTestData(1); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | limit 100").build(); Request request = prepareRequest(); String mediaType = attachBody(builder, request); RequestOptions.Builder options = request.getOptions().toBuilder(); @@ -495,10 +496,16 @@ private static String attachBody(RequestObjectBuilder requestObject, Request req private static HttpEntity performRequest(Request request, List allowedWarnings) throws IOException { Response response = client().performRequest(request); assertEquals(200, response.getStatusLine().getStatusCode()); - assertMap(response.getWarnings(), matchesList(allowedWarnings)); + List warnings = new ArrayList<>(response.getWarnings()); + warnings.removeAll(mutedWarnings()); + assertMap(warnings, matchesList(allowedWarnings)); return response.getEntity(); } + private static Set mutedWarnings() { + return Set.of("No limit defined, adding default limit of [500]"); + } + private static void bulkLoadTestData(int count) throws IOException { Request request = new Request("PUT", "/" + testIndexName()); request.setJsonEntity(""" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index d67ff99cff054..d35f7898d937f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -166,4 +166,11 @@ public static List> getValuesList(Iterator> values }); return valuesList; } + + public static List withDefaultLimitWarning(List warnings) { + List result = warnings == null ? new ArrayList<>() : new ArrayList<>(warnings); + result.add("No limit defined, adding default limit of [500]"); + return result; + } + } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 818d58e91a91c..4db49c4d76c89 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.analysis; +import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.mapper.DateFieldMapper; @@ -614,9 +615,16 @@ private static class AddImplicitLimit extends ParameterizedRule limits = logicalPlan.collectFirstChildren(Limit.class::isInstance); - var limit = limits.isEmpty() == false - ? context.configuration().resultTruncationMaxSize() // user provided a limit: cap result entries to the max - : context.configuration().resultTruncationDefaultSize(); // user provided no limit: cap to a default + int limit; + if (limits.isEmpty()) { + HeaderWarning.addWarning( + "No limit defined, adding default limit of [{}]", + context.configuration().resultTruncationDefaultSize() + ); + limit = context.configuration().resultTruncationDefaultSize(); // user provided no limit: cap to a default + } else { + limit = context.configuration().resultTruncationMaxSize(); // user provided a limit: cap result entries to the max + } return new Limit(Source.EMPTY, new Literal(Source.EMPTY, limit, DataTypes.INTEGER), logicalPlan); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 24d7a3abb6960..67f5d12756e71 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -422,7 +422,11 @@ private static void opportunisticallyAssertPlanSerialization(PhysicalPlan... pla private void assertWarnings(List warnings) { List normalized = new ArrayList<>(warnings.size()); for (String w : warnings) { - normalized.add(HeaderWarning.extractWarningValueFromWarningHeader(w, false)); + String normW = HeaderWarning.extractWarningValueFromWarningHeader(w, false); + if (normW.startsWith("No limit defined, adding default limit of [") == false) { + // too many tests do not have a LIMIT, we'll test this warning separately + normalized.add(normW); + } } assertMap(normalized, matchesList(testCase.expectedWarnings)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 6cbc1f93bcdf1..0349a1874415b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -39,6 +39,7 @@ import java.util.stream.IntStream; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzer; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; @@ -1375,4 +1376,10 @@ private void assertProjectionWithMapping(String query, String mapping, String... var limit = as(plan, Limit.class); assertThat(Expressions.names(limit.output()), contains(names)); } + + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } + } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index e863288386eed..2596340e7f206 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.List; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.hamcrest.Matchers.containsString; @@ -340,4 +341,9 @@ private String error(String query, Analyzer analyzer, Object... params) { int index = message.indexOf(pattern); return message.substring(index + pattern.length()); } + + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java index f5bc458b19a57..4796b31148e27 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java @@ -25,6 +25,7 @@ import java.util.function.Function; import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.SerializationTestUtils.serializeDeserialize; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze; import static org.hamcrest.Matchers.equalTo; @@ -145,4 +146,9 @@ public void testSourceSerialization() { assertThat(sources.apply(planIn), equalTo(sources.apply(planOut))); } } + + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index 8a0e4af35036c..ae9d7b7d1b5f0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.xpack.ql.type.EsField; import org.junit.BeforeClass; +import java.util.List; import java.util.Map; import static org.elasticsearch.xpack.esql.EsqlTestUtils.L; @@ -38,6 +39,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForMissingField; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -282,4 +284,9 @@ private LogicalPlan localPlan(LogicalPlan plan, SearchStats searchStats) { // System.out.println(localPlan); return localPlan; } + + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index a148a0e396311..511a7ee08b5e1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -57,6 +57,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.plan.physical.AggregateExec.Mode.FINAL; import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType; import static org.hamcrest.Matchers.contains; @@ -407,4 +408,9 @@ private PhysicalPlan physicalPlan(String query) { var physical = mapper.map(logical); return physical; } + + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 848a7bc12aeef..a22bb3b91ff0b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -90,6 +90,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptySource; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.localSource; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.ql.TestUtils.relation; import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; @@ -1968,4 +1969,8 @@ public static RLike rlike(Expression left, String exp) { return new RLike(EMPTY, left, new RLikePattern(exp)); } + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 2e01d33139f23..f0c3b9d541b16 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -89,6 +89,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForMissingField; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; import static org.elasticsearch.xpack.esql.plan.physical.AggregateExec.Mode.FINAL; import static org.elasticsearch.xpack.ql.expression.Expressions.name; @@ -1963,4 +1964,8 @@ private QueryBuilder sv(QueryBuilder builder, String fieldName) { return sv.next(); } + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java index 25ebe9fa94a38..db3d2419ee2ee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java @@ -36,12 +36,14 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.util.List; import java.util.Map; import static java.util.Arrays.asList; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; import static org.elasticsearch.xpack.ql.util.Queries.Clause.FILTER; import static org.elasticsearch.xpack.ql.util.Queries.Clause.MUST; @@ -291,4 +293,9 @@ private QueryBuilder restFilterQuery(String field) { private QueryBuilder filterQueryForTransportNodes(PhysicalPlan plan) { return PlannerUtils.detectFilter(plan, AT_TIMESTAMP); } + + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java index c59230e06d48a..cceb3a2ab835b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java @@ -41,6 +41,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyPolicyResolution; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; public class DataNodeRequestTests extends AbstractWireSerializingTestCase { @@ -182,4 +183,9 @@ static PhysicalPlan mapAndMaybeOptimize(LogicalPlan logicalPlan) { } return physical; } + + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java index 95a6389af7fc5..1947249086568 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java @@ -27,11 +27,13 @@ import org.mockito.stubbing.Answer; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -117,4 +119,9 @@ private Map> fields(String[] indices) { fields.put(barField.getName(), singletonMap(barField.getName(), barField)); return fields; } + + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java index 08dd7174c8eaa..eb91a540e6e82 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java @@ -14,6 +14,9 @@ import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.ql.index.IndexResolution; +import java.util.List; + +import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzer; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.DISSECT; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.EVAL; @@ -196,4 +199,9 @@ private Counters esql(String esql, Verifier v) { return metrics == null ? null : metrics.stats(); } + + @Override + protected List filteredWarnings() { + return withDefaultLimitWarning(super.filteredWarnings()); + } } From a545fe2f141be7dc38c6df21f0087cffa7fc87ff Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 18 Oct 2023 12:54:48 +0100 Subject: [PATCH 16/23] [ML] Minor tidyups in ML serverless extension (#101039) Followup to #99129 and #100956 --- .../xpack/ml/DefaultMachineLearningExtension.java | 1 + .../elasticsearch/xpack/ml/MachineLearningExtension.java | 9 ++------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java index cddbe721d3d56..fa94bf96c1167 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java @@ -56,6 +56,7 @@ public String[] getAnalyticsDestIndexAllowedSettings() { return ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS; } + @Override public AbstractNodeAvailabilityZoneMapper getNodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings) { return new NodeRealAvailabilityZoneMapper(settings, clusterSettings); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java index b0e65d524036c..552344b4ef10e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.ml.autoscaling.AbstractNodeAvailabilityZoneMapper; -import org.elasticsearch.xpack.ml.autoscaling.NodeRealAvailabilityZoneMapper; public interface MachineLearningExtension { @@ -26,11 +25,7 @@ default void configure(Settings settings) {} boolean isNlpEnabled(); - default String[] getAnalyticsDestIndexAllowedSettings() { - return DefaultMachineLearningExtension.ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS; - } + String[] getAnalyticsDestIndexAllowedSettings(); - default AbstractNodeAvailabilityZoneMapper getNodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings) { - return new NodeRealAvailabilityZoneMapper(settings, clusterSettings); - } + AbstractNodeAvailabilityZoneMapper getNodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings); } From e3cb876babcebfa20ce6c52a9f8d127ab3ed3a7c Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 18 Oct 2023 14:36:32 +0200 Subject: [PATCH 17/23] Adjust DateHistogram's bucket accounting to be iteratively (#101012) Adjust DateHistogram's consumeBucketsAndMaybeBreak to be iteratively during reduce instead accounting all buckets at the end of the reduce. In case of many non-empty buckets accounting the number of buckets at the end of the reduce may be too late. Elasticsearch may already have failed with an OOME. This change changes the accounting to happen iteratively during the reduce for non-empty bucket. Note that for empty buckets accounting of the number of buckets already happens iteratively. --- docs/changelog/101012.yaml | 5 +++++ .../histogram/InternalDateHistogram.java | 18 +++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 docs/changelog/101012.yaml diff --git a/docs/changelog/101012.yaml b/docs/changelog/101012.yaml new file mode 100644 index 0000000000000..1d5f62bdddba7 --- /dev/null +++ b/docs/changelog/101012.yaml @@ -0,0 +1,5 @@ +pr: 101012 +summary: Adjust `DateHistogram's` bucket accounting to be iteratively +area: Aggregations +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java index 82716dca7311c..4eaec7034b7f4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java @@ -310,6 +310,7 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent reducedBuckets = new ArrayList<>(); if (pq.size() > 0) { // list of buckets coming from different shards that have the same key @@ -323,6 +324,10 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent= minDocCount || reduceContext.isFinalReduce() == false) { + if (consumeBucketCount++ >= REPORT_EMPTY_EVERY) { + reduceContext.consumeBucketsAndMaybeBreak(consumeBucketCount); + consumeBucketCount = 0; + } reducedBuckets.add(reduced); } currentBuckets.clear(); @@ -344,10 +349,14 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent= minDocCount || reduceContext.isFinalReduce() == false) { reducedBuckets.add(reduced); + if (consumeBucketCount++ >= REPORT_EMPTY_EVERY) { + reduceContext.consumeBucketsAndMaybeBreak(consumeBucketCount); + consumeBucketCount = 0; + } } } } - + reduceContext.consumeBucketsAndMaybeBreak(consumeBucketCount); return reducedBuckets; } @@ -387,7 +396,7 @@ private void addEmptyBuckets(List list, AggregationReduceContext reduceC * consumeBucketsAndMaybeBreak. */ class Counter implements LongConsumer { - private int size = list.size(); + private int size; @Override public void accept(long key) { @@ -490,11 +499,9 @@ private void iterateEmptyBuckets(List list, ListIterator iter, L @Override public InternalAggregation reduce(List aggregations, AggregationReduceContext reduceContext) { List reducedBuckets = reduceBuckets(aggregations, reduceContext); - boolean alreadyAccountedForBuckets = false; if (reduceContext.isFinalReduce()) { if (minDocCount == 0) { addEmptyBuckets(reducedBuckets, reduceContext); - alreadyAccountedForBuckets = true; } if (InternalOrder.isKeyDesc(order)) { // we just need to reverse here... @@ -508,9 +515,6 @@ public InternalAggregation reduce(List aggregations, Aggreg CollectionUtil.introSort(reducedBuckets, order.comparator()); } } - if (false == alreadyAccountedForBuckets) { - reduceContext.consumeBucketsAndMaybeBreak(reducedBuckets.size()); - } return new InternalDateHistogram( getName(), reducedBuckets, From 3ce905b75432e36648ae013a3580b1f12537cae9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 18 Oct 2023 13:56:40 +0100 Subject: [PATCH 18/23] Remove more unnecessary ActionType subclasses (#101053) Relates #97721 --- .../elasticsearch/plugin/noop/NoopPlugin.java | 13 ++-- .../noop/action/bulk/NoopBulkAction.java | 21 ------ .../action/bulk/TransportNoopBulkAction.java | 3 +- .../noop/action/search/NoopSearchAction.java | 20 ------ .../action/search/RestNoopSearchAction.java | 3 +- .../search/TransportNoopSearchAction.java | 3 +- .../DeleteDataStreamLifecycleAction.java | 12 ++-- .../ExplainDataStreamLifecycleAction.java | 13 ++-- .../action/GetDataStreamLifecycleAction.java | 12 ++-- .../action/PutDataStreamLifecycleAction.java | 12 ++-- ...nsportDeleteDataStreamLifecycleAction.java | 2 +- ...sportExplainDataStreamLifecycleAction.java | 2 +- ...TransportGetDataStreamLifecycleAction.java | 2 +- ...TransportPutDataStreamLifecycleAction.java | 2 +- .../ingest/common/GrokProcessorGetAction.java | 14 ++-- .../stats/GeoIpDownloaderStatsAction.java | 11 ++-- .../GeoIpDownloaderStatsTransportAction.java | 2 +- .../mustache/MultiSearchTemplateIT.java | 5 +- .../script/mustache/SearchTemplateIT.java | 8 +-- .../mustache/MultiSearchTemplateAction.java | 21 ------ .../script/mustache/MustachePlugin.java | 14 +++- .../RestMultiSearchTemplateAction.java | 2 +- .../RestRenderSearchTemplateAction.java | 2 +- .../mustache/RestSearchTemplateAction.java | 2 +- .../script/mustache/SearchTemplateAction.java | 21 ------ .../SearchTemplateRequestBuilder.java | 5 +- .../TransportMultiSearchTemplateAction.java | 2 +- .../TransportSearchTemplateAction.java | 8 ++- .../action/PainlessContextAction.java | 17 +++-- .../action/PainlessExecuteAction.java | 11 ++-- .../index/rankeval/RankEvalRequestIT.java | 44 ++++++------- .../index/rankeval/RankEvalAction.java | 24 ------- .../index/rankeval/RankEvalPlugin.java | 5 +- .../index/rankeval/RestRankEvalAction.java | 2 +- .../rankeval/TransportRankEvalAction.java | 2 +- .../documentation/ReindexDocumentationIT.java | 3 +- .../elasticsearch/reindex/ReindexPlugin.java | 9 ++- .../reindex/RestRethrottleAction.java | 2 +- .../reindex/RethrottleAction.java | 21 ------ .../reindex/TransportRethrottleAction.java | 4 +- .../reindex/ReindexTestCase.java | 2 +- .../TransportRethrottleActionTests.java | 2 +- .../elasticsearch/rest/root/MainAction.java | 22 ------- .../rest/root/MainRestPlugin.java | 6 +- .../rest/root/RestMainAction.java | 2 +- .../rest/root/TransportMainAction.java | 2 +- .../node/tasks/TaskStorageRetryIT.java | 2 +- .../admin/cluster/node/tasks/TasksIT.java | 64 +++++++++---------- .../org/elasticsearch/action/ActionType.java | 4 +- .../action/ActionModuleTests.java | 7 +- .../org/elasticsearch/action/ActionTests.java | 11 +--- .../cluster/node/tasks/TestTaskPlugin.java | 34 +++------- .../InternalOrPrivateSettingsPlugin.java | 14 ++-- .../persistent/TestPersistentTasksPlugin.java | 25 +++----- .../test/seektracker/SeekTrackerPluginIT.java | 2 +- .../test/seektracker/RestSeekStatsAction.java | 2 +- .../test/seektracker/SeekStatsAction.java | 22 ------- .../test/seektracker/SeekTrackerPlugin.java | 5 +- .../seektracker/TransportSeekStatsAction.java | 2 +- .../authz/store/ReservedRolesStoreTests.java | 4 +- 60 files changed, 223 insertions(+), 392 deletions(-) delete mode 100644 client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkAction.java delete mode 100644 client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchAction.java delete mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java delete mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java delete mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java delete mode 100644 modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleAction.java delete mode 100644 modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainAction.java delete mode 100644 test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekStatsAction.java diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java index c5699514c12aa..571efd88aafec 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java @@ -9,16 +9,17 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.plugin.noop.action.bulk.NoopBulkAction; import org.elasticsearch.plugin.noop.action.bulk.RestNoopBulkAction; import org.elasticsearch.plugin.noop.action.bulk.TransportNoopBulkAction; -import org.elasticsearch.plugin.noop.action.search.NoopSearchAction; import org.elasticsearch.plugin.noop.action.search.RestNoopSearchAction; import org.elasticsearch.plugin.noop.action.search.TransportNoopSearchAction; import org.elasticsearch.plugins.ActionPlugin; @@ -31,11 +32,15 @@ import java.util.function.Supplier; public class NoopPlugin extends Plugin implements ActionPlugin { + + public static final ActionType NOOP_SEARCH_ACTION = new ActionType<>("mock:data/read/search", SearchResponse::new); + public static final ActionType NOOP_BULK_ACTION = new ActionType<>("mock:data/write/bulk", BulkResponse::new); + @Override public List> getActions() { return Arrays.asList( - new ActionHandler<>(NoopBulkAction.INSTANCE, TransportNoopBulkAction.class), - new ActionHandler<>(NoopSearchAction.INSTANCE, TransportNoopSearchAction.class) + new ActionHandler<>(NOOP_BULK_ACTION, TransportNoopBulkAction.class), + new ActionHandler<>(NOOP_SEARCH_ACTION, TransportNoopSearchAction.class) ); } diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkAction.java deleted file mode 100644 index 2def6e2185bea..0000000000000 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkAction.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.plugin.noop.action.bulk; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.bulk.BulkResponse; - -public class NoopBulkAction extends ActionType { - public static final String NAME = "mock:data/write/bulk"; - - public static final NoopBulkAction INSTANCE = new NoopBulkAction(); - - private NoopBulkAction() { - super(NAME, BulkResponse::new); - } -} diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java index 53283251f453d..9896fd6c84599 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.plugin.noop.NoopPlugin; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; @@ -31,7 +32,7 @@ public class TransportNoopBulkAction extends HandledTransportAction { - public static final NoopSearchAction INSTANCE = new NoopSearchAction(); - public static final String NAME = "mock:data/read/search"; - - private NoopSearchAction() { - super(NAME, SearchResponse::new); - } -} diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java index a1e57b095df99..b39007c3a3691 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.plugin.noop.NoopPlugin; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestChunkedToXContentListener; @@ -38,6 +39,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { SearchRequest searchRequest = new SearchRequest(); - return channel -> client.execute(NoopSearchAction.INSTANCE, searchRequest, new RestChunkedToXContentListener<>(channel)); + return channel -> client.execute(NoopPlugin.NOOP_SEARCH_ACTION, searchRequest, new RestChunkedToXContentListener<>(channel)); } } diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java index 7ad18e5c45644..baefb15e6373a 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.plugin.noop.NoopPlugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.InternalAggregations; @@ -32,7 +33,7 @@ public class TransportNoopSearchAction extends HandledTransportAction) SearchRequest::new, diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java index 690ac51eb2b36..baa163c1ae75e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java @@ -24,14 +24,14 @@ /** * Removes the data stream lifecycle configuration from the requested data streams. */ -public class DeleteDataStreamLifecycleAction extends ActionType { +public class DeleteDataStreamLifecycleAction { - public static final DeleteDataStreamLifecycleAction INSTANCE = new DeleteDataStreamLifecycleAction(); - public static final String NAME = "indices:admin/data_stream/lifecycle/delete"; + public static final ActionType INSTANCE = new ActionType<>( + "indices:admin/data_stream/lifecycle/delete", + AcknowledgedResponse::readFrom + ); - private DeleteDataStreamLifecycleAction() { - super(NAME, AcknowledgedResponse::readFrom); - } + private DeleteDataStreamLifecycleAction() {/* no instances */} public static final class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java index 8f0df2b130517..9e13bd7e0a99b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java @@ -33,13 +33,14 @@ /** * Action for explaining the data stream lifecycle status for one or more indices. */ -public class ExplainDataStreamLifecycleAction extends ActionType { - public static final ExplainDataStreamLifecycleAction INSTANCE = new ExplainDataStreamLifecycleAction(); - public static final String NAME = "indices:admin/data_stream/lifecycle/explain"; +public class ExplainDataStreamLifecycleAction { - public ExplainDataStreamLifecycleAction() { - super(NAME, Response::new); - } + public static final ActionType INSTANCE = new ActionType<>( + "indices:admin/data_stream/lifecycle/explain", + Response::new + ); + + private ExplainDataStreamLifecycleAction() {/* no instances */} /** * Request explaining the data stream lifecycle for one or more indices. diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java index 28e0e0ceb0915..d2e24479df347 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java @@ -34,14 +34,14 @@ /** * This action retrieves the data stream lifecycle from every data stream that has a data stream lifecycle configured. */ -public class GetDataStreamLifecycleAction extends ActionType { +public class GetDataStreamLifecycleAction { - public static final GetDataStreamLifecycleAction INSTANCE = new GetDataStreamLifecycleAction(); - public static final String NAME = "indices:admin/data_stream/lifecycle/get"; + public static final ActionType INSTANCE = new ActionType<>( + "indices:admin/data_stream/lifecycle/get", + Response::new + ); - private GetDataStreamLifecycleAction() { - super(NAME, Response::new); - } + private GetDataStreamLifecycleAction() {/* no instances */} public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamLifecycleAction.java index 641b89a513d40..a4f4b88d17bca 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamLifecycleAction.java @@ -38,14 +38,14 @@ /** * Sets the data stream lifecycle that was provided in the request to the requested data streams. */ -public class PutDataStreamLifecycleAction extends ActionType { +public class PutDataStreamLifecycleAction { - public static final PutDataStreamLifecycleAction INSTANCE = new PutDataStreamLifecycleAction(); - public static final String NAME = "indices:admin/data_stream/lifecycle/put"; + public static final ActionType INSTANCE = new ActionType<>( + "indices:admin/data_stream/lifecycle/put", + AcknowledgedResponse::readFrom + ); - private PutDataStreamLifecycleAction() { - super(NAME, AcknowledgedResponse::readFrom); - } + private PutDataStreamLifecycleAction() {/* no instances */} public static final class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable, ToXContentObject { diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java index c9abf81cd2836..0381014aed24b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java @@ -47,7 +47,7 @@ public TransportDeleteDataStreamLifecycleAction( SystemIndices systemIndices ) { super( - DeleteDataStreamLifecycleAction.NAME, + DeleteDataStreamLifecycleAction.INSTANCE.name(), transportService, clusterService, threadPool, diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java index f9cd68acfa072..a42e8dfefc468 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java @@ -53,7 +53,7 @@ public TransportExplainDataStreamLifecycleAction( DataStreamLifecycleErrorStore dataLifecycleServiceErrorStore ) { super( - ExplainDataStreamLifecycleAction.NAME, + ExplainDataStreamLifecycleAction.INSTANCE.name(), transportService, clusterService, threadPool, diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java index e90f80cec9877..29b88fc5748bf 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java @@ -48,7 +48,7 @@ public TransportGetDataStreamLifecycleAction( IndexNameExpressionResolver indexNameExpressionResolver ) { super( - GetDataStreamLifecycleAction.NAME, + GetDataStreamLifecycleAction.INSTANCE.name(), transportService, clusterService, threadPool, diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java index b893f2b461230..31d7237eeb681 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java @@ -46,7 +46,7 @@ public TransportPutDataStreamLifecycleAction( SystemIndices systemIndices ) { super( - PutDataStreamLifecycleAction.NAME, + PutDataStreamLifecycleAction.INSTANCE.name(), transportService, clusterService, threadPool, diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java index c7ab9f264c686..2fe666c5f208c 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java @@ -40,14 +40,14 @@ import static org.elasticsearch.grok.GrokBuiltinPatterns.ECS_COMPATIBILITY_DISABLED; import static org.elasticsearch.rest.RestRequest.Method.GET; -public class GrokProcessorGetAction extends ActionType { +public class GrokProcessorGetAction { - static final GrokProcessorGetAction INSTANCE = new GrokProcessorGetAction(); - static final String NAME = "cluster:admin/ingest/processor/grok/get"; + static final ActionType INSTANCE = new ActionType<>( + "cluster:admin/ingest/processor/grok/get", + Response::new + ); - private GrokProcessorGetAction() { - super(NAME, Response::new); - } + private GrokProcessorGetAction() {/* no instances */} public static class Request extends ActionRequest { @@ -140,7 +140,7 @@ public TransportAction(TransportService transportService, ActionFilters actionFi PatternBank legacyGrokPatterns, PatternBank ecsV1GrokPatterns ) { - super(NAME, transportService, actionFilters, Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); + super(INSTANCE.name(), transportService, actionFilters, Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.legacyGrokPatterns = legacyGrokPatterns.bank(); this.sortedLegacyGrokPatterns = new TreeMap<>(this.legacyGrokPatterns); this.ecsV1GrokPatterns = ecsV1GrokPatterns.bank(); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java index f1f2923ccd053..9921b144afcac 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java @@ -30,14 +30,11 @@ import java.util.Objects; import java.util.Set; -public class GeoIpDownloaderStatsAction extends ActionType { +public class GeoIpDownloaderStatsAction { - public static final GeoIpDownloaderStatsAction INSTANCE = new GeoIpDownloaderStatsAction(); - public static final String NAME = "cluster:monitor/ingest/geoip/stats"; + public static final ActionType INSTANCE = new ActionType<>("cluster:monitor/ingest/geoip/stats", Response::new); - public GeoIpDownloaderStatsAction() { - super(NAME, Response::new); - } + private GeoIpDownloaderStatsAction() {/* no instances */} public static class Request extends BaseNodesRequest implements ToXContentObject { @@ -55,7 +52,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public int hashCode() { // Nothing to hash atm, so just use the action name - return Objects.hashCode(NAME); + return Objects.hashCode(INSTANCE.name()); } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java index d0dcfb1ca966c..0958002405fbe 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java @@ -45,7 +45,7 @@ public GeoIpDownloaderStatsTransportAction( GeoIpDownloaderTaskExecutor geoIpDownloaderTaskExecutor ) { super( - GeoIpDownloaderStatsAction.NAME, + GeoIpDownloaderStatsAction.INSTANCE.name(), clusterService, transportService, actionFilters, diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java index 3cd800fc82dec..b2a8de93a2c32 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java @@ -142,7 +142,7 @@ public void testBasic() throws Exception { search5.setScriptParams(params5); multiRequest.add(search5); - MultiSearchTemplateResponse response = client().execute(MultiSearchTemplateAction.INSTANCE, multiRequest).get(); + MultiSearchTemplateResponse response = client().execute(MustachePlugin.MULTI_SEARCH_TEMPLATE_ACTION, multiRequest).get(); assertThat(response.getResponses(), arrayWithSize(5)); assertThat(response.getTook().millis(), greaterThan(0L)); @@ -195,7 +195,8 @@ public void testCCSCheckCompatibility() throws Exception { searchTemplateRequest.setRequest(new SearchRequest()); MultiSearchTemplateRequest request = new MultiSearchTemplateRequest(); request.add(searchTemplateRequest); - MultiSearchTemplateResponse multiSearchTemplateResponse = client().execute(MultiSearchTemplateAction.INSTANCE, request).get(); + MultiSearchTemplateResponse multiSearchTemplateResponse = client().execute(MustachePlugin.MULTI_SEARCH_TEMPLATE_ACTION, request) + .get(); Item response = multiSearchTemplateResponse.getResponses()[0]; assertTrue(response.isFailure()); Exception ex = response.getFailure(); diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java index 2ca332eabc029..18365abc820d0 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java @@ -100,7 +100,7 @@ public void testTemplateQueryAsEscapedString() throws Exception { }"""; SearchTemplateRequest request = SearchTemplateRequest.fromXContent(createParser(JsonXContent.jsonXContent, query)); request.setRequest(searchRequest); - SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get(); + SearchTemplateResponse searchResponse = client().execute(MustachePlugin.SEARCH_TEMPLATE_ACTION, request).get(); assertThat(searchResponse.getResponse().getHits().getHits().length, equalTo(1)); } @@ -121,7 +121,7 @@ public void testTemplateQueryAsEscapedStringStartingWithConditionalClause() thro }"""; SearchTemplateRequest request = SearchTemplateRequest.fromXContent(createParser(JsonXContent.jsonXContent, templateString)); request.setRequest(searchRequest); - SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get(); + SearchTemplateResponse searchResponse = client().execute(MustachePlugin.SEARCH_TEMPLATE_ACTION, request).get(); assertThat(searchResponse.getResponse().getHits().getHits().length, equalTo(1)); } @@ -142,7 +142,7 @@ public void testTemplateQueryAsEscapedStringWithConditionalClauseAtEnd() throws }"""; SearchTemplateRequest request = SearchTemplateRequest.fromXContent(createParser(JsonXContent.jsonXContent, templateString)); request.setRequest(searchRequest); - SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get(); + SearchTemplateResponse searchResponse = client().execute(MustachePlugin.SEARCH_TEMPLATE_ACTION, request).get(); assertThat(searchResponse.getResponse().getHits().getHits().length, equalTo(1)); } @@ -363,7 +363,7 @@ public void testCCSCheckCompatibility() throws Exception { request.setRequest(new SearchRequest()); ExecutionException ex = expectThrows( ExecutionException.class, - () -> client().execute(SearchTemplateAction.INSTANCE, request).get() + () -> client().execute(MustachePlugin.SEARCH_TEMPLATE_ACTION, request).get() ); Throwable primary = ex.getCause(); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java deleted file mode 100644 index 44beac575cc62..0000000000000 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script.mustache; - -import org.elasticsearch.action.ActionType; - -public class MultiSearchTemplateAction extends ActionType { - - public static final MultiSearchTemplateAction INSTANCE = new MultiSearchTemplateAction(); - public static final String NAME = "indices:data/read/msearch/template"; - - private MultiSearchTemplateAction() { - super(NAME, MultiSearchTemplateResponse::new); - } -} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java index 0f79e44464eea..b9996484c5bc0 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.ClusterSettings; @@ -32,6 +33,15 @@ public class MustachePlugin extends Plugin implements ScriptPlugin, ActionPlugin, SearchPlugin { + public static final ActionType SEARCH_TEMPLATE_ACTION = new ActionType<>( + "indices:data/read/search/template", + SearchTemplateResponse::new + ); + public static final ActionType MULTI_SEARCH_TEMPLATE_ACTION = new ActionType<>( + "indices:data/read/msearch/template", + MultiSearchTemplateResponse::new + ); + @Override public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { return new MustacheScriptEngine(); @@ -40,8 +50,8 @@ public ScriptEngine getScriptEngine(Settings settings, Collection> getActions() { return Arrays.asList( - new ActionHandler<>(SearchTemplateAction.INSTANCE, TransportSearchTemplateAction.class), - new ActionHandler<>(MultiSearchTemplateAction.INSTANCE, TransportMultiSearchTemplateAction.class) + new ActionHandler<>(SEARCH_TEMPLATE_ACTION, TransportSearchTemplateAction.class), + new ActionHandler<>(MULTI_SEARCH_TEMPLATE_ACTION, TransportMultiSearchTemplateAction.class) ); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index 2313419fea91c..d9466536ac46c 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -59,7 +59,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { MultiSearchTemplateRequest multiRequest = parseRequest(request, allowExplicitIndex); - return channel -> client.execute(MultiSearchTemplateAction.INSTANCE, multiRequest, new RestToXContentListener<>(channel)); + return channel -> client.execute(MustachePlugin.MULTI_SEARCH_TEMPLATE_ACTION, multiRequest, new RestToXContentListener<>(channel)); } /** diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java index 97216026a0967..18e18a49f7ddb 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java @@ -56,6 +56,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client renderRequest.setScript(id); } - return channel -> client.execute(SearchTemplateAction.INSTANCE, renderRequest, new RestToXContentListener<>(channel)); + return channel -> client.execute(MustachePlugin.SEARCH_TEMPLATE_ACTION, renderRequest, new RestToXContentListener<>(channel)); } } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index 7d6c02096e438..0dbb810902b44 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -77,7 +77,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client searchTemplateRequest.setExplain(searchRequest.source().explain()); } return channel -> client.execute( - SearchTemplateAction.INSTANCE, + MustachePlugin.SEARCH_TEMPLATE_ACTION, searchTemplateRequest, new RestToXContentListener<>(channel, SearchTemplateResponse::status) ); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java deleted file mode 100644 index d806a9f5a7744..0000000000000 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script.mustache; - -import org.elasticsearch.action.ActionType; - -public class SearchTemplateAction extends ActionType { - - public static final SearchTemplateAction INSTANCE = new SearchTemplateAction(); - public static final String NAME = "indices:data/read/search/template"; - - private SearchTemplateAction() { - super(NAME, SearchTemplateResponse::new); - } -} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java index c606d10ff4cfa..6bed28f84bdc8 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java @@ -9,6 +9,7 @@ package org.elasticsearch.script.mustache; import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.internal.ElasticsearchClient; import org.elasticsearch.script.ScriptType; @@ -17,12 +18,12 @@ public class SearchTemplateRequestBuilder extends ActionRequestBuilder { - SearchTemplateRequestBuilder(ElasticsearchClient client, SearchTemplateAction action) { + SearchTemplateRequestBuilder(ElasticsearchClient client, ActionType action) { super(client, action, new SearchTemplateRequest()); } public SearchTemplateRequestBuilder(ElasticsearchClient client) { - this(client, SearchTemplateAction.INSTANCE); + this(client, MustachePlugin.SEARCH_TEMPLATE_ACTION); } public SearchTemplateRequestBuilder setRequest(SearchRequest searchRequest) { diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java index 8df836e1cfe72..d859fb509e915 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java @@ -46,7 +46,7 @@ public TransportMultiSearchTemplateAction( UsageService usageService ) { super( - MultiSearchTemplateAction.NAME, + MustachePlugin.MULTI_SEARCH_TEMPLATE_ACTION.name(), transportService, actionFilters, MultiSearchTemplateRequest::new, diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java index 926c72b27f794..ef19579c87625 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java @@ -55,7 +55,13 @@ public TransportSearchTemplateAction( NodeClient client, UsageService usageService ) { - super(SearchTemplateAction.NAME, transportService, actionFilters, SearchTemplateRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); + super( + MustachePlugin.SEARCH_TEMPLATE_ACTION.name(), + transportService, + actionFilters, + SearchTemplateRequest::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); this.scriptService = scriptService; this.xContentRegistry = xContentRegistry; this.client = client; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java index 4db8ab14d5709..376278f8f0c52 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java @@ -52,16 +52,13 @@ * retrieves all available information about the API for this specific context * */ -public class PainlessContextAction extends ActionType { +public class PainlessContextAction { - public static final PainlessContextAction INSTANCE = new PainlessContextAction(); - private static final String NAME = "cluster:admin/scripts/painless/context"; + public static final ActionType INSTANCE = new ActionType<>("cluster:admin/scripts/painless/context", Response::new); private static final String SCRIPT_CONTEXT_NAME_PARAM = "context"; - private PainlessContextAction() { - super(NAME, PainlessContextAction.Response::new); - } + private PainlessContextAction() {/* no instances */} public static class Request extends ActionRequest { @@ -143,7 +140,13 @@ public static class TransportAction extends HandledTransportAction) Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); + super( + INSTANCE.name(), + transportService, + actionFilters, + (Writeable.Reader) Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); this.painlessScriptEngine = painlessScriptEngine; } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index d885db5ed39a9..8ec90c7d04979 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -113,14 +113,11 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; -public class PainlessExecuteAction extends ActionType { +public class PainlessExecuteAction { - public static final PainlessExecuteAction INSTANCE = new PainlessExecuteAction(); - private static final String NAME = "cluster:admin/scripts/painless/execute"; + public static final ActionType INSTANCE = new ActionType<>("cluster:admin/scripts/painless/execute", Response::new); - private PainlessExecuteAction() { - super(NAME, Response::new); - } + private PainlessExecuteAction() {/* no instances */} public static class Request extends SingleShardRequest implements ToXContentObject { @@ -507,7 +504,7 @@ public TransportAction( IndicesService indicesServices ) { super( - NAME, + INSTANCE.name(), threadPool, clusterService, transportService, diff --git a/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 1c473b8fee21b..7e879d9959f6d 100644 --- a/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -88,11 +88,11 @@ public void testPrecisionAtRequest() { PrecisionAtK metric = new PrecisionAtK(1, false, 10); RankEvalSpec task = new RankEvalSpec(specifications, metric); - RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); + RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalPlugin.ACTION, new RankEvalRequest()); builder.setRankEvalSpec(task); String indexToUse = randomBoolean() ? TEST_INDEX : INDEX_ALIAS; - RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices(indexToUse)).actionGet(); + RankEvalResponse response = client().execute(RankEvalPlugin.ACTION, builder.request().indices(indexToUse)).actionGet(); // the expected Prec@ for the first query is 4/6 and the expected Prec@ for the // second is 1/6, divided by 2 to get the average double expectedPrecision = (1.0 / 6.0 + 4.0 / 6.0) / 2.0; @@ -133,9 +133,9 @@ public void testPrecisionAtRequest() { metric = new PrecisionAtK(1, false, 3); task = new RankEvalSpec(specifications, metric); - builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX })); + builder = new RankEvalRequestBuilder(client(), RankEvalPlugin.ACTION, new RankEvalRequest(task, new String[] { TEST_INDEX })); - response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + response = client().execute(RankEvalPlugin.ACTION, builder.request()).actionGet(); // if we look only at top 3 documente, the expected P@3 for the first query is // 2/3 and the expected Prec@ for the second is 1/3, divided by 2 to get the average expectedPrecision = (1.0 / 3.0 + 2.0 / 3.0) / 2.0; @@ -167,20 +167,20 @@ public void testDCGRequest() { RankEvalRequestBuilder builder = new RankEvalRequestBuilder( client(), - RankEvalAction.INSTANCE, + RankEvalPlugin.ACTION, new RankEvalRequest(task, new String[] { TEST_INDEX }) ); - RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + RankEvalResponse response = client().execute(RankEvalPlugin.ACTION, builder.request()).actionGet(); assertEquals(DiscountedCumulativeGainTests.EXPECTED_DCG, response.getMetricScore(), 10E-14); // test that a different window size k affects the result metric = new DiscountedCumulativeGain(false, null, 3); task = new RankEvalSpec(specifications, metric); - builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX })); + builder = new RankEvalRequestBuilder(client(), RankEvalPlugin.ACTION, new RankEvalRequest(task, new String[] { TEST_INDEX })); - response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + response = client().execute(RankEvalPlugin.ACTION, builder.request()).actionGet(); assertEquals(12.39278926071437, response.getMetricScore(), 10E-14); } @@ -198,11 +198,11 @@ public void testMRRRequest() { RankEvalRequestBuilder builder = new RankEvalRequestBuilder( client(), - RankEvalAction.INSTANCE, + RankEvalPlugin.ACTION, new RankEvalRequest(task, new String[] { TEST_INDEX }) ); - RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + RankEvalResponse response = client().execute(RankEvalPlugin.ACTION, builder.request()).actionGet(); // the expected reciprocal rank for the amsterdam_query is 1/5 // the expected reciprocal rank for the berlin_query is 1/1 // dividing by 2 to get the average @@ -213,9 +213,9 @@ public void testMRRRequest() { metric = new MeanReciprocalRank(1, 3); task = new RankEvalSpec(specifications, metric); - builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX })); + builder = new RankEvalRequestBuilder(client(), RankEvalPlugin.ACTION, new RankEvalRequest(task, new String[] { TEST_INDEX })); - response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + response = client().execute(RankEvalPlugin.ACTION, builder.request()).actionGet(); // limiting to top 3 results, the amsterdam_query has no relevant document in it // the reciprocal rank for the berlin_query is 1/1 // dividing by 2 to get the average @@ -243,12 +243,12 @@ public void testBadQuery() { RankEvalRequestBuilder builder = new RankEvalRequestBuilder( client(), - RankEvalAction.INSTANCE, + RankEvalPlugin.ACTION, new RankEvalRequest(task, new String[] { TEST_INDEX }) ); builder.setRankEvalSpec(task); - RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + RankEvalResponse response = client().execute(RankEvalPlugin.ACTION, builder.request()).actionGet(); assertEquals(1, response.getFailures().size()); ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(response.getFailures().get("broken_query")); assertEquals("java.lang.NumberFormatException: For input string: \"noStringOnNumericFields\"", rootCauses[0].getCause().toString()); @@ -268,7 +268,7 @@ public void testIndicesOptions() { RankEvalRequest request = new RankEvalRequest(task, new String[] { TEST_INDEX, "test2" }); request.setRankEvalSpec(task); - RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); + RankEvalResponse response = client().execute(RankEvalPlugin.ACTION, request).actionGet(); Detail details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(7, details.getRetrieved()); assertEquals(6, details.getRelevantRetrieved()); @@ -277,7 +277,7 @@ public void testIndicesOptions() { assertTrue(indicesAdmin().prepareClose("test2").get().isAcknowledged()); request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); - response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); + response = client().execute(RankEvalPlugin.ACTION, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(6, details.getRetrieved()); assertEquals(5, details.getRelevantRetrieved()); @@ -285,37 +285,37 @@ public void testIndicesOptions() { // test that ignore_unavailable=false or default settings throw an IndexClosedException assertTrue(indicesAdmin().prepareClose("test2").get().isAcknowledged()); request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); - response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); + response = client().execute(RankEvalPlugin.ACTION, request).actionGet(); assertEquals(1, response.getFailures().size()); assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class)); // test expand_wildcards request = new RankEvalRequest(task, new String[] { "tes*" }); request.indicesOptions(IndicesOptions.fromParameters("none", "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); - response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); + response = client().execute(RankEvalPlugin.ACTION, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(0, details.getRetrieved()); request.indicesOptions(IndicesOptions.fromParameters("open", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); - response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); + response = client().execute(RankEvalPlugin.ACTION, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(6, details.getRetrieved()); assertEquals(5, details.getRelevantRetrieved()); request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); - response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); + response = client().execute(RankEvalPlugin.ACTION, request).actionGet(); assertEquals(1, response.getFailures().size()); assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class)); // test allow_no_indices request = new RankEvalRequest(task, new String[] { "bad*" }); request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); - response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); + response = client().execute(RankEvalPlugin.ACTION, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(0, details.getRetrieved()); request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); - response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); + response = client().execute(RankEvalPlugin.ACTION, request).actionGet(); assertEquals(1, response.getFailures().size()); assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexNotFoundException.class)); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java deleted file mode 100644 index 93bca430af97b..0000000000000 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.rankeval; - -import org.elasticsearch.action.ActionType; - -/** - * ActionType for explaining evaluating search ranking results. - */ -public class RankEvalAction extends ActionType { - - public static final RankEvalAction INSTANCE = new RankEvalAction(); - public static final String NAME = "indices:data/read/rank_eval"; - - private RankEvalAction() { - super(NAME, RankEvalResponse::new); - } -} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index 03216a937d694..814abcf02c569 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -31,9 +32,11 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { + public static final ActionType ACTION = new ActionType<>("indices:data/read/rank_eval", RankEvalResponse::new); + @Override public List> getActions() { - return Arrays.asList(new ActionHandler<>(RankEvalAction.INSTANCE, TransportRankEvalAction.class)); + return Arrays.asList(new ActionHandler<>(ACTION, TransportRankEvalAction.class)); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 8b12387ce0f4d..150d7053e05eb 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -99,7 +99,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli parseRankEvalRequest(rankEvalRequest, request, parser); } return channel -> client.executeLocally( - RankEvalAction.INSTANCE, + RankEvalPlugin.ACTION, rankEvalRequest, new RestToXContentListener(channel) ); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index f543111f4a8df..63995d3b6151e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -70,7 +70,7 @@ public TransportRankEvalAction( ScriptService scriptService, NamedXContentRegistry namedXContentRegistry ) { - super(RankEvalAction.NAME, transportService, actionFilters, RankEvalRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); + super(RankEvalPlugin.ACTION.name(), transportService, actionFilters, RankEvalRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.scriptService = scriptService; this.namedXContentRegistry = namedXContentRegistry; this.client = client; diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java index eb963ce4a1341..dd9bde8035ef4 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.reindex.CancelTests; import org.elasticsearch.reindex.ReindexPlugin; -import org.elasticsearch.reindex.RethrottleAction; import org.elasticsearch.reindex.RethrottleRequestBuilder; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -222,7 +221,7 @@ public void testTasks() throws Exception { } { // tag::update-by-query-rethrottle - new RethrottleRequestBuilder(client, RethrottleAction.INSTANCE) + new RethrottleRequestBuilder(client, ReindexPlugin.RETHROTTLE_ACTION) .setTargetTaskId(taskId) .setRequestsPerSecond(2.0f) .get(); diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java index e79f9bdaffee9..f8f87011405fc 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java @@ -10,6 +10,8 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -52,13 +54,18 @@ public class ReindexPlugin extends Plugin implements ActionPlugin { public static final String NAME = "reindex"; + public static final ActionType RETHROTTLE_ACTION = new ActionType<>( + "cluster:admin/reindex/rethrottle", + ListTasksResponse::new + ); + @Override public List> getActions() { return Arrays.asList( new ActionHandler<>(ReindexAction.INSTANCE, TransportReindexAction.class), new ActionHandler<>(UpdateByQueryAction.INSTANCE, TransportUpdateByQueryAction.class), new ActionHandler<>(DeleteByQueryAction.INSTANCE, TransportDeleteByQueryAction.class), - new ActionHandler<>(RethrottleAction.INSTANCE, TransportRethrottleAction.class) + new ActionHandler<>(RETHROTTLE_ACTION, TransportRethrottleAction.class) ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestRethrottleAction.java index 052749d6f666c..5d89fdb45cbde 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestRethrottleAction.java @@ -55,7 +55,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC internalRequest.setRequestsPerSecond(requestsPerSecond); final String groupBy = request.param("group_by", "nodes"); return channel -> client.execute( - RethrottleAction.INSTANCE, + ReindexPlugin.RETHROTTLE_ACTION, internalRequest, listTasksResponseListener(nodesInCluster, groupBy, channel) ); diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleAction.java deleted file mode 100644 index 267e3bc776d91..0000000000000 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleAction.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.reindex; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; - -public class RethrottleAction extends ActionType { - public static final RethrottleAction INSTANCE = new RethrottleAction(); - public static final String NAME = "cluster:admin/reindex/rethrottle"; - - private RethrottleAction() { - super(NAME, ListTasksResponse::new); - } -} diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java index d9c759d784075..bc89928358dc2 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java @@ -39,7 +39,7 @@ public TransportRethrottleAction( Client client ) { super( - RethrottleAction.NAME, + ReindexPlugin.RETHROTTLE_ACTION.name(), clusterService, transportService, actionFilters, @@ -101,7 +101,7 @@ private static void rethrottleParentTask( subRequest.setRequestsPerSecond(newRequestsPerSecond / runningSubtasks); subRequest.setTargetParentTaskId(new TaskId(localNodeId, task.getId())); logger.debug("rethrottling children of task [{}] to [{}] requests per second", task.getId(), subRequest.getRequestsPerSecond()); - client.execute(RethrottleAction.INSTANCE, subRequest, ActionListener.wrap(r -> { + client.execute(ReindexPlugin.RETHROTTLE_ACTION, subRequest, ActionListener.wrap(r -> { r.rethrowFailures("Rethrottle"); listener.onResponse(task.taskInfoGivenSubtaskInfo(localNodeId, r.getTasks())); }, listener::onFailure)); diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexTestCase.java index 2bc30ba612f80..cf6eb4b8aa888 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexTestCase.java @@ -50,7 +50,7 @@ protected DeleteByQueryRequestBuilder deleteByQuery() { } protected RethrottleRequestBuilder rethrottle() { - return new RethrottleRequestBuilder(client(), RethrottleAction.INSTANCE); + return new RethrottleRequestBuilder(client(), ReindexPlugin.RETHROTTLE_ACTION); } public static BulkIndexByScrollResponseMatcher matcher() { diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/TransportRethrottleActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/TransportRethrottleActionTests.java index 4f7a914d9b042..80af095005c9d 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/TransportRethrottleActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/TransportRethrottleActionTests.java @@ -73,7 +73,7 @@ private void rethrottleTestCase( @SuppressWarnings({ "unchecked", "rawtypes" }) // Magical generics incantation..... ArgumentCaptor> subListener = ArgumentCaptor.forClass((Class) ActionListener.class); if (runningSlices > 0) { - verify(client).execute(eq(RethrottleAction.INSTANCE), subRequest.capture(), subListener.capture()); + verify(client).execute(eq(ReindexPlugin.RETHROTTLE_ACTION), subRequest.capture(), subListener.capture()); assertEquals(new TaskId(localNodeId, task.getId()), subRequest.getValue().getTargetParentTaskId()); assertEquals(newRequestsPerSecond / runningSlices, subRequest.getValue().getRequestsPerSecond(), 0.00001f); diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainAction.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainAction.java deleted file mode 100644 index 9376c9a8edebc..0000000000000 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainAction.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.rest.root; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.common.io.stream.Writeable; - -public class MainAction extends ActionType { - - public static final String NAME = "cluster:monitor/main"; - public static final MainAction INSTANCE = new MainAction(); - - public MainAction() { - super(NAME, Writeable.Reader.localOnly()); - } -} diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java index 8dfc8fdfc1a64..62063ddab9129 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.ClusterSettings; @@ -25,6 +26,9 @@ import java.util.function.Supplier; public class MainRestPlugin extends Plugin implements ActionPlugin { + + public static final ActionType MAIN_ACTION = ActionType.localOnly("cluster:monitor/main"); + @Override public List getRestHandlers( Settings settings, @@ -40,6 +44,6 @@ public List getRestHandlers( @Override public List> getActions() { - return List.of(new ActionHandler<>(MainAction.INSTANCE, TransportMainAction.class)); + return List.of(new ActionHandler<>(MAIN_ACTION, TransportMainAction.class)); } } diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/RestMainAction.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/RestMainAction.java index f08c5665e50b6..2b14d52850a13 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/RestMainAction.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/RestMainAction.java @@ -39,7 +39,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - return channel -> client.execute(MainAction.INSTANCE, new MainRequest(), new RestBuilderListener(channel) { + return channel -> client.execute(MainRestPlugin.MAIN_ACTION, new MainRequest(), new RestBuilderListener(channel) { @Override public RestResponse buildResponse(MainResponse mainResponse, XContentBuilder builder) throws Exception { return convertMainResponse(mainResponse, request, builder); diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java index 85e8af01ace35..6b4b0a52b643a 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java @@ -33,7 +33,7 @@ public TransportMainAction( ActionFilters actionFilters, ClusterService clusterService ) { - super(MainAction.NAME, actionFilters, transportService.getTaskManager()); + super(MainRestPlugin.MAIN_ACTION.name(), actionFilters, transportService.getTaskManager()); this.nodeName = Node.NODE_NAME_SETTING.get(settings); this.clusterService = clusterService; } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskStorageRetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskStorageRetryIT.java index 1fb21572f2f48..603c2cb14bd62 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskStorageRetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskStorageRetryIT.java @@ -65,7 +65,7 @@ public void testRetry() throws Exception { TestTaskPlugin.NodesRequest req = new TestTaskPlugin.NodesRequest("foo"); req.setShouldStoreResult(true); req.setShouldBlock(false); - task = nodeClient().executeLocally(TestTaskPlugin.TestTaskAction.INSTANCE, req, future); + task = nodeClient().executeLocally(TestTaskPlugin.TEST_TASK_ACTION, req, future); logger.info("verify that the task has started and is still running"); assertBusy(() -> { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index de51066a1f3ba..c0a54ef874de5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -76,6 +76,8 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singleton; +import static org.elasticsearch.action.admin.cluster.node.tasks.TestTaskPlugin.TEST_TASK_ACTION; +import static org.elasticsearch.action.admin.cluster.node.tasks.TestTaskPlugin.UNBLOCK_TASK_ACTION; import static org.elasticsearch.core.TimeValue.timeValueMillis; import static org.elasticsearch.core.TimeValue.timeValueSeconds; import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE; @@ -498,7 +500,7 @@ public void testTasksCancellation() throws Exception { // Start blocking test task // Get real client (the plugin is not registered on transport nodes) TestTaskPlugin.NodesRequest request = new TestTaskPlugin.NodesRequest("test"); - ActionFuture future = client().execute(TestTaskPlugin.TestTaskAction.INSTANCE, request); + ActionFuture future = client().execute(TEST_TASK_ACTION, request); logger.info("--> started test tasks"); @@ -506,54 +508,51 @@ public void testTasksCancellation() throws Exception { assertBusy( () -> assertEquals( internalCluster().size(), - clusterAdmin().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size() + clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name() + "[n]").get().getTasks().size() ) ); logger.info("--> cancelling the main test task"); - CancelTasksResponse cancelTasksResponse = clusterAdmin().prepareCancelTasks().setActions(TestTaskPlugin.TestTaskAction.NAME).get(); + CancelTasksResponse cancelTasksResponse = clusterAdmin().prepareCancelTasks().setActions(TEST_TASK_ACTION.name()).get(); assertEquals(1, cancelTasksResponse.getTasks().size()); expectThrows(TaskCancelledException.class, future::actionGet); logger.info("--> checking that test tasks are not running"); - assertEquals(0, clusterAdmin().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "*").get().getTasks().size()); + assertEquals(0, clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name() + "*").get().getTasks().size()); } @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/95325") public void testTasksUnblocking() throws Exception { // Start blocking test task TestTaskPlugin.NodesRequest request = new TestTaskPlugin.NodesRequest("test"); - ActionFuture future = client().execute(TestTaskPlugin.TestTaskAction.INSTANCE, request); + ActionFuture future = client().execute(TEST_TASK_ACTION, request); // Wait for the task to start on all nodes assertBusy( () -> assertEquals( internalCluster().size(), - clusterAdmin().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size() + clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name() + "[n]").get().getTasks().size() ) ); - new TestTaskPlugin.UnblockTestTasksRequestBuilder(client(), TestTaskPlugin.UnblockTestTasksAction.INSTANCE).get(); + new TestTaskPlugin.UnblockTestTasksRequestBuilder(client(), UNBLOCK_TASK_ACTION).get(); future.get(); assertBusy( - () -> assertEquals( - 0, - clusterAdmin().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size() - ) + () -> assertEquals(0, clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name() + "[n]").get().getTasks().size()) ); } public void testListTasksWaitForCompletion() throws Exception { waitForCompletionTestCase( randomBoolean(), - id -> clusterAdmin().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME).setWaitForCompletion(true).execute(), + id -> clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name()).setWaitForCompletion(true).execute(), response -> { assertThat(response.getNodeFailures(), empty()); assertThat(response.getTaskFailures(), empty()); assertThat(response.getTasks(), hasSize(1)); TaskInfo task = response.getTasks().get(0); - assertEquals(TestTaskPlugin.TestTaskAction.NAME, task.action()); + assertEquals(TEST_TASK_ACTION.name(), task.action()); } ); } @@ -565,7 +564,7 @@ public void testGetTaskWaitForCompletionWithoutStoringResult() throws Exception assertNull(response.getTask().getResponse()); // But the task's details should still be there because we grabbed a reference to the task before waiting for it to complete assertNotNull(response.getTask().getTask()); - assertEquals(TestTaskPlugin.TestTaskAction.NAME, response.getTask().getTask().action()); + assertEquals(TEST_TASK_ACTION.name(), response.getTask().getTask().action()); }); } @@ -576,7 +575,7 @@ public void testGetTaskWaitForCompletionWithStoringResult() throws Exception { assertEquals(0, response.getTask().getResponseAsMap().get("failure_count")); // The task's details should also be there assertNotNull(response.getTask().getTask()); - assertEquals(TestTaskPlugin.TestTaskAction.NAME, response.getTask().getTask().action()); + assertEquals(TEST_TASK_ACTION.name(), response.getTask().getTask().action()); }); } @@ -591,7 +590,7 @@ private void waitForCompletionTestCase(boolean storeResult, Function future = client().execute(TestTaskPlugin.TestTaskAction.INSTANCE, request); + ActionFuture future = client().execute(TEST_TASK_ACTION, request); ActionFuture waitResponseFuture; TaskId taskId; @@ -627,7 +626,7 @@ public void onTaskUnregistered(Task task) { waitForWaitingToStart.await(); } finally { // Unblock the request so the wait for completion request can finish - new TestTaskPlugin.UnblockTestTasksRequestBuilder(client(), TestTaskPlugin.UnblockTestTasksAction.INSTANCE).get(); + new TestTaskPlugin.UnblockTestTasksRequestBuilder(client(), UNBLOCK_TASK_ACTION).get(); } // Now that the task is unblocked the list response will come back @@ -641,7 +640,7 @@ public void onTaskUnregistered(Task task) { public void testListTasksWaitForTimeout() throws Exception { waitForTimeoutTestCase(id -> { ListTasksResponse response = clusterAdmin().prepareListTasks() - .setActions(TestTaskPlugin.TestTaskAction.NAME) + .setActions(TEST_TASK_ACTION.name()) .setWaitForCompletion(true) .setTimeout(timeValueMillis(100)) .get(); @@ -667,7 +666,7 @@ public void testGetTaskWaitForTimeout() throws Exception { private void waitForTimeoutTestCase(Function> wait) throws Exception { // Start blocking test task TestTaskPlugin.NodesRequest request = new TestTaskPlugin.NodesRequest("test"); - ActionFuture future = client().execute(TestTaskPlugin.TestTaskAction.INSTANCE, request); + ActionFuture future = client().execute(TEST_TASK_ACTION, request); try { TaskId taskId = waitForTestTaskStartOnAllNodes(); @@ -685,7 +684,7 @@ private void waitForTimeoutTestCase(Function { - List tasks = clusterAdmin().prepareListTasks() - .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]") - .get() - .getTasks(); + List tasks = clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name() + "[n]").get().getTasks(); assertEquals(internalCluster().size(), tasks.size()); }); - List task = clusterAdmin().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME).get().getTasks(); + List task = clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name()).get().getTasks(); assertThat(task, hasSize(1)); return task.get(0).taskId(); } @@ -709,7 +705,7 @@ private TaskId waitForTestTaskStartOnAllNodes() throws Exception { public void testTasksListWaitForNoTask() throws Exception { // Spin up a request to wait for no matching tasks ActionFuture waitResponseFuture = clusterAdmin().prepareListTasks() - .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]") + .setActions(TEST_TASK_ACTION.name() + "[n]") .setWaitForCompletion(true) .setTimeout(timeValueMillis(10)) .execute(); @@ -753,7 +749,7 @@ public void testTasksWaitForAllTask() throws Exception { } public void testTaskStoringSuccessfulResult() throws Exception { - registerTaskManagerListeners(TestTaskPlugin.TestTaskAction.NAME); // we need this to get task id of the process + registerTaskManagerListeners(TEST_TASK_ACTION.name()); // we need this to get task id of the process // Start non-blocking test task TestTaskPlugin.NodesRequest request = new TestTaskPlugin.NodesRequest("test"); @@ -762,9 +758,9 @@ public void testTaskStoringSuccessfulResult() throws Exception { TaskId parentTaskId = new TaskId("parent_node", randomLong()); request.setParentTask(parentTaskId); - client().execute(TestTaskPlugin.TestTaskAction.INSTANCE, request).get(); + client().execute(TEST_TASK_ACTION, request).get(); - List events = findEvents(TestTaskPlugin.TestTaskAction.NAME, Tuple::v1); + List events = findEvents(TEST_TASK_ACTION.name(), Tuple::v1); assertEquals(1, events.size()); TaskInfo taskInfo = events.get(0); @@ -803,15 +799,15 @@ public void testTaskStoringSuccessfulResult() throws Exception { assertNull(getResponse.getTask().getError()); // run it again to check that the tasks index has been successfully created and can be re-used - client().execute(TestTaskPlugin.TestTaskAction.INSTANCE, request).get(); + client().execute(TEST_TASK_ACTION, request).get(); - events = findEvents(TestTaskPlugin.TestTaskAction.NAME, Tuple::v1); + events = findEvents(TEST_TASK_ACTION.name(), Tuple::v1); assertEquals(2, events.size()); } public void testTaskStoringFailureResult() throws Exception { - registerTaskManagerListeners(TestTaskPlugin.TestTaskAction.NAME); // we need this to get task id of the process + registerTaskManagerListeners(TEST_TASK_ACTION.name()); // we need this to get task id of the process TestTaskPlugin.NodesRequest request = new TestTaskPlugin.NodesRequest("test"); request.setShouldFail(true); @@ -819,9 +815,9 @@ public void testTaskStoringFailureResult() throws Exception { request.setShouldBlock(false); // Start non-blocking test task that should fail - assertFutureThrows(client().execute(TestTaskPlugin.TestTaskAction.INSTANCE, request), IllegalStateException.class); + assertFutureThrows(client().execute(TEST_TASK_ACTION, request), IllegalStateException.class); - List events = findEvents(TestTaskPlugin.TestTaskAction.NAME, Tuple::v1); + List events = findEvents(TEST_TASK_ACTION.name(), Tuple::v1); assertEquals(1, events.size()); TaskInfo failedTaskInfo = events.get(0); TaskId failedTaskId = failedTaskInfo.taskId(); diff --git a/server/src/main/java/org/elasticsearch/action/ActionType.java b/server/src/main/java/org/elasticsearch/action/ActionType.java index 27820398d10a3..478fab0f2cf36 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionType.java +++ b/server/src/main/java/org/elasticsearch/action/ActionType.java @@ -39,7 +39,7 @@ public String name() { } /** - * Get a reader that can create a new instance of the class from a {@link org.elasticsearch.common.io.stream.StreamInput} + * Get a reader that can read a response from a {@link org.elasticsearch.common.io.stream.StreamInput}. */ public Writeable.Reader getResponseReader() { return responseReader; @@ -47,7 +47,7 @@ public Writeable.Reader getResponseReader() { @Override public boolean equals(Object o) { - return o instanceof ActionType && name.equals(((ActionType) o).name()); + return o instanceof ActionType actionType && name.equals(actionType.name); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java index 614ede7ec96e4..582f9b44af57b 100644 --- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java +++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java @@ -91,12 +91,7 @@ protected FakeTransportAction(String actionName, ActionFilters actionFilters, Ta @Override protected void doExecute(Task task, FakeRequest request, ActionListener listener) {} } - class FakeAction extends ActionType { - protected FakeAction() { - super("fake", null); - } - } - FakeAction action = new FakeAction(); + final var action = new ActionType<>("fake", null); ActionPlugin registersFakeAction = new ActionPlugin() { @Override public List> getActions() { diff --git a/server/src/test/java/org/elasticsearch/action/ActionTests.java b/server/src/test/java/org/elasticsearch/action/ActionTests.java index d80e5ed90af93..4874972314ee6 100644 --- a/server/src/test/java/org/elasticsearch/action/ActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/ActionTests.java @@ -13,14 +13,9 @@ public class ActionTests extends ESTestCase { public void testEquals() { - class FakeAction extends ActionType { - protected FakeAction(String name) { - super(name, null); - } - } - FakeAction fakeAction1 = new FakeAction("a"); - FakeAction fakeAction2 = new FakeAction("a"); - FakeAction fakeAction3 = new FakeAction("b"); + final var fakeAction1 = ActionType.localOnly("a"); + final var fakeAction2 = ActionType.localOnly("a"); + final var fakeAction3 = ActionType.localOnly("b"); String s = "Some random other object"; assertEquals(fakeAction1, fakeAction1); assertEquals(fakeAction2, fakeAction2); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java index 1a772f77f2d91..9e95b3f7c1f79 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java @@ -73,11 +73,17 @@ public class TestTaskPlugin extends Plugin implements ActionPlugin, NetworkPlugi private static final Logger logger = LogManager.getLogger(TestTaskPlugin.class); + public static final ActionType TEST_TASK_ACTION = ActionType.localOnly("cluster:admin/tasks/test"); + public static final ActionType UNBLOCK_TASK_ACTION = new ActionType<>( + "cluster:admin/tasks/testunblock", + UnblockTestTasksResponse::new + ); + @Override public List> getActions() { return Arrays.asList( - new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class), - new ActionHandler<>(UnblockTestTasksAction.INSTANCE, TransportUnblockTestTasksAction.class) + new ActionHandler<>(TEST_TASK_ACTION, TransportTestTaskAction.class), + new ActionHandler<>(UNBLOCK_TASK_ACTION, TransportUnblockTestTasksAction.class) ); } @@ -262,7 +268,7 @@ public static class TransportTestTaskAction extends TransportNodesAction()), @@ -314,16 +320,6 @@ protected NodeResponse nodeOperation(NodeRequest request, Task task) { } } - public static class TestTaskAction extends ActionType { - - public static final TestTaskAction INSTANCE = new TestTaskAction(); - public static final String NAME = "cluster:admin/tasks/test"; - - private TestTaskAction() { - super(NAME, NodesResponse::new); - } - } - public static class UnblockTestTaskResponse implements Writeable { UnblockTestTaskResponse() { @@ -395,7 +391,7 @@ public static class TransportUnblockTestTasksAction extends TransportTasksAction @Inject public TransportUnblockTestTasksAction(ClusterService clusterService, TransportService transportService) { super( - UnblockTestTasksAction.NAME, + UNBLOCK_TASK_ACTION.name(), clusterService, transportService, new ActionFilters(new HashSet<>()), @@ -429,16 +425,6 @@ protected void taskOperation( } - public static class UnblockTestTasksAction extends ActionType { - - public static final UnblockTestTasksAction INSTANCE = new UnblockTestTasksAction(); - public static final String NAME = "cluster:admin/tasks/testunblock"; - - private UnblockTestTasksAction() { - super(NAME, UnblockTestTasksResponse::new); - } - } - public static class UnblockTestTasksRequestBuilder extends ActionRequestBuilder { protected UnblockTestTasksRequestBuilder(ElasticsearchClient client, ActionType action) { diff --git a/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java b/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java index 02ee4b080834f..877f8ce4dcb96 100644 --- a/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java +++ b/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java @@ -59,14 +59,12 @@ public List> getSettings() { return Arrays.asList(INDEX_INTERNAL_SETTING, INDEX_PRIVATE_SETTING); } - public static class UpdateInternalOrPrivateAction extends ActionType { + public static class UpdateInternalOrPrivateAction { - public static final UpdateInternalOrPrivateAction INSTANCE = new UpdateInternalOrPrivateAction(); - private static final String NAME = "indices:admin/settings/update-internal-or-private-index"; - - public UpdateInternalOrPrivateAction() { - super(NAME, UpdateInternalOrPrivateAction.Response::new); - } + public static final ActionType INSTANCE = new ActionType<>( + "indices:admin/settings/update-internal-or-private-index", + UpdateInternalOrPrivateAction.Response::new + ); public static class Request extends MasterNodeRequest { @@ -130,7 +128,7 @@ public TransportUpdateInternalOrPrivateAction( final IndexNameExpressionResolver indexNameExpressionResolver ) { super( - UpdateInternalOrPrivateAction.NAME, + UpdateInternalOrPrivateAction.INSTANCE.name(), transportService, clusterService, threadPool, diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index d91d595c701eb..c131537c1815a 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -76,9 +76,14 @@ */ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin, PersistentTaskPlugin { + public static final ActionType TEST_ACTION = new ActionType<>( + "cluster:admin/persistent/task_test", + TestTasksResponse::new + ); + @Override public List> getActions() { - return Collections.singletonList(new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class)); + return Collections.singletonList(new ActionHandler<>(TEST_ACTION, TransportTestTaskAction.class)); } @Override @@ -413,16 +418,6 @@ protected AllocatedPersistentTask createTask( } } - public static class TestTaskAction extends ActionType { - - public static final TestTaskAction INSTANCE = new TestTaskAction(); - public static final String NAME = "cluster:admin/persistent/task_test"; - - private TestTaskAction() { - super(NAME, TestTasksResponse::new); - } - } - public static class TestTask extends AllocatedPersistentTask { private volatile String operation; @@ -446,9 +441,7 @@ public String toString() { static class TestTaskResponse implements Writeable { - TestTaskResponse() { - - } + TestTaskResponse() {} TestTaskResponse(StreamInput in) throws IOException { in.readBoolean(); @@ -489,7 +482,7 @@ public String getOperation() { public static class TestTasksRequestBuilder extends TasksRequestBuilder { protected TestTasksRequestBuilder(ElasticsearchClient client) { - super(client, TestTaskAction.INSTANCE, new TestTasksRequest()); + super(client, TEST_ACTION, new TestTasksRequest()); } public TestTasksRequestBuilder setOperation(String operation) { @@ -536,7 +529,7 @@ public static class TransportTestTaskAction extends TransportTasksAction< @Inject public TransportTestTaskAction(ClusterService clusterService, TransportService transportService, ActionFilters actionFilters) { super( - TestTaskAction.NAME, + TEST_ACTION.name(), clusterService, transportService, actionFilters, diff --git a/test/external-modules/seek-tracking-directory/src/internalClusterTest/java/org/elasticsearch/test/seektracker/SeekTrackerPluginIT.java b/test/external-modules/seek-tracking-directory/src/internalClusterTest/java/org/elasticsearch/test/seektracker/SeekTrackerPluginIT.java index e942937922603..784730efff999 100644 --- a/test/external-modules/seek-tracking-directory/src/internalClusterTest/java/org/elasticsearch/test/seektracker/SeekTrackerPluginIT.java +++ b/test/external-modules/seek-tracking-directory/src/internalClusterTest/java/org/elasticsearch/test/seektracker/SeekTrackerPluginIT.java @@ -47,7 +47,7 @@ public void testSeekTrackerPlugin() throws InterruptedException { client().prepareSearch("index").setQuery(QueryBuilders.termQuery("field", "term2")).get(); - SeekStatsResponse response = client().execute(SeekStatsAction.INSTANCE, new SeekStatsRequest("index")).actionGet(); + SeekStatsResponse response = client().execute(SeekTrackerPlugin.SEEK_STATS_ACTION, new SeekStatsRequest("index")).actionGet(); List shardSeekStats = response.getSeekStats().get("index"); assertThat(shardSeekStats.size(), greaterThan(0)); } diff --git a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/RestSeekStatsAction.java b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/RestSeekStatsAction.java index 6654077229196..8695a08ce06ae 100644 --- a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/RestSeekStatsAction.java +++ b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/RestSeekStatsAction.java @@ -36,6 +36,6 @@ public List routes() { protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { String[] indices = request.paramAsStringArray("index", Strings.EMPTY_ARRAY); SeekStatsRequest seekStatsRequest = new SeekStatsRequest(indices); - return channel -> client.execute(SeekStatsAction.INSTANCE, seekStatsRequest, new RestToXContentListener<>(channel)); + return channel -> client.execute(SeekTrackerPlugin.SEEK_STATS_ACTION, seekStatsRequest, new RestToXContentListener<>(channel)); } } diff --git a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekStatsAction.java b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekStatsAction.java deleted file mode 100644 index 258cbaec3281c..0000000000000 --- a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekStatsAction.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.test.seektracker; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.common.io.stream.Writeable; - -public class SeekStatsAction extends ActionType { - - public static final SeekStatsAction INSTANCE = new SeekStatsAction(); - public static final String NAME = "cluster:monitor/seek_stats"; - - public SeekStatsAction() { - super(NAME, Writeable.Reader.localOnly()); - } -} diff --git a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java index cc68a3e3ba0cd..de2ac43f7fb51 100644 --- a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java +++ b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -50,6 +51,8 @@ public class SeekTrackerPlugin extends Plugin implements ActionPlugin { Setting.Property.NodeScope ); + public static final ActionType SEEK_STATS_ACTION = ActionType.localOnly("cluster:monitor/seek_stats"); + private final SeekStatsService seekStatsService = new SeekStatsService(); private final boolean enabled; @@ -112,7 +115,7 @@ public List getRestHandlers( @Override public List> getActions() { if (enabled) { - return Collections.singletonList(new ActionHandler<>(SeekStatsAction.INSTANCE, TransportSeekStatsAction.class)); + return Collections.singletonList(new ActionHandler<>(SEEK_STATS_ACTION, TransportSeekStatsAction.class)); } else { return Collections.emptyList(); } diff --git a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/TransportSeekStatsAction.java b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/TransportSeekStatsAction.java index f77a31389bd1f..bd1c35302b043 100644 --- a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/TransportSeekStatsAction.java +++ b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/TransportSeekStatsAction.java @@ -37,7 +37,7 @@ public TransportSeekStatsAction( SeekStatsService seekStatsService ) { super( - SeekStatsAction.NAME, + SeekTrackerPlugin.SEEK_STATS_ACTION.name(), clusterService, transportService, actionFilters, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index ea4350262bf41..477a5e014c105 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -65,7 +65,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.rest.root.MainAction; +import org.elasticsearch.rest.root.MainRestPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.XPackPlugin; @@ -1489,7 +1489,7 @@ public void testMonitoringUserRole() { ) ) ); - assertThat(monitoringUserRole.cluster().check(MainAction.NAME, request, authentication), is(true)); + assertThat(monitoringUserRole.cluster().check(MainRestPlugin.MAIN_ACTION.name(), request, authentication), is(true)); assertThat(monitoringUserRole.cluster().check(XPackInfoAction.NAME, request, authentication), is(true)); assertThat(monitoringUserRole.cluster().check(TransportRemoteInfoAction.TYPE.name(), request, authentication), is(true)); assertThat(monitoringUserRole.cluster().check(ClusterHealthAction.NAME, request, authentication), is(false)); From 63179d760adbb7b202d49150e1ee9c4fcb193b9e Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Wed, 18 Oct 2023 15:35:51 +0200 Subject: [PATCH 19/23] [Fleet] add fleet-server-remote service account (#100950) * add fleet-server-remote service account * fixed test * fix test * added monitor privilege --- .../authc/service/ElasticServiceAccounts.java | 25 +++++++++++++++++-- ...TransportGetServiceAccountActionTests.java | 4 +-- .../service/ServiceAccountServiceTests.java | 2 +- .../test/service_accounts/10_basic.yml | 6 +++-- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java index 95dc21185db5e..a33c45adf814e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java @@ -150,13 +150,34 @@ final class ElasticServiceAccounts { null ) ); + private static final ServiceAccount FLEET_REMOTE_ACCOUNT = new ElasticServiceAccount( + "fleet-server-remote", + new RoleDescriptor( + NAMESPACE + "/fleet-server-remote", + new String[] { "monitor", "manage_own_api_key" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder() + .indices("logs-*", "metrics-*") + .privileges("write", "create_index", "auto_configure") + .build(), }, + null, + null, + null, + null, + null + ) + ); private static final ServiceAccount KIBANA_SYSTEM_ACCOUNT = new ElasticServiceAccount( "kibana", ReservedRolesStore.kibanaSystemRoleDescriptor(NAMESPACE + "/kibana") ); - static final Map ACCOUNTS = Stream.of(ENTERPRISE_SEARCH_ACCOUNT, FLEET_ACCOUNT, KIBANA_SYSTEM_ACCOUNT) - .collect(Collectors.toMap(a -> a.id().asPrincipal(), Function.identity())); + static final Map ACCOUNTS = Stream.of( + ENTERPRISE_SEARCH_ACCOUNT, + FLEET_ACCOUNT, + FLEET_REMOTE_ACCOUNT, + KIBANA_SYSTEM_ACCOUNT + ).collect(Collectors.toMap(a -> a.id().asPrincipal(), Function.identity())); private ElasticServiceAccounts() {} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java index e8d178efe6b4a..b313d94a46ce5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java @@ -47,12 +47,12 @@ public void testDoExecute() { final PlainActionFuture future1 = new PlainActionFuture<>(); transportGetServiceAccountAction.doExecute(mock(Task.class), request1, future1); final GetServiceAccountResponse getServiceAccountResponse1 = future1.actionGet(); - assertThat(getServiceAccountResponse1.getServiceAccountInfos().length, equalTo(3)); + assertThat(getServiceAccountResponse1.getServiceAccountInfos().length, equalTo(4)); assertThat( Arrays.stream(getServiceAccountResponse1.getServiceAccountInfos()) .map(ServiceAccountInfo::getPrincipal) .collect(Collectors.toList()), - containsInAnyOrder("elastic/enterprise-search-server", "elastic/fleet-server", "elastic/kibana") + containsInAnyOrder("elastic/enterprise-search-server", "elastic/fleet-server", "elastic/fleet-server-remote", "elastic/kibana") ); final GetServiceAccountRequest request2 = new GetServiceAccountRequest("elastic", "fleet-server"); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountServiceTests.java index 0c40e3996d288..89a6684108149 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountServiceTests.java @@ -96,7 +96,7 @@ public void stopThreadPool() { public void testGetServiceAccountPrincipals() { assertThat( ServiceAccountService.getServiceAccountPrincipals(), - containsInAnyOrder("elastic/enterprise-search-server", "elastic/fleet-server", "elastic/kibana") + containsInAnyOrder("elastic/enterprise-search-server", "elastic/fleet-server", "elastic/fleet-server-remote", "elastic/kibana") ); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/service_accounts/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/service_accounts/10_basic.yml index 5c6d2d0c78275..47d6cdec2858b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/service_accounts/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/service_accounts/10_basic.yml @@ -31,17 +31,19 @@ teardown: "Test get service accounts": - do: security.get_service_accounts: {} - - length: { '': 3 } + - length: { '': 4 } - is_true: "elastic/enterprise-search-server" - is_true: "elastic/fleet-server" + - is_true: "elastic/fleet-server-remote" - is_true: "elastic/kibana" - do: security.get_service_accounts: namespace: elastic - - length: { '': 3 } + - length: { '': 4 } - is_true: "elastic/enterprise-search-server" - is_true: "elastic/fleet-server" + - is_true: "elastic/fleet-server-remote" - is_true: "elastic/kibana" - do: From 2a6c16079c464daf1c5f656a1f60b9ec57e066c1 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 18 Oct 2023 07:39:39 -0700 Subject: [PATCH 20/23] Migrate painless and runtime fields common tests to new test clusters (#101021) --- modules/lang-painless/build.gradle | 5 +++-- .../LangPainlessClientYamlTestSuiteIT.java | 15 +++++++++++++++ modules/runtime-fields-common/build.gradle | 4 ++-- .../RuntimeFieldsClientYamlTestSuiteIT.java | 10 ++++++++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 7dadb8da8efee..cc557ac2289f6 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -9,8 +9,8 @@ import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask; apply plugin: 'elasticsearch.validate-rest-spec' -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -42,6 +42,7 @@ dependencies { api 'org.ow2.asm:asm-analysis:7.2' api 'org.ow2.asm:asm:7.2' spi project('spi') + clusterModules project(':modules:mapper-extras') } tasks.named("dependencyLicenses").configure { diff --git a/modules/lang-painless/src/yamlRestTest/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java b/modules/lang-painless/src/yamlRestTest/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java index 2c1e86d303d9b..04c944cbc8874 100644 --- a/modules/lang-painless/src/yamlRestTest/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java +++ b/modules/lang-painless/src/yamlRestTest/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java @@ -11,12 +11,22 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; /** Runs yaml rest tests */ public class LangPainlessClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("lang-painless") + .module("mapper-extras") + .systemProperty("es.scripting.update.ctx_in_params", "false") + .systemProperty("es.transport.cname_in_publish_address", "true") + .build(); + public LangPainlessClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +35,9 @@ public LangPainlessClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/modules/runtime-fields-common/build.gradle b/modules/runtime-fields-common/build.gradle index c4db67a89d364..5a2d268cf7a4e 100644 --- a/modules/runtime-fields-common/build.gradle +++ b/modules/runtime-fields-common/build.gradle @@ -7,8 +7,8 @@ */ apply plugin: 'elasticsearch.validate-rest-spec' -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'Module for runtime fields features and extensions that have large dependencies' diff --git a/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java b/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java index af6e8f8242453..1304148911b4c 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java +++ b/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java @@ -11,12 +11,17 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; /** Runs yaml rest tests */ public class RuntimeFieldsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("runtime-fields-common").build(); + public RuntimeFieldsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public RuntimeFieldsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } From 12cd129e6222e28e48deed7c4065ebe1b550020d Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 18 Oct 2023 10:42:17 -0400 Subject: [PATCH 21/23] unmuting test (#101007) it seems that https://github.com/elastic/elasticsearch/issues/96896 was fixed but the test was never unmuted. Or some race condition in folks muting/fixing/unmuting/remuting occurred. --- .../elasticsearch/search/builder/SearchSourceBuilderTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 83a3497e44259..85f6d0a718e48 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -78,7 +78,6 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase { - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/96896") public void testFromXContent() throws IOException { SearchSourceBuilder testSearchSourceBuilder = createSearchSourceBuilder(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); From b35c043407d8c080de03e8c94dc5af04e9f5f41c Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 18 Oct 2023 07:46:21 -0700 Subject: [PATCH 22/23] Run driver with user context (#100724) Today, we have a hierarchy of tasks in ESQL designed to leverage the task framework for reporting status and cancellation. ```mermaid flowchart RESTLayer -->| EsqlQueryRequest indices:data/read/esql | ComputeService ComputeService -->| DriverRequest indices:data/read/esql/compute | Driver ComputeService -->| DataNodeRequest indices:data/read/esql/data | DataNode DataNode -->| DriverRequest indices:data/read/esql/compute | Driver Driver -->| LookupRequest indices:data/read/esql/lookup | EnrichLookupService ``` The primary issue here is that `DriverRequest` is neither `IndicesRequest` nor `CompositeIndicesRequest`. Consequently, the Driver is executed within the context of the system user, leading to access indices with the system user. To address this issue, this PR makes `DriverRequest` a `CompositeIndicesRequest` and ensures that the Driver executes within the user's context. With this fix we can now properly capture the response headers when a Driver is yielded and rescheduled. Relates #100707 Relates #99646 Relates #99926 Closes #100164 --- .../compute/operator/Driver.java | 30 ++- .../compute/operator/DriverTaskRunner.java | 10 +- .../compute/operator/AsyncOperatorTests.java | 2 +- .../compute/operator/DriverTests.java | 182 ++++++++++++++++++ .../operator/ForkingOperatorTestCase.java | 4 +- .../compute/operator/OperatorTestCase.java | 2 +- .../exchange/ExchangeServiceTests.java | 2 +- .../xpack/esql/EsqlSecurityIT.java | 1 - .../xpack/esql/lookup/EnrichLookupIT.java | 2 +- .../esql/enrich/EnrichLookupService.java | 3 +- .../elasticsearch/xpack/esql/CsvTests.java | 8 +- .../xpack/security/authz/RBACEngine.java | 1 + 12 files changed, 228 insertions(+), 19 deletions(-) create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java index 1a1604406892c..be3ee5ff40792 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java @@ -8,8 +8,10 @@ package org.elasticsearch.compute.operator; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.compute.Describable; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Nullable; @@ -252,9 +254,15 @@ private void ensureNotCancelled() { } } - public static void start(Executor executor, Driver driver, int maxIterations, ActionListener listener) { + public static void start( + ThreadContext threadContext, + Executor executor, + Driver driver, + int maxIterations, + ActionListener listener + ) { driver.status.set(driver.updateStatus(DriverStatus.Status.STARTING)); - schedule(DEFAULT_TIME_BEFORE_YIELDING, maxIterations, executor, driver, listener); + schedule(DEFAULT_TIME_BEFORE_YIELDING, maxIterations, threadContext, executor, driver, listener); } // Drains all active operators and closes them. @@ -274,8 +282,16 @@ private void drainAndCloseOperators(@Nullable Exception e) { Releasables.closeWhileHandlingException(releasable); } - private static void schedule(TimeValue maxTime, int maxIterations, Executor executor, Driver driver, ActionListener listener) { + private static void schedule( + TimeValue maxTime, + int maxIterations, + ThreadContext threadContext, + Executor executor, + Driver driver, + ActionListener listener + ) { executor.execute(new AbstractRunnable() { + @Override protected void doRun() { if (driver.isFinished()) { @@ -284,16 +300,18 @@ protected void doRun() { } SubscribableListener fut = driver.run(maxTime, maxIterations); if (fut.isDone()) { - schedule(maxTime, maxIterations, executor, driver, listener); + schedule(maxTime, maxIterations, threadContext, executor, driver, listener); } else { synchronized (driver) { if (driver.isCancelled() == false) { driver.blocked.set(fut); } } - fut.addListener( - ActionListener.wrap(ignored -> schedule(maxTime, maxIterations, executor, driver, listener), this::onFailure) + ActionListener readyListener = ActionListener.wrap( + ignored -> schedule(maxTime, maxIterations, threadContext, executor, driver, listener), + this::onFailure ); + fut.addListener(ContextPreservingActionListener.wrapPreservingContext(readyListener, threadContext)); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java index 221be19cc2871..b486318f85405 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -33,12 +34,12 @@ * A {@link DriverRunner} that executes {@link Driver} with a child task so that we can retrieve the progress with the Task API. */ public class DriverTaskRunner { - public static final String ACTION_NAME = "internal:data/read/esql/compute"; + public static final String ACTION_NAME = "indices:data/read/esql/compute"; private final TransportService transportService; public DriverTaskRunner(TransportService transportService, Executor executor) { this.transportService = transportService; - transportService.registerRequestHandler(ACTION_NAME, executor, DriverRequest::new, new DriverRequestHandler()); + transportService.registerRequestHandler(ACTION_NAME, executor, DriverRequest::new, new DriverRequestHandler(transportService)); } public void executeDrivers(Task parentTask, List drivers, Executor executor, ActionListener listener) { @@ -58,7 +59,7 @@ protected void start(Driver driver, ActionListener driverListener) { runner.runToCompletion(drivers, listener); } - private static class DriverRequest extends ActionRequest { + private static class DriverRequest extends ActionRequest implements CompositeIndicesRequest { private final Driver driver; private final Executor executor; @@ -107,11 +108,12 @@ public Status getStatus() { } } - private record DriverRequestHandler() implements TransportRequestHandler { + private record DriverRequestHandler(TransportService transportService) implements TransportRequestHandler { @Override public void messageReceived(DriverRequest request, TransportChannel channel, Task task) { var listener = new ChannelActionListener(channel); Driver.start( + transportService.getThreadPool().getThreadContext(), request.executor, request.driver, Driver.DEFAULT_MAX_ITERATIONS, diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java index bc78e7ce720fd..4c808907cda91 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java @@ -121,7 +121,7 @@ public void close() { }); PlainActionFuture future = new PlainActionFuture<>(); Driver driver = new Driver(driverContext, sourceOperator, List.of(asyncOperator), outputOperator, () -> assertFalse(it.hasNext())); - Driver.start(threadPool.executor(ESQL_TEST_EXECUTOR), driver, between(1, 10000), future); + Driver.start(threadPool.getThreadContext(), threadPool.executor(ESQL_TEST_EXECUTOR), driver, between(1, 10000), future); future.actionGet(); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java new file mode 100644 index 0000000000000..8640f2c32133b --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.compute.data.BasicBlockTests; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DriverTests extends ESTestCase { + + public void testThreadContext() { + DriverContext driverContext = driverContext(); + ThreadPool threadPool = threadPool(); + try { + List inPages = randomList(1, 100, DriverTests::randomPage); + List outPages = new ArrayList<>(); + WarningsOperator warning1 = new WarningsOperator(threadPool); + WarningsOperator warning2 = new WarningsOperator(threadPool); + Driver driver = new Driver(driverContext, new CannedSourceOperator(inPages.iterator()) { + @Override + public Page getOutput() { + assertRunningWithRegularUser(threadPool); + return super.getOutput(); + } + }, List.of(warning1, new SwitchContextOperator(threadPool), warning2), new PageConsumerOperator(page -> { + assertRunningWithRegularUser(threadPool); + outPages.add(page); + }), () -> {}); + ThreadContext threadContext = threadPool.getThreadContext(); + SubscribableListener future = new SubscribableListener<>(); + try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { + threadContext.putHeader("user", "user1"); + Driver.start(threadContext, threadPool.executor("esql"), driver, between(1, 1000), future); + } + future.addListener(ActionListener.running(() -> { + assertRunningWithRegularUser(threadPool); + assertThat(outPages, equalTo(inPages)); + Map> actualResponseHeaders = new HashMap<>(); + for (Map.Entry> e : threadPool.getThreadContext().getResponseHeaders().entrySet()) { + actualResponseHeaders.put(e.getKey(), Sets.newHashSet(e.getValue())); + } + Map> expectedResponseHeaders = new HashMap<>(warning1.warnings); + for (Map.Entry> e : warning2.warnings.entrySet()) { + expectedResponseHeaders.merge(e.getKey(), e.getValue(), Sets::union); + } + assertThat(actualResponseHeaders, equalTo(expectedResponseHeaders)); + })); + PlainActionFuture completion = new PlainActionFuture<>(); + future.addListener(completion); + completion.actionGet(TimeValue.timeValueSeconds(30)); + } finally { + terminate(threadPool); + } + } + + private static void assertRunningWithRegularUser(ThreadPool threadPool) { + String user = threadPool.getThreadContext().getHeader("user"); + assertThat(user, equalTo("user1")); + } + + private static Page randomPage() { + BasicBlockTests.RandomBlock block = BasicBlockTests.randomBlock( + randomFrom(ElementType.BOOLEAN, ElementType.INT, ElementType.BYTES_REF), + between(1, 10), + randomBoolean(), + 1, + between(1, 2), + 0, + 2 + ); + return new Page(block.block()); + } + + static class SwitchContextOperator extends AsyncOperator { + private final ThreadPool threadPool; + + SwitchContextOperator(ThreadPool threadPool) { + super(between(1, 3)); + this.threadPool = threadPool; + } + + @Override + protected void performAsync(Page page, ActionListener listener) { + assertRunningWithRegularUser(threadPool); + if (randomBoolean()) { + listener.onResponse(page); + return; + } + threadPool.schedule(ActionRunnable.wrap(listener, innerListener -> { + try (ThreadContext.StoredContext ignored = threadPool.getThreadContext().stashContext()) { + threadPool.getThreadContext().putHeader("user", "system"); + innerListener.onResponse(page); + } + }), TimeValue.timeValueNanos(100), threadPool.executor("esql")); + } + + @Override + public void close() { + + } + } + + static class WarningsOperator extends AbstractPageMappingOperator { + private final ThreadPool threadPool; + private final Map> warnings = new HashMap<>(); + + WarningsOperator(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + @Override + protected Page process(Page page) { + assertRunningWithRegularUser(threadPool); + if (randomInt(100) < 10) { + String k = "header-" + between(1, 10); + Set vs = Sets.newHashSet(randomList(1, 2, () -> "value-" + between(1, 20))); + warnings.merge(k, vs, Sets::union); + for (String v : vs) { + threadPool.getThreadContext().addResponseHeader(k, v); + } + } + return page; + } + + @Override + public String toString() { + return "WarningsOperator"; + } + + @Override + public void close() { + + } + } + + private ThreadPool threadPool() { + int numThreads = randomIntBetween(1, 10); + return new TestThreadPool( + getTestClass().getSimpleName(), + new FixedExecutorBuilder(Settings.EMPTY, "esql", numThreads, 1024, "esql", EsExecutors.TaskTrackingConfig.DEFAULT) + ); + } + + private DriverContext driverContext() { + MockBigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofGb(1)); + CircuitBreaker breaker = bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST); + BlockFactory blockFactory = new BlockFactory(breaker, bigArrays); + return new DriverContext(bigArrays, blockFactory); + } + +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java index e9df32282bfeb..1a2c87aff1591 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java @@ -169,7 +169,7 @@ public final void testManyInitialManyPartialFinalRunner() { var runner = new DriverRunner(threadPool.getThreadContext()) { @Override protected void start(Driver driver, ActionListener listener) { - Driver.start(threadPool.executor(ESQL_TEST_EXECUTOR), driver, between(1, 10000), listener); + Driver.start(threadPool.getThreadContext(), threadPool.executor(ESQL_TEST_EXECUTOR), driver, between(1, 10000), listener); } }; PlainActionFuture future = new PlainActionFuture<>(); @@ -193,7 +193,7 @@ public final void testManyInitialManyPartialFinalRunnerThrowing() throws Excepti var runner = new DriverRunner(threadPool.getThreadContext()) { @Override protected void start(Driver driver, ActionListener listener) { - Driver.start(threadPool.executor(ESQL_TEST_EXECUTOR), driver, between(1, 1000), listener); + Driver.start(threadPool.getThreadContext(), threadPool.executor(ESQL_TEST_EXECUTOR), driver, between(1, 1000), listener); } }; PlainActionFuture future = new PlainActionFuture<>(); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java index 78fa3a8e5de19..50b97021c216b 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java @@ -296,7 +296,7 @@ public static void runDriver(List drivers) { var driverRunner = new DriverRunner(threadPool.getThreadContext()) { @Override protected void start(Driver driver, ActionListener driverListener) { - Driver.start(threadPool.executor("esql"), driver, between(1, 10000), driverListener); + Driver.start(threadPool.getThreadContext(), threadPool.executor("esql"), driver, between(1, 10000), driverListener); } }; PlainActionFuture future = new PlainActionFuture<>(); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java index 78042a8587350..1b98d69c313ca 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java @@ -309,7 +309,7 @@ void runConcurrentTest( new DriverRunner(threadPool.getThreadContext()) { @Override protected void start(Driver driver, ActionListener listener) { - Driver.start(threadPool.executor(ESQL_TEST_EXECUTOR), driver, between(1, 10000), listener); + Driver.start(threadPool.getThreadContext(), threadPool.executor(ESQL_TEST_EXECUTOR), driver, between(1, 10000), listener); } }.runToCompletion(drivers, future); future.actionGet(TimeValue.timeValueMinutes(1)); diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java index e067cf271478f..0cd9570927635 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java @@ -119,7 +119,6 @@ public void testRowCommand() throws Exception { assertThat(respMap.get("values"), equalTo(List.of(List.of(2, 5)))); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/100724/") public void testEnrich() throws Exception { createEnrichPolicy(); try { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java index f9d97cbd910e0..56ea27e360c1d 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java @@ -146,7 +146,7 @@ public void testSimple() { @Override protected void start(Driver driver, ActionListener listener) { - Driver.start(executor, driver, between(1, 1000), listener); + Driver.start(transportService.getThreadPool().getThreadContext(), executor, driver, between(1, 1000), listener); } }; Driver driver = new Driver(driverContext(), sourceOperator, List.of(enrichOperator), outputOperator, () -> {}); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 22898b6f846b3..af8732ad9c969 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -283,7 +283,8 @@ private void doLookup( driver.cancel(reason); }); - Driver.start(executor, driver, Driver.DEFAULT_MAX_ITERATIONS, listener.map(ignored -> { + var threadContext = transportService.getThreadPool().getThreadContext(); + Driver.start(threadContext, executor, driver, Driver.DEFAULT_MAX_ITERATIONS, listener.map(ignored -> { Page out = result.get(); if (out == null) { out = createNullResponse(inputPage.getPositionCount(), extractFields); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 67f5d12756e71..b0b7bf17d2ab4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -381,7 +381,13 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception { DriverRunner runner = new DriverRunner(threadPool.getThreadContext()) { @Override protected void start(Driver driver, ActionListener driverListener) { - Driver.start(threadPool.executor(ESQL_THREAD_POOL_NAME), driver, between(1, 1000), driverListener); + Driver.start( + threadPool.getThreadContext(), + threadPool.executor(ESQL_THREAD_POOL_NAME), + driver, + between(1, 1000), + driverListener + ); } }; PlainActionFuture future = new PlainActionFuture<>(); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index d89202909da27..1bb638795615a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -267,6 +267,7 @@ private static boolean shouldAuthorizeIndexActionNameOnly(String action, Transpo case "indices:data/read/sql": case "indices:data/read/sql/translate": case "indices:data/read/esql": + case "indices:data/read/esql/compute": if (request instanceof BulkShardRequest) { return false; } From 49d3baadee416121885624de4e6e70e7d5589d95 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Wed, 18 Oct 2023 10:47:09 -0400 Subject: [PATCH 23/23] The 'too many scroll requests' exception should return 429 status (#100968) --- .../elasticsearch/ElasticsearchException.java | 7 ++++ .../org/elasticsearch/TransportVersions.java | 2 + .../elasticsearch/search/SearchService.java | 9 +---- .../TooManyScrollContextsException.java | 38 +++++++++++++++++++ .../ExceptionSerializationTests.java | 2 + .../search/SearchServiceTests.java | 1 + 6 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/TooManyScrollContextsException.java diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java index 260af26925deb..a185bee34c862 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -32,6 +32,7 @@ import org.elasticsearch.rest.ApiNotAvailableException; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchException; +import org.elasticsearch.search.TooManyScrollContextsException; import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.search.aggregations.UnsupportedAggregationOnDownsampledIndex; import org.elasticsearch.transport.TcpTransport; @@ -1854,6 +1855,12 @@ private enum ElasticsearchExceptionHandle { RecoveryCommitTooNewException::new, 172, TransportVersions.RECOVERY_COMMIT_TOO_NEW_EXCEPTION_ADDED + ), + TOO_MANY_SCROLL_CONTEXTS_NEW_EXCEPTION( + TooManyScrollContextsException.class, + TooManyScrollContextsException::new, + 173, + TransportVersions.TOO_MANY_SCROLL_CONTEXTS_EXCEPTION_ADDED ); final Class exceptionClass; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index eccdd791c8244..a3f36c6a4b6fb 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -143,6 +143,8 @@ static TransportVersion def(int id) { public static final TransportVersion BUILD_QUALIFIER_SEPARATED = def(8_518_00_0); public static final TransportVersion PIPELINES_IN_BULK_RESPONSE_ADDED = def(8_519_00_0); public static final TransportVersion PLUGIN_DESCRIPTOR_STRING_VERSION = def(8_520_00_0); + public static final TransportVersion TOO_MANY_SCROLL_CONTEXTS_EXCEPTION_ADDED = def(8_521_00_0); + /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 88487f528096c..dafcf45454aaf 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -945,14 +945,7 @@ final ReaderContext createAndPutReaderContext( if (request.scroll() != null) { decreaseScrollContexts = openScrollContexts::decrementAndGet; if (openScrollContexts.incrementAndGet() > maxOpenScrollContext) { - throw new ElasticsearchException( - "Trying to create too many scroll contexts. Must be less than or equal to: [" - + maxOpenScrollContext - + "]. " - + "This limit can be set by changing the [" - + MAX_OPEN_SCROLL_CONTEXT.getKey() - + "] setting." - ); + throw new TooManyScrollContextsException(maxOpenScrollContext, MAX_OPEN_SCROLL_CONTEXT.getKey()); } } final ShardSearchContextId id = new ShardSearchContextId(sessionId, idGenerator.incrementAndGet()); diff --git a/server/src/main/java/org/elasticsearch/search/TooManyScrollContextsException.java b/server/src/main/java/org/elasticsearch/search/TooManyScrollContextsException.java new file mode 100644 index 0000000000000..e45d4aa01086b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/TooManyScrollContextsException.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; + +public class TooManyScrollContextsException extends ElasticsearchException { + + public TooManyScrollContextsException(int maxOpenScrollCount, String maxOpenScrollSettingName) { + super( + "Trying to create too many scroll contexts. Must be less than or equal to: [" + + maxOpenScrollCount + + "]. " + + "This limit can be set by changing the [" + + maxOpenScrollSettingName + + "] setting." + ); + } + + public TooManyScrollContextsException(StreamInput in) throws IOException { + super(in); + } + + @Override + public RestStatus status() { + return RestStatus.TOO_MANY_REQUESTS; + } +} diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 45d3233e29ee0..e5d1201c855ed 100644 --- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -75,6 +75,7 @@ import org.elasticsearch.search.SearchContextMissingException; import org.elasticsearch.search.SearchException; import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.TooManyScrollContextsException; import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.search.aggregations.UnsupportedAggregationOnDownsampledIndex; import org.elasticsearch.search.internal.ShardSearchContextId; @@ -834,6 +835,7 @@ public void testIds() { ids.put(170, ElasticsearchRoleRestrictionException.class); ids.put(171, ApiNotAvailableException.class); ids.put(172, RecoveryCommitTooNewException.class); + ids.put(173, TooManyScrollContextsException.class); Map, Integer> reverse = new HashMap<>(); for (Map.Entry> entry : ids.entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index e12163f5cb7e9..ae462d9115abf 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -818,6 +818,7 @@ public void testMaxOpenScrollContexts() throws Exception { + "This limit can be set by changing the [search.max_open_scroll_context] setting.", ex.getMessage() ); + assertEquals(RestStatus.TOO_MANY_REQUESTS, ex.status()); service.freeAllScrollContexts(); }